@stratta/mcp 0.3.0 → 0.4.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@stratta/mcp",
   "mcpName": "io.github.hugogebel-boop/stratta",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "MCP server exposing Swiss engineering norms (SIA / Eurocodes) to Claude clients via Stratta TreeRAG.",
   "license": "UNLICENSED",
   "author": "Hugo Gebel <hugo.gebel@epfl.ch>",
@@ -29,6 +29,7 @@
   "files": [
     "dist",
     "skills",
+    "scripts",
     "README.md"
   ],
   "scripts": {

package/scripts/ingest-prepass.py

@@ -0,0 +1,310 @@
+#!/usr/bin/env python3
+"""Stratta — ingest pre-pass.
+
+Reads a norm PDF with PyMuPDF, builds the hierarchical TreeRAG skeleton (chapters
+from PDF bookmarks + sub-sections via heading regex), extracts per-section raw
+text, and rasterizes each page that contains a figure caption. Output is a
+single JSON consumed by the `ingest-norm` skill, which then enriches sections
+(formulas, tables, cross-refs, summaries) and uploads the figures.
+
+Usage:
+    python scripts/ingest-prepass.py --pdf <path> --output <dir> [--language fr]
+
+Output (under <dir>/):
+    prepass.json       full extracted tree + figure manifest
+    figures/figure-<N>.png  rasterized full-page renders for each caption
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+try:
+    import fitz  # PyMuPDF
+except ImportError:
+    sys.stderr.write(
+        "ERROR: PyMuPDF not installed. Run: pip install --user pymupdf\n"
+    )
+    sys.exit(2)
+
+
+BM_NUM_RE = re.compile(r"^\s*(\d+)\s+(.+?)\s*$")
+BM_ANNEX_RE = re.compile(
+    r"^\s*(?:ANNEXE|Annexe)\s+([A-Z])(?:\s*\(([^)]+)\))?\s*(.*?)\s*$"
+)
+
+SUB_RE = re.compile(
+    r"^(\d+(?:\.\d+){1,3})\s+([A-Za-zÀ-ÿ][^\n]{1,140})$", re.MULTILINE
+)
+TOC_DOT_RE = re.compile(r"\.\s*\.\s*\.")
+CLEAN_DOTS = re.compile(r"\s*(\.\s*){3,}.*$")
+
+CAPTION_RE = re.compile(
+    r"^\s*(?:Figure|Fig\.|Bild|Abbildung|Figura)\s+(\d+[a-z]?)\b[\s:.\-—–]*(.*)$",
+    re.IGNORECASE | re.MULTILINE,
+)
+
+
+def clean(s: str) -> str:
+    return re.sub(r"\s+", " ", CLEAN_DOTS.sub("", s)).strip()
+
+
+def detect_toc_pages(doc: "fitz.Document") -> set[int]:
+    out = set()
+    for i in range(doc.page_count):
+        lines = [l for l in doc[i].get_text("text").splitlines() if l.strip()]
+        leader = sum(1 for l in lines if TOC_DOT_RE.search(l))
+        if leader >= 5:
+            out.add(i)
+    return out
+
+
+def detect_running_text(doc: "fitz.Document") -> set[str]:
+    """Lines appearing on >=5 pages near the top/bottom (running headers/footers)."""
+    counter: dict[str, int] = defaultdict(int)
+    for i in range(doc.page_count):
+        lines = [l.strip() for l in doc[i].get_text("text").splitlines() if l.strip()]
+        for l in lines[:3] + lines[-3:]:
+            counter[l] += 1
+    return {l for l, c in counter.items() if c >= 5 and len(l) > 8}
+
+
+def extract_chapters(doc: "fitz.Document") -> dict[str, dict]:
+    """Top-level chapters from PDF bookmarks (authoritative titles + pages)."""
+    chapters: dict[str, dict] = {}
+    for _depth, title, page in doc.get_toc(simple=True):
+        m = BM_ANNEX_RE.match(title)
+        if m:
+            letter, kind, rest = m.group(1), m.group(2) or "", (m.group(3) or "").strip()
+            path = f"Annexe {letter}"
+            t = path + (f" ({kind})" if kind else "")
+            if rest and rest.lower() != path.lower():
+                t += f" — {rest}"
+            chapters[path] = {"title": t, "pageStart": page}
+            continue
+        m = BM_NUM_RE.match(title)
+        if m:
+            num, rest = m.group(1), m.group(2).strip()
+            chapters[num] = {"title": rest, "pageStart": page}
+    return chapters
+
+
+def extract_subsections(
+    doc: "fitz.Document",
+    chap_prefixes: set[str],
+    toc_pages: set[int],
+    running: set[str],
+) -> dict[str, tuple[str, int]]:
+    """Numeric sub-sections (depths 2-4) detected in body pages."""
+    out: dict[str, tuple[str, int]] = {}
+    for i in range(doc.page_count):
+        if i in toc_pages:
+            continue
+        text = doc[i].get_text("text")
+        for m in SUB_RE.finditer(text):
+            num = m.group(1)
+            title = clean(m.group(2))
+            if not title or title in running:
+                continue
+            top = num.split(".", 1)[0]
+            if top not in chap_prefixes:
+                continue
+            if len(title) < 3 or re.fullmatch(r"[^A-Za-zÀ-ÿ]+", title):
+                continue
+            if num in out:
+                continue
+            out[num] = (title, i + 1)
+    return out
+
+
+def build_tree(chapters: dict, subsections: dict, n_pages: int) -> list[dict]:
+    """Returns flat list of nodes with depth 0-indexed (chapter=0, section=1, ...)."""
+    nodes: list[dict] = []
+    for path, info in chapters.items():
+        nodes.append(
+            {"depth": 0, "path": path, "title": info["title"], "pageStart": info["pageStart"]}
+        )
+    for path, (title, pg) in subsections.items():
+        nodes.append(
+            {"depth": path.count("."), "path": path, "title": title, "pageStart": pg}
+        )
+
+    def sort_key(n):
+        p = n["path"]
+        if p.startswith("Annexe"):
+            return (10**9, ord(p[-1]), [])
+        parts = [int(x) for x in p.split(".")]
+        return (parts[0], 0, parts)
+
+    nodes.sort(key=sort_key)
+
+    # pageEnd via next sibling/ancestor
+    for idx, node in enumerate(nodes):
+        d, pg = node["depth"], node["pageStart"]
+        next_pg = n_pages
+        for j in range(idx + 1, len(nodes)):
+            if nodes[j]["depth"] <= d:
+                next_pg = max(nodes[j]["pageStart"] - 1, pg)
+                break
+        node["pageEnd"] = next_pg
+
+    # Parent links + nodeId + orderIndex
+    parent_stack: list[tuple[int, str]] = []  # (depth, nodeId)
+    for idx, node in enumerate(nodes):
+        d = node["depth"]
+        while parent_stack and parent_stack[-1][0] >= d:
+            parent_stack.pop()
+        node["nodeId"] = f"s-{slugify(node['path'])}"
+        node["parentNodeId"] = parent_stack[-1][1] if parent_stack else None
+        node["orderIndex"] = idx
+        parent_stack.append((d, node["nodeId"]))
+
+    return nodes
+
+
+def slugify(path: str) -> str:
+    return re.sub(r"[^a-zA-Z0-9]+", "-", path).strip("-").lower() or "root"
+
+
+def extract_section_text(doc: "fitz.Document", nodes: list[dict]) -> None:
+    """Attach rawText to each node = concat of pages it spans."""
+    page_texts = [doc[i].get_text("text") for i in range(doc.page_count)]
+    for node in nodes:
+        ps, pe = node["pageStart"], node["pageEnd"]
+        chunks = page_texts[ps - 1 : pe]
+        node["rawText"] = "\n".join(chunks).strip()
+
+
+def extract_figures(
+    doc: "fitz.Document",
+    toc_pages: set[int],
+    out_dir: Path,
+    dpi: int = 150,
+) -> list[dict]:
+    """One figure manifest entry per caption detected. Page is rendered to PNG."""
+    figs_dir = out_dir / "figures"
+    figs_dir.mkdir(parents=True, exist_ok=True)
+    seen: dict[str, dict] = {}
+    matrix = fitz.Matrix(dpi / 72, dpi / 72)
+    for pageno in range(doc.page_count):
+        if pageno in toc_pages:
+            continue
+        text = doc[pageno].get_text("text")
+        rendered = False
+        page_png: Path | None = None
+        for m in CAPTION_RE.finditer(text):
+            num = m.group(1)
+            label_rest = m.group(2).strip()
+            if num in seen:
+                continue
+            caption = f"Figure {num}" + (f" — {label_rest}" if label_rest else "")
+            # render the page once if not done
+            if not rendered:
+                pix = doc[pageno].get_pixmap(matrix=matrix, alpha=False)
+                page_png = figs_dir / f"page-{pageno + 1:03d}.png"
+                pix.save(str(page_png))
+                rendered = True
+            seen[num] = {
+                "figureNumber": num,
+                "caption": caption[:240],
+                "page": pageno + 1,
+                "fileName": f"figures/{page_png.name}" if page_png else None,
+                "renderDpi": dpi,
+                "mimeType": "image/png",
+            }
+    return list(seen.values())
+
+
+def guess_language(doc: "fitz.Document") -> str:
+    """Best-effort language guess based on common French/German/Italian markers."""
+    sample = " ".join(doc[i].get_text("text") for i in range(min(5, doc.page_count))).lower()
+    scores = {
+        "fr": sum(sample.count(w) for w in (" la ", " les ", " sont ", " avec ", " selon ")),
+        "de": sum(sample.count(w) for w in (" der ", " die ", " sind ", " mit ", " nach ")),
+        "it": sum(sample.count(w) for w in (" la ", " sono ", " con ", " per ", " della ")),
+        "en": sum(sample.count(w) for w in (" the ", " are ", " with ", " of ", " shall ")),
+    }
+    return max(scores, key=scores.get)
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(description="Stratta ingest pre-pass (PyMuPDF).")
+    ap.add_argument("--pdf", required=True, help="Path to the norm PDF.")
+    ap.add_argument("--output", required=True, help="Output directory.")
+    ap.add_argument(
+        "--language",
+        choices=["fr", "de", "it", "en"],
+        help="Override detected language.",
+    )
+    ap.add_argument("--dpi", type=int, default=150, help="Figure render DPI.")
+    args = ap.parse_args()
+
+    pdf_path = Path(args.pdf).resolve()
+    out_dir = Path(args.output).resolve()
+    if not pdf_path.exists():
+        sys.stderr.write(f"PDF not found: {pdf_path}\n")
+        sys.exit(1)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    doc = fitz.open(str(pdf_path))
+    toc_pages = detect_toc_pages(doc)
+    running = detect_running_text(doc)
+    chapters = extract_chapters(doc)
+    chap_prefixes = {k for k in chapters if k.isdigit()}
+    subsections = extract_subsections(doc, chap_prefixes, toc_pages, running)
+    nodes = build_tree(chapters, subsections, doc.page_count)
+    extract_section_text(doc, nodes)
+    figures = extract_figures(doc, toc_pages, out_dir, dpi=args.dpi)
+
+    lang = args.language or guess_language(doc)
+
+    by_depth: dict[int, int] = defaultdict(int)
+    for n in nodes:
+        by_depth[n["depth"]] += 1
+
+    manifest = {
+        "doc": {
+            "sourcePath": str(pdf_path),
+            "pageCount": doc.page_count,
+            "language": lang,
+            "tocSource": "bookmarks+regex" if doc.get_toc() else "regex-only",
+            "metadata": {k: v for k, v in doc.metadata.items() if v},
+            "tocPagesDetected": sorted(toc_pages),
+            "runningHeadersDetected": sorted(running)[:10],
+        },
+        "stats": {
+            "sectionCount": len(nodes),
+            "byDepth": dict(by_depth),
+            "figureCount": len(figures),
+        },
+        "sections": nodes,
+        "figures": figures,
+    }
+
+    manifest_path = out_dir / "prepass.json"
+    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    sys.stdout.write(
+        json.dumps(
+            {
+                "ok": True,
+                "manifest": str(manifest_path),
+                "sectionCount": len(nodes),
+                "byDepth": dict(by_depth),
+                "figureCount": len(figures),
+                "language": lang,
+                "pageCount": doc.page_count,
+            },
+            ensure_ascii=False,
+        )
+        + "\n"
+    )
+
+
+if __name__ == "__main__":
+    main()

package/skills/ingest-norm/SKILL.md

@@ -1,75 +1,128 @@
----
-name: ingest-norm
-description: Use when the user wants to add an engineering norm (SIA, Eurocode, etc.) they are licensed for into THEIR Stratta workspace. Reads the PDF, builds a hierarchical tree, enriches sections (LaTeX formulas, tables, figures, cross-references), and writes everything to Stratta via the MCP `ingest_*` tools — scoped to the user's own organization. Trigger phrases: "ingère cette norme", "ingest SIA", "ajoute la norme X à Stratta".
----
-
-# Ingest a norm into your Stratta workspace
-
-This skill turns a norm PDF **you are licensed to use** into a queryable TreeRAG
-inside **your own** Stratta workspace. Everything runs on your machine through
-the Stratta MCP server (authenticated by your `STRATTA_API_KEY`); the norm is
-stored privately and scoped to your organization — no one else can see it.
-
-> ⚠️ **Licence**: only ingest norms your organization holds a valid licence for.
-> You are responsible for your usage rights (see Stratta's Terms).
-
-## Prerequisites
-- The Stratta MCP server is installed and `STRATTA_API_KEY` is set.
-- The norm PDF is available locally.
-- (Optional, for figures) `poppler` tools (`pdftoppm`, `pdfimages`) to rasterize
-  pages/extract images.
-
-## Tools used (all scoped to your workspace)
-`ingest_status` · `ingest_create_document` · `ingest_create_sections` ·
-`ingest_attach_formula` · `ingest_attach_table` · `ingest_attach_cross_ref` ·
-`ingest_upload_figure` · `ingest_normalize_cross_refs` · `ingest_publish` ·
-`ingest_delete`.
-
-## Workflow
-
-### 1. Check for an existing copy
-Call `ingest_status { code }` (e.g. `"SIA 261"`). If it already exists and you
-want to re-ingest, call `ingest_delete { documentId }` first.
-
-### 2. Read the PDF and plan the tree
-Read the PDF natively. Build a PageIndex-style hierarchy: chapters (depth 0),
-sections (depth 1), sub-sections (depth 2+), plus annexes. For each node decide
-a stable `nodeId`, a human `path` (e.g. `"4.2.1"`, `"Annexe B.1"`), `title`,
-a 1-3 sentence `summary` (used for navigation), `pageStart`/`pageEnd`, and an
-`orderIndex`. **Never skip annexes** — they hold key numeric values.
-
-### 3. Create the document
-`ingest_create_document { code, year, title, language, totalPages }` →
-returns `documentId`. Keep it for all subsequent calls.
-
-### 4. Insert sections (in batches)
-`ingest_create_sections { documentId, sections: [...] }`. Each section carries
-`nodeId`, optional `parentNodeId` (must already exist in this or a prior batch),
-`path`, `title`, `summary`, `depth`, `content` (full enriched text, LaTeX inline
-OK), `rawContent`, `pageStart`, `pageEnd`, `orderIndex`. The call returns a
-`nodeId → sectionId` map — **use those sectionIds** to attach formulas, tables,
-figures and cross-refs. Batch large docs (e.g. 30-50 sections per call).
-
-### 5. Enrich sections
-- `ingest_attach_formula { sectionId, latex, description, formulaNumber }`
-- `ingest_attach_table { sectionId, data: { headers, rows }, caption, tableNumber }`
-- `ingest_attach_cross_ref { sourceSectionId, targetDocumentCode, targetSectionPath?, refText, refType }`
-
-### 6. Figures
-For each relevant figure: rasterize/extract the image, read its bytes, base64-encode,
-then `ingest_upload_figure { sectionId, base64, mimeType, caption, figureNumber }`
-(png/jpeg/webp, ≤ 8 MB). The tool stores the image and links it in one call.
-
-### 7. Auto cross-references (optional)
-`ingest_normalize_cross_refs { documentId }` scans every section's text for
-references to other norms (SIA / SN EN / EN / ISO / DIN …) and rebuilds the
-cross-ref index. Idempotent.
-
-### 8. Publish
-`ingest_publish { documentId }`. The norm is now queryable in your workspace via
-`list_norms`, `get_toc`, `get_section`, `search_in_norm`, `get_figure`, etc.
-
-## Quality bar
-- Citations depend on accurate `path` + `pageStart/pageEnd` — get them right.
-- Summaries drive navigation — make them specific (mention formulas/values when present).
-- Keep `content` faithful to the source; don't invent values.
+---
+name: ingest-norm
+description: Use when the user wants to add an engineering norm (SIA, Eurocode, etc.) they are licensed for into THEIR Stratta workspace. A Python pre-pass (PyMuPDF) extracts the hierarchical tree and rasterizes figures; an agentic pass enriches sections (LaTeX formulas, tables, cross-references, summaries) and writes everything via the MCP `ingest_*` tools — scoped to the user's own organization. Trigger phrases: "ingère cette norme", "/ingest-norm", "ingest SIA", "ajoute la norme X à Stratta".
+---
+
+# Ingest a norm into your Stratta workspace
+
+This skill turns a norm PDF **you are licensed to use** into a queryable
+TreeRAG inside **your own** Stratta workspace. A Python pre-pass (PyMuPDF)
+does the deterministic heavy lifting — TOC tree, per-section raw text, figure
+captions and full-page renders. Then an agentic pass enriches sections with
+summaries, LaTeX formulas, structured tables, and cross-references. Final
+writes go through the Stratta MCP `ingest_*` tools, scoped to your org.
+
+> ⚠️ **Licence**: only ingest norms your organization holds a valid licence
+> for. You are responsible for your usage rights (see Stratta's Terms).
+
+## Prerequisites
+- The Stratta MCP server is installed and your `STRATTA_API_KEY` resolves
+  (via env, `~/.stratta/config.json`, or first-call elicitation).
+- **Python ≥ 3.10 with PyMuPDF**. Install once:
+  `python -m pip install --user pymupdf` (or `uv pip install pymupdf`).
+- The norm PDF is available locally.
+
+## Tools used (all scoped to your workspace)
+`ingest_status` · `ingest_create_document` · `ingest_create_sections` ·
+`ingest_attach_formula` · `ingest_attach_table` · `ingest_attach_cross_ref` ·
+`ingest_upload_figure` · `ingest_normalize_cross_refs` · `ingest_publish` ·
+`ingest_delete`.
+
+## Workflow
+
+### 1. Locate the pre-pass script
+It ships inside this package at `scripts/ingest-prepass.py`. Resolve its path:
+```bash
+node -e "console.log(require.resolve('@stratta/mcp/package.json'))"
+# → <root>/package.json  →  <root>/scripts/ingest-prepass.py
+```
+If the user is working in the Stratta monorepo, the script also lives at
+`packages/mcp/scripts/ingest-prepass.py`.
+
+### 2. Check for an existing copy
+`ingest_status { code }` (e.g. `"SIA 261"`). To re-ingest, call
+`ingest_delete { documentId }` first.
+
+### 3. Run the pre-pass
+```bash
+python <pkg-root>/scripts/ingest-prepass.py \
+  --pdf <path-to-pdf> \
+  --output .stratta-ingest/<code-slug>
+```
+Output under `.stratta-ingest/<code-slug>/`:
+- `prepass.json` — full manifest (see below).
+- `figures/page-NNN.png` — one PNG per page that contains a `Figure N` caption.
+
+`prepass.json` structure:
+- `doc` — `pageCount`, detected `language`, `tocSource`, raw `metadata`.
+- `stats` — `sectionCount`, `byDepth`, `figureCount`.
+- `sections[]` — full hierarchical tree (depth **0 = chapter**, 1+ = sub-sections),
+  each with `nodeId`, `parentNodeId`, `path` (`"4.2.1"` or `"Annexe B"`),
+  `title`, `depth`, `pageStart`, `pageEnd`, `orderIndex`, `rawText` (concat
+  of the pages the node spans).
+- `figures[]` — one entry per `Figure N` caption: `figureNumber`, `caption`,
+  `page`, `fileName`, `mimeType`.
+
+The pre-pass is **exhaustive** (e.g. ~550 nodes on SIA 261). You decide what
+to keep in the next step.
+
+### 4. Create the document
+Read `prepass.json`, then:
+`ingest_create_document { code, year, title, language: <doc.language>, totalPages: <doc.pageCount> }`
+→ returns `documentId`. Keep it for every subsequent call.
+
+### 5. Decide section granularity + generate summaries
+Iterate `sections[]` and decide what to keep. Two viable strategies:
+- **Keep all** — most faithful, ~500 sections on a typical SIA norm. Great
+  for fine-grained navigation but verbose.
+- **Aggregate trivial leaves** — fold paragraph-level nodes (`6.1.1`...`6.1.11`)
+  into their parent (`6.1`), concatenating their `rawText`. Typical result:
+  100-150 sections. Recommended unless the user asks for max granularity.
+
+For each kept section, prepare:
+- `summary` — 1-3 sentences derived from `rawText` (mention formulas/values).
+- `content` — enriched text with LaTeX inline where the source has math
+  (e.g. `$\sigma_d = f_{yd} \cdot \gamma$`). Open the PDF visually for pages
+  that contain formulas or multi-column tables — PyMuPDF mangles those.
+- `rawContent` — use the pre-pass `rawText` as-is.
+
+Keep the `nodeId` / `parentNodeId` / `path` / `pageStart` / `pageEnd` /
+`orderIndex` / `depth` from the pre-pass — those are deterministic.
+
+### 6. Insert sections (batched)
+`ingest_create_sections { documentId, sections: [...] }` in batches of 30-50.
+Parent links resolve via `parentNodeId` within the batch and across prior
+batches. The call returns a `nodeId → sectionId` map — **use those `sectionId`s**
+for every enrichment call below.
+
+### 7. Enrich
+- `ingest_attach_formula { sectionId, latex, description, formulaNumber }`
+- `ingest_attach_table { sectionId, data: { headers, rows }, caption, tableNumber }`
+- `ingest_attach_cross_ref { sourceSectionId, targetDocumentCode, targetSectionPath?, refText, refType }`
+
+### 8. Upload figures
+For each figure in `prepass.json#figures`:
+- Read `.stratta-ingest/<code>/<fileName>` and base64-encode the bytes.
+- Find the owning section: the kept section whose `pageStart..pageEnd`
+  range includes the figure's `page`.
+- `ingest_upload_figure { sectionId, base64, mimeType: "image/png", caption, figureNumber }`
+  (≤ 8 MB per image).
+
+### 9. Auto cross-references (optional but recommended)
+`ingest_normalize_cross_refs { documentId }` scans every section's text for
+references to other norms (SIA / SN EN / EN / ISO / DIN …) and rebuilds the
+cross-ref index. Idempotent.
+
+### 10. Publish
+`ingest_publish { documentId }`. The norm is now queryable in your workspace
+via `list_norms`, `get_toc`, `get_section`, `search_in_norm`, `get_figure`, etc.
+
+## Quality bar
+- Trust the pre-pass for `path` + `pageStart`/`pageEnd` — it's deterministic
+  and verified against the PDF bookmarks.
+- Summaries drive navigation — be specific (mention key formulas/values).
+- Keep `content` faithful to the source; don't invent values.
+- If PyMuPDF mangled a formula (Greek letters, fractions, exponents broken),
+  re-read the relevant PDF page visually and write proper LaTeX.
+- Never skip annexes — they hold key numeric values (zones, coefficients,
+  characteristic loads).