@sitar_fiercer4c/skills 0.1.0

package/skills/pdf-to-json/scripts/pdf_extract.py

@@ -0,0 +1,1313 @@
+#!/usr/bin/env python3
+"""Extract a PDF into structured JSON plus a readable Markdown rendering.
+
+    pdf_extract.py <pdf-or-dir> [--out-dir DIR] [--mode fast|balanced|accurate]
+
+Submits the PDF to the Datalab Convert API (datalab.to), polls until the
+conversion completes, then writes two files per input:
+
+    <out-dir>/<stem>.json   every block, every table, with page and bbox
+    <out-dir>/<stem>.md     the same document rendered for a human to read
+
+One API call produces both. `output_format=json` returns a block tree where
+each block carries its own self-contained HTML, so the Markdown is rendered
+locally rather than paid for a second time.
+
+Needs DATALAB_API_KEY in the environment or in a .env / .env.local found by
+walking up from the PDF's directory and from the working directory.
+"""
+
+import argparse
+import base64
+import binascii
+import datetime as dt
+import hashlib
+import itertools
+import json
+import os
+import re
+import sys
+import time
+import urllib.parse
+from html.parser import HTMLParser
+from pathlib import Path
+
+import requests
+
+DEFAULT_BASE_URL = "https://www.datalab.to/api/v1"
+DEFAULT_OUT_DIR = Path("/home/dev/workspaces/murtaza-workspaces")
+MODES = ("fast", "balanced", "accurate")
+POLL_SECONDS = 2
+POLL_ATTEMPTS = 450  # 15 minutes; long scanned catalogues genuinely take that
+
+TABLE_TYPES = ("Table", "TableOfContents")
+IMAGE_TYPES = ("Figure", "Picture", "FigureGroup", "PictureGroup", "Diagram")
+
+
+# ----------------------------------------------------------------- config
+
+def find_env_value(key: str, start_dirs) -> str | None:
+    """Read `key` from the first .env.local / .env found walking upward.
+
+    The Backend/.env of this project is the canonical home of DATALAB_API_KEY,
+    so searching up from both the PDF and the working directory finds it
+    whether the script is run from the repo root, from Backend/, or from
+    wherever the PDF happens to live.
+    """
+    seen = set()
+    for start in start_dirs:
+        for directory in [start, *start.parents]:
+            if directory in seen:
+                continue
+            seen.add(directory)
+            for name in (".env.local", ".env", "Backend/.env", "Backend/.env.local"):
+                env = directory / name
+                if not env.is_file():
+                    continue
+                for line in env.read_text(errors="ignore").splitlines():
+                    line = line.strip()
+                    if line.startswith("#") or not line.startswith(f"{key}="):
+                        continue
+                    value = line.split("=", 1)[1].strip().strip("'\"")
+                    if value:
+                        return value
+    return None
+
+
+def api_key(pdf: Path) -> str:
+    key = os.environ.get("DATALAB_API_KEY") or find_env_value(
+        "DATALAB_API_KEY", [pdf.resolve().parent, Path.cwd()])
+    if not key:
+        raise SystemExit(
+            "DATALAB_API_KEY not found. Set it in the environment, or in a .env "
+            "in any parent directory (this project keeps it in Backend/.env).")
+    return key
+
+
+def base_url(pdf: Path) -> str:
+    # datalab.to without `www` 302-redirects, and the redirect turns the POST
+    # into a GET and drops the multipart body, so the host matters.
+    return (os.environ.get("DATALAB_BASE_URL")
+            or find_env_value("DATALAB_BASE_URL", [pdf.resolve().parent, Path.cwd()])
+            or DEFAULT_BASE_URL).rstrip("/")
+
+
+# ------------------------------------------------------------ api access
+
+def strip_images(node):
+    """Drop base64 payloads from a response destined for the cache.
+
+    Images are stored twice — top-level `images` and again inside each figure
+    block — and on a scanned catalogue they are the overwhelming majority of
+    the bytes. Names are preserved either way, so nothing is lost except the
+    pixels themselves.
+    """
+    if isinstance(node, list):
+        for item in node:
+            strip_images(item)
+    elif isinstance(node, dict):
+        if node.get("images"):
+            node["images"] = {name: "" for name in node["images"]}
+        for key in ("children", "json"):
+            if node.get(key):
+                strip_images(node[key])
+    return node
+
+
+def cache_path(pdf: Path, opts: dict, cache_dir: Path, keep_images: bool) -> Path:
+    """Key the cache on the resolved path *and* every option that changes the body.
+
+    Two different PDFs named report.pdf would otherwise share an entry and
+    silently serve each other's blocks, and a re-run with --force-ocr would
+    return the non-OCR response and look like the flag does nothing. keep_images
+    belongs in the key for the same reason: an earlier run strips the base64
+    payloads before caching, so without it a later --keep-images run reads that
+    stripped entry and writes no images at all while reporting success.
+    """
+    stamp = json.dumps([str(pdf.resolve()), opts, keep_images], sort_keys=True)
+    digest = hashlib.sha1(stamp.encode()).hexdigest()[:10]
+    images = ".img" if keep_images else ""
+    return cache_dir / f"{pdf.stem}.{opts['mode']}{images}.{digest}.json"
+
+
+def convert(pdf: Path, key: str, url: str, opts: dict, cache_dir: Path,
+            refresh: bool, keep_images: bool) -> dict:
+    """Submit one PDF and poll until Datalab finishes with it."""
+    cache = cache_path(pdf, opts, cache_dir, keep_images)
+    if cache.exists() and not refresh:
+        print(f"  cached -> {cache}", file=sys.stderr)
+        return json.loads(cache.read_text())
+
+    data = {"output_format": "json", "mode": opts["mode"]}
+    if opts.get("page_range"):
+        data["page_range"] = opts["page_range"]
+    if opts.get("max_pages"):
+        data["max_pages"] = str(opts["max_pages"])
+    if opts.get("force_ocr"):
+        data["force_ocr"] = "true"
+    if opts.get("use_llm"):
+        data["use_llm"] = "true"
+
+    headers = {"X-API-Key": key}
+    with pdf.open("rb") as fh:
+        resp = requests.post(f"{url}/convert", headers=headers,
+                             files={"file": (pdf.name, fh, "application/pdf")},
+                             data=data, timeout=180)
+    if resp.status_code >= 400:
+        raise SystemExit(f"{pdf.name}: submit failed (HTTP {resp.status_code}): {resp.text[:400]}")
+    submitted = resp.json()
+    if not submitted.get("success"):
+        raise SystemExit(f"{pdf.name}: submit rejected: {submitted.get('error')}")
+
+    check_url = submitted["request_check_url"]
+    started = time.time()
+    for attempt in range(POLL_ATTEMPTS):
+        result = requests.get(check_url, headers=headers, timeout=60).json()
+        status = result.get("status")
+        if status == "complete":
+            result["request_id"] = submitted.get("request_id")
+            if not keep_images:
+                strip_images(result)
+            cache_dir.mkdir(parents=True, exist_ok=True)
+            cache.write_text(json.dumps(result, indent=2))
+            print(f"  converted in {time.time() - started:.1f}s -> cached {cache}", file=sys.stderr)
+            return result
+        if status == "failed" or result.get("error"):
+            raise SystemExit(f"{pdf.name}: conversion failed: {result.get('error')}")
+        if attempt and attempt % 15 == 0:
+            print(f"  still {status}... {time.time() - started:.0f}s", file=sys.stderr)
+        time.sleep(POLL_SECONDS)
+    raise SystemExit(f"{pdf.name}: timed out after {POLL_ATTEMPTS * POLL_SECONDS}s")
+
+
+# ---------------------------------------------------------- block walking
+
+def walk_blocks(node, out=None):
+    """Depth-first walk yielding every block in document order.
+
+    Blocks nest arbitrarily — a Table can sit inside a Group inside a Page —
+    so recursion is the only reliable traversal. Most leaf blocks come back
+    with `children: []` and their full content already inlined in `html`.
+    """
+    if out is None:
+        out = []
+    if isinstance(node, list):
+        for item in node:
+            walk_blocks(item, out)
+        return out
+    if not isinstance(node, dict):
+        return out
+    if node.get("block_type"):
+        out.append(node)
+    walk_blocks(node.get("children") or [], out)
+    return out
+
+
+def page_number(block, fallback=None):
+    """Page index as a 1-based number.
+
+    Blocks carry a 0-based `page` int; Page blocks sometimes only have the id
+    `/page/3/Page/3`. Prefer the field, fall back to parsing the id, so a
+    block never lands on an unknown page.
+    """
+    value = block.get("page")
+    if isinstance(value, int):
+        return value + 1
+    match = re.match(r"/page/(\d+)", block.get("id") or "")
+    if match:
+        return int(match.group(1)) + 1
+    return fallback
+
+
+# ----------------------------------------------------------- html -> text
+
+class TableParser(HTMLParser):
+    """Flatten an HTML table into rows of cell strings.
+
+    Cells are placed by occupancy rather than appended, so colspan/rowspan
+    consume the columns they actually cover. PDF tables merge header cells
+    constantly, and appending would shift every later cell left — a header
+    spanning three columns followed by one more would put that second header
+    at column 1 and misalign it against every data row beneath.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.cells, self.filled_rows = {}, set()
+        self.covered = {}
+        self.row_idx, self.col_idx = -1, 0
+        self.cell, self.span = None, (1, 1)
+        self.header_rows = set()
+
+    @staticmethod
+    def _span(attrs, name):
+        try:
+            return max(1, int(dict(attrs).get(name, 1)))
+        except (TypeError, ValueError):
+            return 1
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "tr":
+            self.row_idx += 1
+            self.col_idx = 0
+        elif tag in ("td", "th"):
+            self.cell = []
+            self.span = (self._span(attrs, "colspan"), self._span(attrs, "rowspan"))
+            if tag == "th":
+                self.header_rows.add(self.row_idx)
+        elif tag == "br" and self.cell is not None:
+            self.cell.append(" ")
+
+    def handle_endtag(self, tag):
+        if tag not in ("td", "th") or self.cell is None:
+            return
+        text = re.sub(r"\s+", " ", "".join(self.cell)).strip()
+        self.cell = None
+        colspan, rowspan = self.span
+        while (self.row_idx, self.col_idx) in self.cells:
+            self.col_idx += 1
+        for dr in range(rowspan):
+            for dc in range(colspan):
+                if dr == dc == 0:
+                    self.cells[(self.row_idx, self.col_idx)] = text
+                else:
+                    # A span continuation is blank as text but is *not* an empty
+                    # cell: the markup states this row is covered by `text`.
+                    # Recording that lets a later forward-fill cite the document
+                    # rather than guess.
+                    self.cells[(self.row_idx + dr, self.col_idx + dc)] = ""
+                    self.covered[(self.row_idx + dr, self.col_idx + dc)] = text
+        self.filled_rows.add(self.row_idx)
+        self.col_idx += colspan
+
+    def handle_data(self, data):
+        if self.cell is not None:
+            self.cell.append(data)
+
+    def _shape(self):
+        width = max(c for _, c in self.cells) + 1
+        rows = sorted({r for r, _ in self.cells} | self.filled_rows)
+        return width, rows
+
+    def grid(self):
+        if not self.cells:
+            return []
+        width, rows = self._shape()
+        return [[self.cells.get((r, c), "") for c in range(width)] for r in rows]
+
+    def span_grid(self):
+        """Same shape as `grid()`, holding the value that covers each cell.
+
+        `None` where the cell owns its own content (or is genuinely empty);
+        a string where a rowspan/colspan from elsewhere reaches into it.
+        """
+        if not self.cells:
+            return []
+        width, rows = self._shape()
+        return [[self.covered.get((r, c)) for c in range(width)] for r in rows]
+
+    def owned_grid(self):
+        """Which positions the row wrote itself, as opposed to inheriting."""
+        if not self.cells:
+            return []
+        width, rows = self._shape()
+        return [[(r, c) in self.cells and (r, c) not in self.covered
+                 for c in range(width)] for r in rows]
+
+
+def repair_short_rows(grid, spans, owned):
+    """Re-place rows the parser had to left-justify because a span ran out.
+
+    A merged category cell whose `rowspan` Datalab counts one row short leaves
+    that row with no cell to inherit, so its own cells start at column 0 and
+    every field lands one place left — the SKU under Category, the price under
+    the wrong header. The row still renders fine, which is why it survives
+    review, and a single such row is enough to turn a grouping column into a
+    list of SKUs.
+
+    The rest of the block already shows where each kind of value belongs, so
+    the run of cells is slid to wherever its content fits the column profiles
+    best, and only when that beats leaving it alone by a clear margin.
+    """
+    if not grid:
+        return grid, spans
+    width = len(grid[0])
+    lengths = [sum(row) for row in owned]
+    modal = max(set(lengths), key=lengths.count) if lengths else 0
+    typical = [grid[i] for i, n in enumerate(lengths) if n == modal]
+    if len(typical) < 3 or modal == width:
+        return grid, spans
+    profiles = [column_profile(typical, c) for c in range(width)]
+
+    for i, flags in enumerate(owned):
+        positions = [c for c, on in enumerate(flags) if on]
+        if not positions or len(positions) == width:
+            continue
+        if positions != list(range(positions[0], positions[-1] + 1)):
+            continue
+        # Candidates follow the parser's own rule — cells fill the free columns
+        # in order, stepping over whatever a span already covers — so the only
+        # question is which free column the run starts at.
+        free = [c for c in range(width) if spans[i][c] is None]
+        values = [grid[i][c] for c in positions]
+        if len(free) <= len(values):
+            continue
+        placements = [free[j:j + len(values)]
+                      for j in range(len(free) - len(values) + 1)]
+        scored = sorted(
+            ((sum(profile_similarity(profiles[c], {cell_kind(v): 1.0})
+                  for c, v in zip(cols, values)) / len(values), cols)
+             for cols in placements), key=lambda s: s[0], reverse=True)
+        best, cols = scored[0]
+        if cols == positions or best - scored[1][0] < 0.15:
+            continue
+        row, span = [""] * width, [None] * width
+        for c in range(width):
+            if spans[i][c] is not None:
+                row[c], span[c] = grid[i][c], spans[i][c]
+        for c, value in zip(cols, values):
+            row[c] = value
+        grid[i], spans[i] = row, span
+    return grid, spans
+
+
+def to_grid(html: str):
+    """Return (grid, has_header, spans).
+
+    `has_header` comes from whether Datalab used <th> in the first row, which
+    is the only trustworthy signal available. Assuming row 0 is always a header
+    quietly eats a real data row on every table that continues from the
+    previous page — those come back as pure <td> with no header at all.
+
+    `spans` is a parallel grid naming the merged cell that covers each blank,
+    which is what makes a category column recoverable rather than guessable.
+    """
+    parser = TableParser()
+    parser.feed(html or "")
+    grid, spans = repair_short_rows(parser.grid(), parser.span_grid(),
+                                    parser.owned_grid())
+    return grid, 0 in parser.header_rows, spans
+
+
+class TextParser(HTMLParser):
+    """Plain text from a block's HTML, keeping list items on their own lines."""
+
+    def __init__(self):
+        super().__init__()
+        self.parts = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ("li", "br", "p", "tr"):
+            self.parts.append("\n")
+        if tag == "img":
+            alt = dict(attrs).get("alt") or dict(attrs).get("src") or "image"
+            self.parts.append(f"[{alt}]")
+        if tag in ("td", "th"):
+            self.parts.append("\t")
+
+    def handle_endtag(self, tag):
+        if tag in ("li", "p", "tr", "div"):
+            self.parts.append("\n")
+
+    def handle_data(self, data):
+        self.parts.append(data)
+
+
+def to_text(html: str) -> str:
+    parser = TextParser()
+    parser.feed(html or "")
+    text = "".join(parser.parts)
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n\s*\n+", "\n", text)
+    return text.strip()
+
+
+# ------------------------------------------------------------ html -> md
+
+def md_escape(cell: str) -> str:
+    return cell.replace("|", "\\|").replace("\n", " ").strip()
+
+
+def md_image(path: str) -> str:
+    """An image link that survives a stem with spaces in it.
+
+    PDF filenames routinely contain spaces, and the images directory is named
+    after the file, so an unencoded link stops at the first space and every
+    picture renders broken.
+    """
+    return f"![]({urllib.parse.quote(path)})"
+
+
+def grid_to_markdown(grid, has_header: bool, headers=None) -> list[str]:
+    """Render a full table. Nothing is truncated — this file is the document.
+
+    Markdown tables require a header row, so a headerless fragment gets either
+    the headers carried over from the table it continues or `col_1..col_n`
+    placeholders. Promoting its first data row instead would hide a real row.
+    """
+    if not grid:
+        return ["_empty table_"]
+    width = max(len(r) for r in grid)
+    padded = [r + [""] * (width - len(r)) for r in grid]
+    if has_header:
+        header, body = padded[0], padded[1:]
+    else:
+        header, body = list(headers or []), padded
+    header = (header + [""] * width)[:width]
+    header = [h or f"col_{i + 1}" for i, h in enumerate(header)]
+    lines = ["| " + " | ".join(md_escape(h) for h in header) + " |",
+             "| " + " | ".join("---" for _ in header) + " |"]
+    lines += ["| " + " | ".join(md_escape(c) for c in row) + " |" for row in body]
+    return lines
+
+
+def image_description(html: str) -> str:
+    """Datalab's own description of a picture, which is generated text.
+
+    It arrives in a `<div class="img-description">` alongside the `<img>`. It is
+    often genuinely useful — for a flow diagram it can be a full transcription —
+    but it is the model's reading of the image, not text present in the PDF, so
+    it is kept apart from block text everywhere rather than blended in.
+    """
+    return to_text(re.sub(r"<img[^>]*>", "", html or ""))
+
+
+def block_to_markdown(block, tables_by_id=None, image_dir="", figures_by_block=None) -> list[str]:
+    """One block as Markdown, chosen by block type rather than by parsing HTML.
+
+    Datalab already told us what each block *is*; honouring that gives cleaner
+    Markdown than a generic html2md pass, and keeps headings as headings so the
+    output has a navigable structure.
+    """
+    btype = block.get("block_type") or ""
+    html = block.get("html") or ""
+
+    if btype in TABLE_TYPES:
+        resolved = (tables_by_id or {}).get(block.get("id"))
+        if not resolved:
+            grid, has_header, _ = to_grid(html)
+            return grid_to_markdown(grid, has_header, None)
+
+        # Rendered from the aligned rows rather than the raw grid, so a
+        # fragment prints under the headers it actually belongs to. The merged
+        # category is repeated on every row it covers: the PDF draws it once
+        # down the side of the block, which does not survive flattening.
+        rows = [list(r) for r in resolved["rows_data"]]
+        group = resolved.get("group_column")
+        if group and group["index"] is not None and resolved.get("categories"):
+            for i, row in enumerate(rows):
+                if group["index"] < len(row):
+                    row[group["index"]] = resolved["categories"][i]
+        headers = list(resolved.get("headers") or [])
+        photos = resolved.get("row_images")
+        if photos:
+            # The picture goes back into the column the PDF drew it in, so the
+            # row reads as one product. A merged photo covering several rows is
+            # repeated on each of them, for the same reason the category is.
+            column = resolved.get("image_column")
+            if column is None:
+                column = max(len(headers), max((len(r) for r in rows), default=0))
+                headers = (headers + [""] * (column + 1 - len(headers)))
+                headers[column] = headers[column] or "Image"
+            for i, row in enumerate(rows):
+                names = photos[i] if i < len(photos) else []
+                if not names:
+                    continue
+                while len(row) <= column:
+                    row.append("")
+                row[column] = " ".join(md_image(image_dir + n) for n in names)
+        lines = grid_to_markdown(rows, False, headers)
+
+        notes = []
+        source = resolved.get("header_source", "")
+        if source.startswith("carried"):
+            note = f"continues the table at `{source.split(':', 1)[1]}`; headers carried over"
+            if resolved.get("alignment"):
+                a = resolved["alignment"]
+                note += (f", realigned ({a['mode']} {a['left']}/{a['right']}, "
+                         f"confidence {a['confidence']})")
+            notes.append(note)
+        elif source == "none":
+            notes.append("**no headers could be matched to this fragment**")
+        if resolved.get("dropped_header_rows"):
+            notes.append(f"{resolved['dropped_header_rows']} reprinted header row(s) removed")
+        inferred = sum(1 for s in resolved.get("category_fill", []) if s.startswith("carried"))
+        if inferred:
+            notes.append(f"category inferred for {inferred} of {len(rows)} rows")
+        if notes:
+            lines = ["_(" + "; ".join(notes) + ")_", ""] + lines
+        return lines
+
+    if btype in IMAGE_TYPES:
+        names = list((block.get("images") or {}).keys())
+        # Written with the directory the files actually land in. A bare filename
+        # renders as a broken image in every viewer, which reads as a failed
+        # extraction when the picture is sitting right there on disk.
+        rendered = (figures_by_block or {}).get(block.get("id"))
+        src = rendered or (f"{image_dir}{names[0]}" if names else None)
+        lines = [f"![{btype}]({urllib.parse.quote(src)})"] if src else []
+        caption = image_description(html)
+        if caption:
+            # Labelled because this text is Datalab's reading of the picture, not
+            # anything written in the PDF. Unlabelled it reads as authorial prose
+            # and ends up quoted back as if the document said it.
+            lines += ["", "> **Datalab image description (generated, not PDF text):**",
+                      "> " + caption.replace("\n", "\n> ")]
+        return lines
+
+    text = to_text(html)
+    if not text:
+        return []
+
+    if btype == "SectionHeader":
+        # Shift two levels down: the file title owns h1 and page markers own h2,
+        # so the document's own h1 has to start at h3 to keep the outline honest.
+        match = re.search(r"<h([1-6])", html)
+        level = min(6, int(match.group(1)) + 2) if match else 3
+        return ["#" * level + " " + text.replace("\n", " ")]
+
+    if btype in ("ListGroup", "ListItem"):
+        return [f"- {line.strip()}" for line in text.splitlines() if line.strip()]
+
+    if btype in ("Code", "Equation"):
+        return ["```", text, "```"]
+
+    return [text]
+
+
+# ------------------------------------------------- aligning table fragments
+
+def cell_kind(value: str) -> str:
+    """Coarse type of one cell, used to recognise a column by what it holds.
+
+    Column *names* vanish when a table fragments across a page break, but the
+    shape of the values does not: a price column stays decimal, a SKU column
+    stays code-like. Comparing those shapes is what lets a headerless fragment
+    be matched to the right headers instead of merely the right column count.
+    """
+    value = (value or "").strip()
+    if not value:
+        return "blank"
+    if value.lower().startswith("image:"):
+        return "image"
+    if re.fullmatch(r"\d+", value):
+        return "int"
+    if re.fullmatch(r"\d[\d,]*\.\d+", value):
+        return "decimal"
+    if re.fullmatch(r"[A-Za-z0-9][\w./\-]*", value) and any(c.isdigit() for c in value):
+        return "code"
+    return "text"
+
+
+def column_profile(rows, index: int) -> dict:
+    """Fraction of each cell kind in one column."""
+    kinds = {}
+    for row in rows:
+        kind = cell_kind(row[index] if index < len(row) else "")
+        kinds[kind] = kinds.get(kind, 0) + 1
+    total = sum(kinds.values()) or 1
+    return {k: v / total for k, v in kinds.items()}
+
+
+def profile_similarity(a: dict, b: dict) -> float:
+    """Histogram intersection: 1.0 when two columns hold the same mix of kinds."""
+    return sum(min(a.get(k, 0.0), b.get(k, 0.0)) for k in set(a) | set(b))
+
+
+def normalise_header(value: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", (value or "").lower())
+
+
+def is_header_echo(row, header_sets) -> bool:
+    """True when a <td> row is really the header line repeated mid-table.
+
+    Price lists reprint the header at the top of every page, and when the page
+    break lands inside one Datalab block that reprint arrives as ordinary data.
+    Left alone it becomes a product whose price is the string "List Price AED"
+    and — worse — whose category is the string "Category".
+
+    Every known header set is tried, not just the most recent one. A document
+    can run several table schemas at once, and a reprint of schema A landing in
+    a block that currently continues schema B is exactly the case that slips
+    through when only the latest headers are consulted.
+    """
+    seen = [normalise_header(c) for c in row if (c or "").strip()]
+    if len(seen) < 3:
+        return False
+    for headers in header_sets:
+        wanted = {normalise_header(h) for h in headers if normalise_header(h)}
+        if len(wanted) < 3:
+            continue
+        if sum(1 for c in seen if c in wanted) >= 0.8 * len(seen):
+            return True
+    return False
+
+
+def all_blank(rows, index: int) -> bool:
+    return all(not (row[index] if index < len(row) else "").strip() for row in rows)
+
+
+def align_rows(rows, headers, reference_rows):
+    """Fit rows of the wrong width onto `headers`, or refuse to.
+
+    A fragment loses whichever columns the page break cut off — on this
+    project's Stanley price list that is the *leading* merged Category column,
+    not the trailing image column, so a rule like "drop the last blank column"
+    would slide every field one place left and produce a table that looks
+    immaculate and is wrong in every row.
+
+    So the offset is chosen by evidence: try each way of lining the fragment up
+    against the headers, score it on how well each column's content matches the
+    same column in the table the headers came from, and take the winner only if
+    it is both good and unambiguous. Widening is lossless (missing columns come
+    back as blanks) and narrowing only ever drops columns that are entirely
+    empty, so no cell is discarded to make a fit.
+
+    Returns (aligned_rows, plan, score) or None when nothing fits. A plan is one
+    fragment column index — or None for "no column here" — per header.
+    """
+    width = max((len(r) for r in rows), default=0)
+    rows = [(r + [""] * (width - len(r)))[:width] for r in rows]
+    target = len(headers)
+    if not rows or not target:
+        return None
+
+    # Datalab sometimes emits a spare empty column *inside* a fragment, not
+    # only at its ends, so a fit may need to drop a middle column and pad both
+    # sides at once. Only entirely blank columns are ever droppable, which is
+    # what keeps the transform lossless however the ends move.
+    blanks = [i for i in range(width) if all_blank(rows, i)]
+    candidates = []
+    least = max(0, width - target)
+    for k in range(least, min(len(blanks), least + 2) + 1):
+        for drop in itertools.combinations(blanks, k):
+            keep = [i for i in range(width) if i not in drop]
+            if len(keep) > target:
+                continue
+            for left in range(target - len(keep) + 1):
+                plan = [None] * target
+                for j, col in enumerate(keep):
+                    plan[left + j] = col
+                candidates.append(tuple(plan))
+    candidates = list(dict.fromkeys(candidates))
+    if not candidates:
+        return None
+
+    # Every header is scored, including ones the plan leaves empty, so a fit
+    # that quietly abandons a column the reference fills is not rewarded for
+    # having fewer columns to be judged on.
+    empty = {"blank": 1.0}
+    refs = [column_profile(reference_rows, h) for h in range(target)]
+    frags = [column_profile(rows, c) for c in range(width)]
+    blank_set = set(blanks)
+
+    # Two plans that disagree only about which *empty* column sits where put
+    # every real value in the same place, so they are the same answer. Scoring
+    # them as rivals would make an unambiguous fit look contested and throw
+    # away a table that was never in doubt — so plans are grouped by where they
+    # send the columns that actually hold data, and only those groups compete.
+    grouped = {}
+    for plan in candidates:
+        score = sum(profile_similarity(refs[h], empty if c is None else frags[c])
+                    for h, c in enumerate(plan)) / target
+        signature = tuple((h, c) for h, c in enumerate(plan)
+                          if c is not None and c not in blank_set)
+        dropped = width - sum(1 for c in plan if c is not None)
+        current = grouped.get(signature)
+        if current is None or (score, -dropped) > (current[0], -current[2]):
+            grouped[signature] = (score, plan, dropped)
+
+    ranked = sorted(grouped.values(), key=lambda s: s[0], reverse=True)
+    best, plan, _ = ranked[0]
+    runner_up = ranked[1][0] if len(ranked) > 1 else 0.0
+    # A weak or contested fit is worse than no fit: misaligned rows read as
+    # valid data, whereas an empty `headers` list announces the problem.
+    if best < 0.55 or (len(ranked) > 1 and best - runner_up < 0.02):
+        return None
+
+    return apply_plan(rows, plan), plan, round(best, 3)
+
+
+def apply_plan(rows, plan, pad=""):
+    """Reshape rows by an alignment plan: one source column per header slot."""
+    width = max((len(r) for r in rows), default=0)
+    rows = [(r + [pad] * (width - len(r)))[:width] for r in rows]
+    return [[pad if c is None else row[c] for c in plan] for row in rows]
+
+
+def describe_plan(plan, width):
+    """The human-readable shape of an alignment plan."""
+    used = [c for c in plan if c is not None]
+    return {
+        "mode": "pad" if len(used) < len(plan) else "trim",
+        "left": plan.index(used[0]) if used else 0,
+        "right": len(plan) - 1 - plan.index(used[-1]) if used else 0,
+        "dropped_columns": [i for i in range(width) if i not in used],
+    }
+
+
+def is_identity(plan, width) -> bool:
+    return len(plan) == width and all(c == i for i, c in enumerate(plan))
+
+
+# ------------------------------------------------------- category recovery
+
+GROUP_HEADER_RE = re.compile(r"categor|group|section|family|range", re.I)
+LABEL_TYPES = ("Text", "SectionHeader", "TextInlineMath", "Caption")
+
+
+def looks_like_group(rows, spans, index: int) -> bool:
+    """Does this column behave like a grouping column, whatever it is called?
+
+    A column headed "Category" that holds a different SKU on every row is not a
+    category column — it is a misaligned table, and trusting the name turns one
+    bad alignment into hundreds of invented single-row categories. Grouping
+    values are words rather than codes, and there are far fewer of them than
+    there are rows. An entirely blank column still qualifies: that is the shape
+    of a merged cell Datalab lifted out into the margin.
+    """
+    # Measured per row covered, not per distinct cell. A merged column prints
+    # one value for thirty rows, so counting cells gives a label the same
+    # weight as a single SKU that Datalab shifted a place left — and that one
+    # stray cell would then veto the grouping for the whole block. Spreading
+    # each value over the rows its span reaches puts the evidence in
+    # proportion.
+    values = []
+    for i, row in enumerate(rows):
+        own = (row[index] or "").strip() if index < len(row) else ""
+        cover = spans[i][index] if i < len(spans) and index < len(spans[i]) else None
+        value = own or (cover or "").strip()
+        if value:
+            values.append(value)
+    if not values:
+        return True
+    words = sum(1 for v in values if cell_kind(v) == "text")
+    if words < 0.8 * len(values):
+        return False
+    return len(set(values)) <= max(1, len(rows) // 3)
+
+
+def find_group_column(headers, rows, spans):
+    """Which column carries the grouping label (the merged 'Category' column).
+
+    A matching header name is the first hint, but it is only accepted if the
+    column's contents agree. Otherwise the signal is rowspan: a column that
+    merges across many rows is a grouping column, whereas a price or a SKU
+    almost never merges. The image column also merges, so columns whose merged
+    values do not read as words are skipped.
+    """
+    for i, header in enumerate(headers):
+        if header and GROUP_HEADER_RE.search(header) and looks_like_group(rows, spans, i):
+            return i
+    for i in range(len(headers)):
+        covers = [spans[r][i] for r in range(len(rows))
+                  if i < len(spans[r]) and spans[r][i]]
+        if covers and all(cell_kind(c) == "text" for c in covers) \
+                and looks_like_group(rows, spans, i):
+            return i
+    return None
+
+
+def margin_labels_for(table_bbox, blocks):
+    """Blocks sitting in the left margin beside a table, in document order.
+
+    When a table fragments, Datalab often lifts the merged category cell out of
+    the table entirely and returns it as a narrow, tall text block printed
+    sideways in the margin — "Blade and Knives" beside the knives rows. Those
+    blocks are the document *stating* the category for rows whose own cell is
+    gone, which is far better evidence than carrying a value down from
+    whatever happened to come before.
+    """
+    if not table_bbox:
+        return []
+    tx0, ty0, tx1, ty1 = table_bbox
+    found = []
+    for block in blocks:
+        if block.get("block_type") not in LABEL_TYPES:
+            continue
+        bbox = block.get("bbox")
+        if not bbox:
+            continue
+        bx0, by0, bx1, by1 = bbox
+        # Left of the table body, narrow relative to it, and vertically
+        # alongside it — the shape of a rotated spanning cell.
+        if bx1 > tx0 + 8 or (bx1 - bx0) > 0.2 * max(1, tx1 - tx0):
+            continue
+        if by1 <= ty0 or by0 >= ty1:
+            continue
+        text = to_text(block.get("html")).strip()
+        if not text or len(text) > 60 or cell_kind(text) != "text":
+            continue
+        found.append({"text": text, "y0": by0, "y1": by1, "id": block.get("id")})
+    return sorted(found, key=lambda b: b["y0"])
+
+
+# -------------------------------------------------------------- structure
+
+def records_from_rows(headers, rows):
+    """Header-keyed rows, but only when the headers can actually key them.
+
+    Blank or duplicated headers are common in PDF tables — merged spans, a unit
+    row beneath the header row, a spacer column. Building records anyway would
+    collapse two columns into one key and silently drop a field, so return None
+    and let the consumer read `rows_data` instead. An honest absence beats a
+    lossy dict that looks fine until someone trusts it.
+    """
+    if not headers or not rows:
+        return None
+    # A blank trailing header is usually an unlabelled column (a product image,
+    # a notes column) — naming it keeps its values instead of discarding them.
+    # Duplicates are the genuinely ambiguous case, and there records are refused.
+    clean = [h.strip() or f"col_{i + 1}" for i, h in enumerate(headers)]
+    if len(set(clean)) != len(clean):
+        return None
+    width = len(clean)
+    return [dict(zip(clean, (row + [""] * width)[:width])) for row in rows]
+
+
+def build_table(block, num, references):
+    """One Table block, lined up with the headers it belongs to.
+
+    `references` accumulates every table that carried its own <th> row, newest
+    first, so a fragment can be matched against the schema it actually
+    continues rather than merely the last one seen.
+    """
+    grid, has_header, spans = to_grid(block.get("html"))
+    raw_width = len(grid[0]) if grid else 0
+    if has_header:
+        headers, body, body_spans = list(grid[0]), grid[1:], spans[1:]
+    else:
+        headers, body, body_spans = None, grid, spans
+
+    # A reprinted column-header line arrives as ordinary <td> when the page
+    # break falls inside one block. Left in place it becomes a product priced
+    # "List Price AED", so it is removed and counted rather than silently kept.
+    echo_against = ([headers] if headers else []) + [r["headers"] for r in references]
+    keep = [i for i, row in enumerate(body) if not is_header_echo(row, echo_against)]
+    echoes = len(body) - len(keep)
+    body = [body[i] for i in keep]
+    body_spans = [body_spans[i] for i in keep]
+
+    plan, score, source = None, None, "none"
+    if has_header:
+        source = "own"
+        headers = (headers + [""] * raw_width)[:raw_width]
+        rows = [(r + [""] * raw_width)[:raw_width] for r in body]
+        row_spans = [(s + [None] * raw_width)[:raw_width] for s in body_spans]
+    else:
+        rows, row_spans = body, body_spans
+        headers = []
+        # Every reference is scored on content, including the ones whose width
+        # already matches. Equal width is not evidence of equal meaning: a
+        # document running two schemas at once will happily offer an 8-column
+        # reference whose first header is "Category" to a fragment whose first
+        # column holds SKUs, and an unchecked width match writes those SKUs out
+        # as category names. Scoring costs nothing and catches it.
+        best = None
+        for ref in references:
+            fit = align_rows(body, ref["headers"], ref["rows"])
+            if fit and (best is None or fit[2] > best[0][2]):
+                best = (fit, ref)
+        if best:
+            (rows, plan, score), ref = best
+            row_spans = apply_plan(body_spans, plan, pad=None)
+            headers = list(ref["headers"])
+            # An identity plan means the fragment sat under the headers as-is;
+            # anything else moved columns and deserves the louder label.
+            if is_identity(plan, raw_width):
+                source, plan = f"carried:{ref['id']}", None
+            else:
+                source = f"carried-trimmed:{ref['id']}"
+
+    table = {
+        "id": block.get("id"),
+        "page": num,
+        "bbox": block.get("bbox"),
+        "columns": len(headers) if headers else raw_width,
+        "columns_raw": raw_width,
+        "row_count": len(rows),
+        "has_header": has_header,
+        "header_source": source,
+        "headers": headers,
+        "rows_data": rows,
+        "_spans": row_spans,
+    }
+    if plan:
+        table["alignment"] = dict(describe_plan(plan, raw_width), confidence=score)
+    if echoes:
+        table["dropped_header_rows"] = echoes
+    # A reference with no rows has no content profile, so nothing that carries
+    # its headers can ever be checked against it — it would hand its column
+    # names to any fragment of the same width, sight unseen.
+    if has_header and rows:
+        references.insert(0, {"headers": headers, "id": block.get("id"), "rows": rows})
+    return table
+
+
+def assign_categories(tables, blocks_by_page):
+    """Give every row a category, and say where each one came from.
+
+    Three sources, strongest first: the row's own cell; a rowspan that the
+    markup says covers it; a margin label the document prints beside it. Only
+    when none of those exist is a value carried from a neighbouring row — and
+    a carry is stopped at the point a margin label visibly ends, because past
+    that line the document has moved on to the next group even though the next
+    label may not be printed until the following page.
+    """
+    for table in tables:
+        headers, rows, spans = table["headers"], table["rows_data"], table["_spans"]
+        index = find_group_column(headers, rows, spans) if headers else None
+        table["group_column"] = ({"index": index, "name": headers[index]}
+                                 if index is not None else None)
+        stated = []
+        for r, row in enumerate(rows):
+            own = (row[index] or "").strip() if index is not None and index < len(row) else ""
+            cover = (spans[r][index] or "").strip() if (
+                index is not None and r < len(spans) and index < len(spans[r])
+                and spans[r][index]) else ""
+            if own:
+                stated.append((own, "own"))
+            elif cover:
+                stated.append((cover, "rowspan"))
+            else:
+                stated.append(None)
+        table["_stated"] = stated
+
+        labels = ([] if any(stated) else
+                  margin_labels_for(table["bbox"], blocks_by_page.get(table["page"], [])))
+        table["_labels"] = labels
+        if labels and rows:
+            if len(labels) == 1:
+                table["_stated"] = [(labels[0]["text"], "margin-label")] * len(rows)
+            else:
+                y0, y1 = table["bbox"][1], table["bbox"][3]
+                step = (y1 - y0) / len(rows)
+                for r in range(len(rows)):
+                    mid = y0 + step * (r + 0.5)
+                    near = min(labels, key=lambda l: abs((l["y0"] + l["y1"]) / 2 - mid))
+                    table["_stated"][r] = (near["text"], "margin-label")
+
+    # Forward pass. `barrier` is the y at which the current margin label ran
+    # out; rows below it on the same page are a different group even though we
+    # cannot yet name it, so the carry is withheld rather than guessed.
+    live, live_page, barrier = None, None, None
+    for table in tables:
+        filled = []
+        for r, item in enumerate(table["_stated"]):
+            if item:
+                live, live_page, barrier = item[0], table["page"], None
+                if table["_labels"]:
+                    barrier = max(l["y1"] for l in table["_labels"])
+                filled.append(item)
+                continue
+            dead = (barrier is not None and table["page"] == live_page
+                    and table["bbox"] and table["bbox"][1] >= barrier)
+            filled.append((live, "carried-forward") if live and not dead else None)
+        table["_filled"] = filled
+
+    # Backward pass for anything the forward pass refused to guess at.
+    nxt = None
+    for table in reversed(tables):
+        for r in range(len(table["_filled"]) - 1, -1, -1):
+            if table["_filled"][r]:
+                nxt = table["_filled"][r][0]
+            elif nxt:
+                table["_filled"][r] = (nxt, "carried-back")
+
+    for table in tables:
+        resolved = [f or ("", "none") for f in table["_filled"]]
+        index = (table["group_column"] or {}).get("index")
+        for key in ("_spans", "_stated", "_labels", "_filled"):
+            table.pop(key, None)
+        # A document with no grouping column gets no category keys at all,
+        # rather than columns of empty strings that read like a failure.
+        if not any(v for v, _ in resolved):
+            continue
+        table["categories"] = [v for v, _ in resolved]
+        table["category_fill"] = [s for _, s in resolved]
+        table["category_raw"] = [
+            (row[index] or "").strip() if index is not None and index < len(row) else ""
+            for row in table["rows_data"]]
+
+
+def build_document(pdf: Path, result: dict, opts: dict, digest: str) -> dict:
+    root = result.get("json") or {}
+    blocks = walk_blocks(root)
+
+    pages, tables, figures = {}, [], []
+    counts, references = {}, []
+    for block in blocks:
+        btype = block.get("block_type")
+        counts[btype] = counts.get(btype, 0) + 1
+        num = page_number(block)
+        if btype == "Page":
+            bbox = block.get("bbox") or [0, 0, 0, 0]
+            pages.setdefault(num, {"page": num, "blocks": []})
+            pages[num]["size"] = {"width": bbox[2], "height": bbox[3]}
+            continue
+        page = pages.setdefault(num, {"page": num, "blocks": []})
+
+        entry = {
+            "id": block.get("id"),
+            "type": btype,
+            "bbox": block.get("bbox"),
+        }
+        if btype in TABLE_TYPES:
+            table = build_table(block, num, references)
+            entry["rows"] = table["row_count"]
+            entry["columns"] = table["columns"]
+            entry["has_header"] = table["has_header"]
+            tables.append(table)
+        elif btype in IMAGE_TYPES:
+            names = list((block.get("images") or {}).keys())
+            entry["images"] = names
+            # Deliberately not `text`: this is Datalab describing the picture,
+            # so it must not be counted or quoted as document content.
+            entry["description"] = image_description(block.get("html"))
+            entry["description_source"] = "datalab-generated"
+            for name in names:
+                figures.append({"name": name, "page": num, "block_id": block.get("id"),
+                                "bbox": block.get("bbox")})
+        else:
+            entry["text"] = to_text(block.get("html"))
+        page["blocks"].append(entry)
+
+    blocks_by_page = {}
+    for block in blocks:
+        if block.get("block_type") != "Page":
+            blocks_by_page.setdefault(page_number(block), []).append(block)
+    assign_categories(tables, blocks_by_page)
+
+    for table in tables:
+        records = records_from_rows(table["headers"], table["rows_data"])
+        if records is not None:
+            for i, record in enumerate(records if "categories" in table else []):
+                # `category` is a fixed key so downstream code need not know
+                # what this document calls the column; `category_raw` keeps
+                # what was actually printed in the cell, and `category_fill`
+                # says which of the two you are looking at.
+                record["category"] = table["categories"][i]
+                record["category_raw"] = table["category_raw"][i]
+                record["category_fill"] = table["category_fill"][i]
+                group = table.get("group_column")
+                if group:
+                    record[group["name"] or f"col_{group['index'] + 1}"] = \
+                        table["categories"][i]
+            table["records"] = records
+
+    ordered = [pages[k] for k in sorted(pages)]
+    text_chars = sum(len(b.get("text") or "") for p in ordered for b in p["blocks"])
+    meta = result.get("metadata") or {}
+
+    fills, groups = {}, {}
+    grouped = any(t.get("group_column") for t in tables)
+    for table in tables:
+        for i, source in enumerate(table.get("category_fill") or []):
+            fills[source] = fills.get(source, 0) + 1
+            groups[table["categories"][i]] = groups.get(table["categories"][i], 0) + 1
+    total_rows = sum(t["row_count"] for t in tables)
+    header_sources = {}
+    for table in tables:
+        kind = table["header_source"].split(":")[0]
+        header_sources[kind] = header_sources.get(kind, 0) + 1
+
+    return {
+        "source": {
+            "file": pdf.name,
+            "path": str(pdf.resolve()),
+            "bytes": pdf.stat().st_size,
+            "sha256": digest,
+            "extracted_at": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
+        },
+        "extraction": {
+            "provider": "datalab.to/api/v1/convert",
+            "output_format": "json",
+            "mode": opts["mode"],
+            "page_range": opts.get("page_range"),
+            "max_pages": opts.get("max_pages"),
+            "force_ocr": bool(opts.get("force_ocr")),
+            "use_llm": bool(opts.get("use_llm")),
+            "request_id": result.get("request_id"),
+            "runtime_seconds": result.get("runtime"),
+            "cost_cents": (result.get("cost_breakdown") or {}).get("final_cost_cents"),
+            "page_count": result.get("page_count"),
+            "failed_pages": meta.get("failed_pages") or [],
+        },
+        "summary": {
+            "pages": len(ordered),
+            "blocks": sum(len(p["blocks"]) for p in ordered),
+            "tables": len(tables),
+            "figures": len(figures),
+            "text_characters": text_chars,
+            "block_types": dict(sorted(counts.items())),
+            "table_rows": total_rows,
+            "header_sources": dict(sorted(header_sources.items())),
+            # How each category was arrived at. "own"/"rowspan"/"margin-label"
+            # are stated by the document; the "carried-*" counts are this
+            # script's inference and are the number to be sceptical about.
+            # Both keys are absent when no table has a grouping column, so an
+            # empty category never looks like a failed extraction.
+            **({"category_fill": dict(sorted(fills.items())),
+                "categories": dict(sorted(groups.items(), key=lambda kv: -kv[1]))}
+               if grouped else {}),
+        },
+        "pages": ordered,
+        "tables": tables,
+        "figures": figures,
+    }
+
+
+def render_markdown(doc: dict, result: dict, image_dir="") -> str:
+    src, ex, summary = doc["source"], doc["extraction"], doc["summary"]
+    lines = [
+        f"# {src['file']}",
+        "",
+        f"Extracted {src['extracted_at']} via Datalab (`mode={ex['mode']}`"
+        + (f", `page_range={ex['page_range']}`" if ex["page_range"] else "")
+        + (", `force_ocr`" if ex["force_ocr"] else "")
+        + (", `use_llm`" if ex["use_llm"] else "") + ").",
+        "",
+        f"{summary['pages']} page(s) · {summary['blocks']} blocks · "
+        f"{summary['tables']} table(s) · {summary['figures']} figure(s)",
+    ]
+    if ex["failed_pages"]:
+        lines += ["", f"**Pages Datalab could not parse: {ex['failed_pages']}**"]
+    lines.append("")
+
+    tables_by_id = {t["id"]: t for t in doc["tables"]}
+    figures_by_block = {f["block_id"]: f["file"] for f in doc.get("figures") or []
+                        if f.get("block_id") and f.get("file")}
+    root = result.get("json") or {}
+    by_page = {}
+    for block in walk_blocks(root):
+        if block.get("block_type") == "Page":
+            continue
+        by_page.setdefault(page_number(block), []).append(block)
+
+    for num in sorted(by_page):
+        lines += [f"## Page {num}", ""]
+        for block in by_page[num]:
+            rendered = block_to_markdown(block, tables_by_id, image_dir, figures_by_block)
+            if rendered:
+                lines += rendered + [""]
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def write_images(result: dict, out_dir: Path, stem: str) -> int:
+    """Write base64 images out as real files next to the JSON.
+
+    Kept out of the JSON deliberately: a base64 blob makes the file unreadable
+    and unusable by anything that wants to grep it, and figures are far more
+    useful as files you can open.
+    """
+    images = result.get("images") or {}
+    written = 0
+    target = out_dir / f"{stem}_images"
+    for name, payload in images.items():
+        if not payload:
+            continue
+        try:
+            data = base64.b64decode(payload)
+        except (binascii.Error, ValueError):
+            continue
+        target.mkdir(parents=True, exist_ok=True)
+        (target / name).write_bytes(data)
+        written += 1
+    if written:
+        print(f"  images -> {target} ({written})", file=sys.stderr)
+    return written
+
+
+# ------------------------------------------------------------------- main
+
+def sha256_of(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as fh:
+        for chunk in iter(lambda: fh.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def collect_inputs(source: Path) -> list[Path]:
+    if source.is_dir():
+        return sorted(p for p in source.iterdir() if p.suffix.lower() == ".pdf")
+    return [source]
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("input", type=Path, help="a PDF file or a directory of PDFs")
+    parser.add_argument("--out-dir", type=Path, default=DEFAULT_OUT_DIR,
+                        help=f"where the .json and .md land (default: {DEFAULT_OUT_DIR})")
+    parser.add_argument("--mode", choices=MODES, default="balanced",
+                        help="Datalab quality mode (default: balanced)")
+    parser.add_argument("--pages", dest="page_range",
+                        help="0-indexed page selection, e.g. 0-4 or 0,2,7-9")
+    parser.add_argument("--max-pages", type=int, help="stop after N pages")
+    parser.add_argument("--force-ocr", action="store_true",
+                        help="re-OCR every page; use when the text layer is garbled")
+    parser.add_argument("--use-llm", action="store_true",
+                        help="LLM pass for messy tables and merged cells (slower, costs more)")
+    parser.add_argument("--page-images", action="store_true",
+                        help="crop every image out of the rendered PDF and attach "
+                             "it to the table rows it sits beside")
+    parser.add_argument("--dpi", type=int, default=200,
+                        help="render resolution for --page-images (default 200)")
+    parser.add_argument("--keep-images", action="store_true",
+                        help="also write figures as image files beside the JSON")
+    parser.add_argument("--cache-dir", type=Path, default=Path("datalab_raw"),
+                        help="where raw API responses are cached (default: ./datalab_raw)")
+    parser.add_argument("--refresh", action="store_true",
+                        help="ignore the cache and call the API again")
+    args = parser.parse_args()
+
+    files = collect_inputs(args.input)
+    if not files:
+        parser.error(f"no PDF found at {args.input}")
+    missing = [p for p in files if not p.is_file()]
+    if missing:
+        parser.error(f"not a file: {missing[0]}")
+
+    opts = {"mode": args.mode, "page_range": args.page_range, "max_pages": args.max_pages,
+            "force_ocr": args.force_ocr, "use_llm": args.use_llm}
+
+    key = api_key(files[0])
+    url = base_url(files[0])
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+
+    for pdf in files:
+        print(f"{pdf.name} ({pdf.stat().st_size / 1e6:.1f} MB)", file=sys.stderr)
+        result = convert(pdf, key, url, opts, args.cache_dir, args.refresh, args.keep_images)
+        doc = build_document(pdf, result, opts, sha256_of(pdf))
+
+        photos = None
+        if args.page_images:
+            sys.path.insert(0, str(Path(__file__).resolve().parent))
+            from extract_figures import attach_figures
+            photos = attach_figures(doc, pdf, args.out_dir, pdf.stem, args.dpi)
+
+        json_path = args.out_dir / f"{pdf.stem}.json"
+        md_path = args.out_dir / f"{pdf.stem}.md"
+        json_path.write_text(json.dumps(doc, indent=2, ensure_ascii=False) + "\n")
+        md_path.write_text(render_markdown(doc, result, f"{pdf.stem}_images/"))
+        if args.keep_images:
+            write_images(result, args.out_dir, pdf.stem)
+        if photos:
+            print(f"  images -> {args.out_dir / (pdf.stem + '_images')} "
+                  f"({photos['extracted']} at {photos['dpi']}dpi; "
+                  f"{photos['rows_with_image']} of "
+                  f"{photos['rows_with_image'] + photos['rows_without_image']} rows"
+                  f", {photos['unplaced_images']} outside any row)", file=sys.stderr)
+
+        s, e = doc["summary"], doc["extraction"]
+        cost = "" if e["cost_cents"] is None else f", cost {e['cost_cents']}c"
+        print(f"  {s['pages']} pages, {s['blocks']} blocks, {s['tables']} tables{cost}",
+              file=sys.stderr)
+        if e["failed_pages"]:
+            print(f"  WARNING failed pages: {e['failed_pages']}", file=sys.stderr)
+        print(f"  json -> {json_path}\n  md   -> {md_path}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()