@sitar_fiercer4c/skills 0.1.0

package/skills/pdf-to-json/scripts/extract_figures.py

@@ -0,0 +1,319 @@
+#!/usr/bin/env python3
+"""Recover product photos from a PDF and attach them to the table rows they belong to.
+
+Datalab returns `Picture` blocks for the big illustrations on a page and ignores
+the small in-cell product photos entirely — on the Stanley price list that is 14
+figures against 328 images actually embedded in the file. No mode or flag
+changes this, so the pictures have to come from the PDF itself.
+
+Each image is cut out of the *rendered* page rather than pulled from the PDF's
+object store. Catalogue photos are almost always a raster plus a soft mask, and
+extracting the raster alone gives you a black rectangle where the transparency
+should be; rendering flattens the mask onto white and sidesteps CMYK and
+indexed-colour conversion at the same time.
+
+Rows are located by finding where each SKU string physically sits on the page,
+not by dividing the table's height into equal bands — a table with a merged
+category column or a wrapped description has rows of visibly different heights,
+and equal bands quietly shift half the photos onto the neighbouring product.
+
+    python scripts/extract_figures.py <stem>.json [--dpi 200]
+
+Also importable: `attach_figures(doc, pdf, out_dir, stem, dpi)` mutates `doc`,
+which is how `pdf_extract.py --page-images` uses it.
+"""
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+# Deliberately permissive. A catalogue photo of a screwdriver bit is genuinely
+# 4pt wide, so anything stricter silently drops real products; only degenerate
+# placements — hairline rules, spacer pixels — need excluding.
+MIN_SIDE = 2.0       # points
+MIN_AREA = 20.0      # points²
+PAD = 1.5            # points of breathing room around a crop
+MIN_OVERLAP = 1.0    # points a row and an image must share before they are related
+MIN_SHARE = 0.15     # ...as a fraction of the shorter of the two, to reject grazing
+
+
+def normalise_header(value: str) -> str:
+    return re.sub(r"[^a-z0-9]", "", (value or "").lower())
+
+
+def page_images(page):
+    """Image placements on one page, clamped to the page and sorted top-down.
+
+    Placements are kept separate rather than clustered. On a price list the
+    photos in the image column butt up against each other with a 0.1pt gap, so
+    any merge-by-proximity rule collapses a whole column of products into one
+    picture. One placement is one photo unless the document proves otherwise.
+    """
+    found = []
+    for image in page.images:
+        x0 = max(0.0, min(image["x0"], page.width))
+        x1 = max(0.0, min(image["x1"], page.width))
+        top = max(0.0, min(image["top"], page.height))
+        bottom = max(0.0, min(image["bottom"], page.height))
+        # Off-page overhang is normal — one Stanley banner starts at top=-104 —
+        # and pdfplumber refuses to crop outside the page box, so clamp first
+        # and drop whatever is left with no visible area.
+        if x1 - x0 < MIN_SIDE or bottom - top < MIN_SIDE:
+            continue
+        if (x1 - x0) * (bottom - top) < MIN_AREA:
+            continue
+        found.append({"name": image.get("name"), "bbox": [x0, top, x1, bottom]})
+    return sorted(found, key=lambda i: (i["bbox"][1], i["bbox"][0]))
+
+
+def word_index(page):
+    """Every word on the page, keyed by its normalised text.
+
+    A SKU can legitimately appear twice on a page (in the body and in a footer),
+    so the value is a list and the caller picks by position.
+    """
+    lut = {}
+    for word in page.extract_words():
+        lut.setdefault(normalise_header(word["text"]), []).append(word)
+    return lut
+
+
+def sku_column(headers):
+    for i, header in enumerate(headers or []):
+        if normalise_header(header) == "sku":
+            return i
+    return None
+
+
+def anchor_rows(tables, lut, page_height):
+    """Give every row the vertical band it occupies, read off the page.
+
+    Each row is anchored on its SKU, then the band is stretched to the midpoint
+    between neighbouring anchors so the gaps between text lines belong to
+    somebody. Rows whose SKU cannot be found are left unanchored rather than
+    interpolated: a guessed band silently mislabels a photo, and a missing one
+    is at least visible in the coverage count.
+    """
+    anchors = []
+    for table in tables:
+        index = sku_column(table.get("headers"))
+        for row_no, row in enumerate(table.get("rows_data") or []):
+            candidates = []
+            if index is not None and index < len(row):
+                candidates.append(row[index])
+            candidates += [c for c in row if (c or "").strip()]
+            word = None
+            for candidate in candidates:
+                hits = lut.get(normalise_header(candidate or ""))
+                if hits:
+                    word = hits[0]
+                    break
+            anchors.append({"table": table["id"], "row": row_no,
+                            "sku": (row[index] if index is not None and index < len(row)
+                                    else ""),
+                            "word": word})
+
+    placed = [a for a in anchors if a["word"]]
+    placed.sort(key=lambda a: a["word"]["top"])
+    for i, anchor in enumerate(placed):
+        word = anchor["word"]
+        above = placed[i - 1]["word"]["bottom"] if i else 0.0
+        below = placed[i + 1]["word"]["top"] if i + 1 < len(placed) else page_height
+        anchor["band"] = [max(0.0, (word["top"] + above) / 2),
+                          min(page_height, (word["bottom"] + below) / 2)]
+    return anchors
+
+
+def overlap(a, b) -> float:
+    return max(0.0, min(a[1], b[1]) - max(a[0], b[0]))
+
+
+def safe_stem(page_no: int, sku: str, used: set) -> str:
+    base = re.sub(r"[^A-Za-z0-9._-]", "_", (sku or "").strip())
+    stem = f"p{page_no:02d}_{base}" if base else f"p{page_no:02d}_img"
+    name, n = stem, 1
+    while name in used:
+        n += 1
+        name = f"{stem}-{n}"
+    used.add(name)
+    return name
+
+
+def image_column(table):
+    """Which column holds the picture, so Markdown can put the photo back in it."""
+    rows = table.get("rows_data") or []
+    headers = table.get("headers") or []
+    width = len(headers) or max((len(r) for r in rows), default=0)
+    best, score = None, 0.0
+    for c in range(width):
+        hits = sum(1 for r in rows
+                   if c < len(r) and (r[c] or "").strip().lower().startswith("image:"))
+        if hits and hits / max(1, len(rows)) > score:
+            best, score = c, hits / max(1, len(rows))
+    if best is not None:
+        return best
+    # No cell says "Image:", so fall back to a trailing column that is blank
+    # everywhere — on a fragmented table that is where the picture used to be.
+    if width and rows and all(not (r[width - 1] if width - 1 < len(r) else "").strip()
+                              for r in rows):
+        return width - 1
+    return None
+
+
+def attach_figures(doc: dict, pdf_path: Path, out_dir: Path, stem: str,
+                   dpi: int = 200) -> dict:
+    """Crop every embedded image and record which rows it sits beside."""
+    import pdfplumber
+
+    target = out_dir / f"{stem}_images"
+    tables_by_page = {}
+    for table in doc.get("tables") or []:
+        tables_by_page.setdefault(table["page"], []).append(table)
+    sizes = {p["page"]: p.get("size") or {} for p in doc.get("pages") or []}
+    wanted = {p["page"] for p in doc.get("pages") or []}
+
+    figures, used_names = [], set()
+    row_images = {t["id"]: [[] for _ in (t.get("rows_data") or [])]
+                  for t in doc.get("tables") or []}
+    anchored = unanchored = 0
+
+    with pdfplumber.open(pdf_path) as pdf:
+        for page_no in sorted(wanted):
+            if page_no > len(pdf.pages):
+                continue
+            page = pdf.pages[page_no - 1]
+            images = page_images(page)
+            tables = tables_by_page.get(page_no, [])
+            anchors = anchor_rows(tables, word_index(page), page.height) if tables else []
+            anchored += sum(1 for a in anchors if a.get("band"))
+            unanchored += sum(1 for a in anchors if not a.get("band"))
+            banded = [a for a in anchors if a.get("band")]
+
+            # Asked row-first rather than image-first. The photo column keeps its
+            # own vertical rhythm — it drifts a few points away from the text
+            # rows over the length of a page — so "which rows does this image
+            # cover" leaves a row stranded in the gap between two photos, while
+            # "which image is this row nearest to" always answers. An image
+            # taller than one line is a merged cell covering a run of products,
+            # exactly like the category column, and several rows simply pick it.
+            claims = {}
+            for anchor in banded:
+                band = anchor["band"]
+                best = None
+                for index, image in enumerate(images):
+                    _, top, _, bottom = image["bbox"]
+                    covered = overlap((top, bottom), band)
+                    if covered < MIN_OVERLAP:
+                        continue
+                    share = covered / max(1e-6, min(bottom - top, band[1] - band[0]))
+                    if best is None or (share, covered) > best[0]:
+                        best = ((share, covered), index)
+                if best and best[0][0] >= MIN_SHARE:
+                    claims.setdefault(best[1], []).append(anchor)
+
+            target.mkdir(parents=True, exist_ok=True)
+            for number, image in enumerate(images):
+                x0, top, x1, bottom = image["bbox"]
+                touching = claims.get(number, [])
+                primary = touching[0] if touching else None
+
+                name = safe_stem(page_no, primary["sku"] if primary else "", used_names)
+                crop = page.crop((max(0.0, x0 - PAD), max(0.0, top - PAD),
+                                  min(page.width, x1 + PAD),
+                                  min(page.height, bottom + PAD)))
+                path = target / f"{name}.png"
+                # Saved through PIL rather than PageImage.save, which writes PNG
+                # bytes under whatever extension it is handed — a file called
+                # .jpg that is really a PNG breaks importers that trust the name.
+                crop.to_image(resolution=dpi).original.convert("RGB").save(path)
+
+                entry = {
+                    "name": path.name,
+                    "file": f"{stem}_images/{path.name}",
+                    "page": page_no,
+                    "source": "pdf-render",
+                    "bbox_pdf": [round(v, 1) for v in image["bbox"]],
+                    "rows": [{"table": a["table"], "row": a["row"], "sku": a["sku"]}
+                             for a in touching],
+                }
+                size = sizes.get(page_no) or {}
+                if size.get("width") and size.get("height"):
+                    sx, sy = size["width"] / page.width, size["height"] / page.height
+                    entry["bbox"] = [round(x0 * sx), round(top * sy),
+                                     round(x1 * sx), round(bottom * sy)]
+                figures.append(entry)
+                for a in touching:
+                    row_images[a["table"]][a["row"]].append(path.name)
+
+            # Datalab's own Picture blocks are rendered too. Their bytes only
+            # come back when the API was asked for them, so without this every
+            # figure link in the Markdown points at a file that was never
+            # written — a broken image reads as a failed extraction.
+            size = sizes.get(page_no) or {}
+            if size.get("width") and size.get("height"):
+                sx, sy = page.width / size["width"], page.height / size["height"]
+                for figure in doc.get("figures") or []:
+                    if figure.get("page") != page_no or not figure.get("bbox"):
+                        continue
+                    fx0, ftop, fx1, fbot = figure["bbox"]
+                    box = (max(0.0, fx0 * sx), max(0.0, ftop * sy),
+                           min(page.width, fx1 * sx), min(page.height, fbot * sy))
+                    if box[2] - box[0] < MIN_SIDE or box[3] - box[1] < MIN_SIDE:
+                        continue
+                    name = Path(figure["name"]).stem + ".png"
+                    page.crop(box).to_image(resolution=dpi).original \
+                        .convert("RGB").save(target / name)
+                    figure["file"] = f"{stem}_images/{name}"
+                    figure["file_source"] = "pdf-render"
+
+    for table in doc.get("tables") or []:
+        names = row_images.get(table["id"]) or []
+        if any(names):
+            table["row_images"] = names
+            table["image_column"] = image_column(table)
+            for i, record in enumerate(table.get("records") or []):
+                record["images"] = names[i]
+
+    for existing in doc.get("figures") or []:
+        existing.setdefault("source", "datalab")
+    doc.setdefault("figures", [])
+    doc["figures"] = list(doc["figures"]) + figures
+
+    rows_total = doc.get("summary", {}).get("table_rows", 0)
+    with_photo = sum(1 for t in doc.get("tables") or []
+                     for names in (t.get("row_images") or []) if names)
+    doc.setdefault("summary", {})["page_images"] = {
+        "extracted": len(figures),
+        "dpi": dpi,
+        "rows_with_image": with_photo,
+        "rows_without_image": rows_total - with_photo,
+        "rows_anchored": anchored,
+        "rows_unanchored": unanchored,
+        "unplaced_images": sum(1 for f in figures if not f["rows"]),
+    }
+    return doc["summary"]["page_images"]
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("document", type=Path, help="a <stem>.json from pdf_extract.py")
+    parser.add_argument("--pdf", type=Path, help="override the PDF path recorded in the JSON")
+    parser.add_argument("--dpi", type=int, default=200)
+    args = parser.parse_args()
+
+    doc = json.loads(args.document.read_text())
+    pdf = args.pdf or Path(doc["source"]["path"])
+    if not pdf.is_file():
+        parser.error(f"PDF not found: {pdf}")
+
+    stats = attach_figures(doc, pdf, args.document.parent, args.document.stem, args.dpi)
+    args.document.write_text(json.dumps(doc, indent=2, ensure_ascii=False) + "\n")
+    print(json.dumps(stats, indent=2), file=sys.stderr)
+    print(f"  json -> {args.document}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

package/skills/pdf-to-json/scripts/load_mongo.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""Load extracted price-list rows into MongoDB, keyed on SKU.
+
+    python scripts/load_mongo.py <stem>.json --collection stanley_tools [--dry-run]
+
+Five fields per product: Category, Sku, Description, List Price AED and
+Suggested End User Price (AED). Prices are stored as numbers rather than the
+strings the PDF printed, because a price list you cannot sort or compare on is
+a price list you have to re-parse at every query.
+
+Two things this refuses to do quietly, both learned from the Stanley file:
+
+Not every extracted row is a product. A reprinted column header that wrapped
+across two lines, or a category label printed sideways in the page margin and
+swept into the table body, arrives looking exactly like a row. Loaded, they
+become products called "Category Sku" and "h al T ols nic Me a o c". Rows whose
+SKU is not code-like are held back and listed, not dropped in silence — the
+count of what did not load is the number worth seeing.
+
+SKU is the upsert key, so a SKU the document uses twice collapses two products
+into one. On the Stanley list `1-51-033` is both a 16oz and a 20oz hammer. That
+is the document's error rather than the extraction's, but the loss happens here,
+so it is reported here.
+
+The connection string comes from MONGODB_URI (or MONGO_URI) in the environment,
+or from any .env found walking up from the JSON, the working directory and this
+script — the last of those is what finds the project's Backend/.env when the
+command is run from wherever the PDF happens to live, which is usually outside
+the repo. It is never printed: credentials in a scrollback outlive the session.
+"""
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from pdf_extract import find_env_value, normalise_header
+
+URI_KEYS = ("MONGODB_URI", "MONGO_URI")
+
+# What the PDF calls each field, normalised. The Stanley list runs two schemas
+# at once and the narrower one truncates its last header to "Suggested End", so
+# matching on the full name alone silently drops the price on 6 of 26 blocks.
+FIELDS = {
+    "Category": ("category",),
+    "Sku": ("sku",),
+    "Description": ("description",),
+    "List Price AED": ("listpriceaed", "listprice"),
+    "Suggested End User Price (AED)": ("suggestedenduserpriceaed", "suggestedend",
+                                       "suggesteduserprice"),
+}
+NUMERIC = ("List Price AED", "Suggested End User Price (AED)")
+SKU_RE = re.compile(r"[A-Za-z0-9][\w./-]*")
+
+
+def to_number(value: str):
+    """A price as a number, or None if the cell does not hold one."""
+    text = (value or "").replace(",", "").strip()
+    if not text:
+        return None
+    match = re.fullmatch(r"(\d+(?:\.\d+)?)", text)
+    return float(match.group(1)) if match else None
+
+
+def column_map(headers):
+    seen = {normalise_header(h): i for i, h in enumerate(headers or []) if h}
+    return {field: next((seen[a] for a in aliases if a in seen), None)
+            for field, aliases in FIELDS.items()}
+
+
+def documents(doc: dict):
+    """Turn the tables into products, keeping what was rejected and why."""
+    products, rejected = [], []
+    for table in doc.get("tables") or []:
+        columns = column_map(table.get("headers"))
+        categories = table.get("categories") or []
+        fills = table.get("category_fill") or []
+        for i, row in enumerate(table.get("rows_data") or []):
+            def cell(field):
+                index = columns.get(field)
+                return (row[index] or "").strip() if index is not None and index < len(row) else ""
+
+            sku = cell("Sku")
+            # The filled category, not the printed cell: a rowspan-merged column
+            # prints its value once and leaves the rest of the run blank.
+            category = categories[i] if i < len(categories) else cell("Category")
+            record = {
+                "Category": category,
+                "Sku": sku,
+                "Description": cell("Description"),
+                "List Price AED": to_number(cell("List Price AED")),
+                "Suggested End User Price (AED)": to_number(
+                    cell("Suggested End User Price (AED)")),
+            }
+            context = {"page": table.get("page"), "table": table.get("id"), "row": i,
+                       "category_fill": fills[i] if i < len(fills) else None}
+
+            if not sku:
+                rejected.append((record, context, "no SKU"))
+            elif not SKU_RE.fullmatch(sku):
+                # Spaces and stray capitals are how a wrapped header line and a
+                # sideways margin label both present once they land in a cell.
+                rejected.append((record, context, "SKU is not a product code"))
+            elif record["List Price AED"] is None:
+                rejected.append((record, context, "no usable List Price AED"))
+            else:
+                products.append((record, context))
+    return products, rejected
+
+
+def connection_uri(document: Path | None):
+    """The URI from the environment or a .env, and where it points, minus secrets.
+
+    The script's own directory is searched as well as the JSON's and the working
+    directory: a skill is invoked from wherever the PDF lives, which is often
+    outside the repo that holds Backend/.env, and walking up from the script is
+    what reaches it. Both the symlinked and the real path, since the skill is
+    reachable through either and only one of them sits under the repo.
+    """
+    here = Path(__file__).parent
+    roots = [Path.cwd(), here.absolute(), here.resolve()]
+    if document is not None:
+        roots.insert(0, document.resolve().parent)
+    for key in URI_KEYS:
+        uri = os.environ.get(key) or find_env_value(key, roots)
+        if uri:
+            # Host and database only. Everything before the '@' is the password.
+            tail = re.sub(r"^[^:]+://(?:[^@/]*@)?", "", uri)
+            host, _, path = tail.partition("/")
+            return uri, key, host, path.split("?")[0] or None
+    return None, None, None, None
+
+
+def describe_target(uri, database_name, collection_name):
+    """What is actually at the other end, read-only — writes nothing."""
+    from pymongo import MongoClient
+    client = MongoClient(uri, serverSelectionTimeoutMS=15000, appname="pdf-to-json")
+    try:
+        database = client.get_database(database_name) if database_name \
+            else client.get_default_database()
+        if database is None:
+            raise SystemExit("the URI names no database — pass --database")
+        names = sorted(database.list_collection_names())
+        print(f"database: {database.name}", file=sys.stderr)
+        print(f"collections ({len(names)}):", file=sys.stderr)
+        for name in names:
+            mark = " <- target" if name == collection_name else ""
+            print(f"    {database[name].estimated_document_count():7d}  {name}{mark}",
+                  file=sys.stderr)
+        if collection_name not in names:
+            print(f"  '{collection_name}' is not in this database", file=sys.stderr)
+    finally:
+        client.close()
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("document", type=Path, nargs="?",
+                        help="a <stem>.json from pdf_extract.py")
+    parser.add_argument("--collection", required=True, help="collection to upsert into")
+    parser.add_argument("--database", help="defaults to the database named in the URI")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="report exactly what would be written, and connect to nothing")
+    parser.add_argument("--check", action="store_true",
+                        help="report which host and database the URI points at, and what "
+                             "collections are already there; writes nothing")
+    parser.add_argument("--emit", type=Path,
+                        help="write the validated documents to a JSON file instead of "
+                             "loading them, for review or for a loader of your own")
+    args = parser.parse_args()
+
+    if args.check:
+        uri, key, host, path = connection_uri(args.document)
+        if not uri:
+            parser.error(f"no connection string: set one of {', '.join(URI_KEYS)}")
+        print(f"{key} -> {host}, database in URI: {path or '(none)'}", file=sys.stderr)
+        describe_target(uri, args.database, args.collection)
+        return
+
+    if args.document is None:
+        parser.error("a <stem>.json is required unless --check is given")
+
+    doc = json.loads(args.document.read_text())
+    products, rejected = documents(doc)
+
+    counts = {}
+    for record, _, reason in rejected:
+        counts[reason] = counts.get(reason, 0) + 1
+    seen = {}
+    for record, context in products:
+        seen.setdefault(record["Sku"], []).append(context)
+    collisions = {s: c for s, c in seen.items() if len(c) > 1}
+
+    total = doc.get("summary", {}).get("table_rows", len(products) + len(rejected))
+    print(f"{total} extracted rows -> {len(products)} products "
+          f"({len(seen)} distinct SKUs), {len(rejected)} held back", file=sys.stderr)
+    for reason, n in sorted(counts.items(), key=lambda kv: -kv[1]):
+        print(f"    {n:4d}  {reason}", file=sys.stderr)
+    for sku, places in collisions.items():
+        pages = ", ".join(f"p{c['page']}" for c in places)
+        print(f"  WARNING SKU {sku} appears {len(places)} times ({pages}); "
+              f"upserting on SKU keeps only the last", file=sys.stderr)
+    inferred = sum(1 for _, c in products if (c["category_fill"] or "").startswith("carried"))
+    if inferred:
+        print(f"  NOTE {inferred} of {len(products)} categories are inferred from "
+              f"neighbouring rows, not stated by the document", file=sys.stderr)
+
+    if rejected:
+        print("  held back:", file=sys.stderr)
+        for record, context, reason in rejected[:25]:
+            print(f"    p{context['page']:<3} {reason:<28} "
+                  f"Sku={record['Sku']!r} Description={record['Description'][:34]!r}",
+                  file=sys.stderr)
+        if len(rejected) > 25:
+            print(f"    ... and {len(rejected) - 25} more", file=sys.stderr)
+
+    if args.emit:
+        # Last write wins, matching what upserting on Sku would leave behind, so
+        # the file is what the collection would contain rather than a longer list.
+        deduped = {r["Sku"]: r for r, _ in products}
+        args.emit.write_text(json.dumps(list(deduped.values()), indent=2,
+                                        ensure_ascii=False) + "\n")
+        print(f"\n{len(deduped)} documents -> {args.emit} (nothing loaded)", file=sys.stderr)
+        return
+
+    if args.dry_run:
+        print(f"\ndry run: nothing written. Would upsert {len(seen)} documents "
+              f"into '{args.collection}'.", file=sys.stderr)
+        print(json.dumps([r for r, _ in products[:3]], indent=2, ensure_ascii=False))
+        return
+
+    uri, key, host, _ = connection_uri(args.document)
+    if not uri:
+        parser.error(f"no connection string: set one of {', '.join(URI_KEYS)}")
+    # Naming the host is not the same as printing the URI, and without it a load
+    # that went to the wrong cluster looks identical to one that went to the right
+    # one — the collection is simply "missing" wherever you happen to be looking.
+    print(f"\nconnecting via {key} -> {host}", file=sys.stderr)
+
+    from pymongo import MongoClient, UpdateOne
+    from pymongo.errors import PyMongoError
+
+    client = MongoClient(uri, serverSelectionTimeoutMS=15000, appname="pdf-to-json")
+    database = client.get_database(args.database) if args.database \
+        else client.get_default_database()
+    if database is None:
+        parser.error("the URI names no database — pass --database")
+    collection = database[args.collection]
+
+    try:
+        collection.create_index("Sku", unique=True)
+        result = collection.bulk_write(
+            [UpdateOne({"Sku": r["Sku"]}, {"$set": r}, upsert=True) for r, _ in products],
+            ordered=False)
+        # Read the collection back rather than trusting the write acknowledgement.
+        # A driver report of "788 inserted" describes the request that was
+        # accepted, and it is possible to hold one of those in hand while the
+        # documents are not in the database you think you are looking at — so
+        # count them, name the database, and let the two figures disagree out loud.
+        stored = collection.count_documents({})
+        listed = args.collection in database.list_collection_names()
+    except PyMongoError as error:
+        # Deliberately not printing the URI: it carries the password.
+        print(f"MongoDB refused the write: {error}", file=sys.stderr)
+        raise SystemExit(1)
+    finally:
+        client.close()
+
+    print(f"\n{host}/{database.name}.{args.collection}: "
+          f"{result.upserted_count} inserted, {result.modified_count} updated, "
+          f"{len(seen) - result.upserted_count - result.modified_count} unchanged",
+          file=sys.stderr)
+    missing = "" if listed else " — but it is NOT in list_collection_names()"
+    print(f"  verified: {stored} documents now in the collection{missing}",
+          file=sys.stderr)
+    if stored < len(seen) or not listed:
+        print("  WARNING the write was acknowledged but the collection does not read "
+              "back as expected — check you are inspecting this host and database",
+              file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()