@sitar_fiercer4c/skills 0.1.0

package/skills/clarke-list-excel/scripts/load_clarke.py

@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+"""Upload the product photos to S3 and upsert the products into MongoDB.
+
+    python scripts/load_clarke.py <out-dir>/clarke-products.json            # loads
+    python scripts/load_clarke.py <out-dir>/clarke-products.json --dry-run  # preview
+    python scripts/load_clarke.py <out-dir>/clarke-products.json --check    # where does it point?
+
+**This writes by default.** The owner of the bucket and the cluster has given
+standing permission for this pipeline, so making the load conditional on a
+second flag just meant the data sat on disk while someone retyped the command.
+
+What that removes is a prompt, not the care behind it. The run still prints
+which .env it read and which host and database it reached, still refuses to
+touch the database if any upload failed, still names duplicate part numbers
+before collapsing them, and still counts the collection back through the
+connection it wrote through rather than trusting the acknowledgement.
+
+Order is deliberate: photos first, products second. Each product stores the key
+of its image, so uploading first means a document never references an object
+that is not there. If the upload half fails, no products are written and the
+run can simply be repeated.
+
+Products go to the `clarke` collection, which holds this catalogue and nothing
+else. That is worth stating because it was not always so: these products were
+first loaded into `uken-products` alongside the UKEN catalogue, and the
+collision guard below was written for that arrangement. A dedicated collection
+removes the risk rather than managing it — nothing else keys on partNo here, so
+there is nothing to overwrite.
+
+The guard is kept anyway. It costs one query, and `--collection` can still
+point this at a shared collection, which is exactly when an upsert that
+silently overwrote another catalogue's product would leave nothing behind to
+notice.
+"""
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from clarke_common import credential, load_env, require
+
+COLLECTION = "clarke"
+# S3 keys have no leading slash. The brief says "/products/catalog"; written
+# literally that creates a bucket entry whose first path segment is empty,
+# which renders as a nameless folder in the console and is a nuisance to
+# remove. The vendor segment keeps this catalogue's objects separable from the
+# UKEN ones already under the same prefix.
+PREFIX = "products/catalog/clarke"
+SUPPLIER = "Clarke"
+
+CONTENT_TYPES = {"jpeg": "image/jpeg", "jpg": "image/jpeg", "png": "image/png",
+                 "gif": "image/gif", "webp": "image/webp", "bmp": "image/bmp"}
+# Formats a browser will not render. They are uploaded only if someone asks,
+# because an <img> pointing at an EMF is a broken image on a product page and
+# is indistinguishable from a product that simply has no photo.
+UNRENDERABLE = {"emf", "wmf", "tiff", "tif", "unknown", "bin"}
+
+UNSAFE = re.compile(r"[^A-Za-z0-9._-]+")
+
+
+def object_key(part_no: str, extension: str) -> str:
+    """`products/catalog/clarke/<part no>.<ext>`, safe for a key.
+
+    Part numbers print with spaces, quotes and slashes — `1/2 DR` is a real
+    one. A slash left in place silently becomes a folder level, so the object
+    lands somewhere nobody will look for it; the rest are merely awkward. The
+    printed form survives on the document as `partNoRaw`, so nothing is lost
+    by flattening it here.
+    """
+    safe = UNSAFE.sub("-", part_no).strip("-") or "unnamed"
+    return f"{PREFIX}/{safe}.{extension}"
+
+
+def document(product: dict, image_key: str | None, source: str) -> dict:
+    return {
+        "partNo": product["partNo"],
+        # Kept because the key is flattened for S3 and normalised for Mongo;
+        # this is what the price list actually printed.
+        "partNoRaw": product.get("partNoRaw"),
+        "description": product["description"],
+        "category": product.get("category"),
+        "subCategory": product.get("subCategory"),
+        "pktQty": product.get("pktQty"),
+        "ctnQty": product.get("ctnQty"),
+        "price": product.get("price"),
+        "priceLabel": product.get("priceLabel", "TRADER Price"),
+        "currency": product.get("currency", "AED"),
+        # Provenance rather than a discriminator now that the collection holds
+        # one catalogue — but it is what lets these documents be merged into a
+        # shared collection later without becoming anonymous, and it is what
+        # the collision guard reads.
+        "supplier": SUPPLIER,
+        "source": source,
+        "sheetRow": product.get("sheetRow"),
+        "imageKey": image_key,
+        # "span" means the photo's own extent covered this row; "nearest"
+        # means it covered no product row and was attached to the closest one.
+        # The second is inference and a reader should be able to see which.
+        "imageMatch": product.get("imageMatch"),
+        # True when the photo's span covered several product rows and this is
+        # one of them — the sheet illustrated the group, not this part number
+        # alone. Worth carrying: it is the difference between "here is this
+        # product" and "here is what this family looks like".
+        "imageShared": bool(product.get("imageShared")),
+    }
+
+
+def resolve(args):
+    values, env_path = load_env(args.env)
+    return values, env_path
+
+
+def check(values, env_path) -> int:
+    """Answer 'where does this point?' without writing anything."""
+    from pymongo import MongoClient
+    import boto3
+
+    print(f"credentials: {env_path}")
+    bucket = require(values, "S3_BUCKET")
+    region = credential(values, "AWS_REGION") or "us-east-1"
+    print(f"s3: s3://{bucket}/{PREFIX}/  (region {region})")
+    session = boto3.session.Session(
+        aws_access_key_id=require(values, "AWS_ACCESS_KEY_ID"),
+        aws_secret_access_key=require(values, "AWS_SECRET_ACCESS_KEY"),
+        region_name=region)
+    s3 = session.client("s3")
+    listing = s3.list_objects_v2(Bucket=bucket, Prefix=PREFIX + "/", MaxKeys=5)
+    print(f"    objects already under that prefix: {listing.get('KeyCount', 0)}")
+
+    uri = require(values, "MONGODB_URI", "MONGO_URI")
+    client = MongoClient(uri, serverSelectionTimeoutMS=15000)
+    database = client.get_default_database()
+    print(f"mongo: {database.name} on "
+          f"{','.join(h[0] for h in client.topology_description.server_descriptions())}")
+    collection = database[COLLECTION]
+    total = collection.estimated_document_count()
+    mine = collection.count_documents({"supplier": SUPPLIER})
+    print(f"    {COLLECTION}: {total:,} documents, {mine:,} of them {SUPPLIER}")
+    print(f"    indexes: {[i['name'] for i in collection.list_indexes()]}")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("json_path")
+    parser.add_argument("--env")
+    parser.add_argument("--collection", default=COLLECTION)
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--check", action="store_true")
+    parser.add_argument("--upload-unrenderable", action="store_true",
+                        help="also upload EMF/WMF media a browser cannot show")
+    args = parser.parse_args()
+
+    values, env_path = resolve(args)
+    if args.check:
+        return check(values, env_path)
+
+    payload = json.loads(Path(args.json_path).read_text())
+    products = payload["products"]
+    source = payload["source"]["file"]
+    image_dir = Path(args.json_path).resolve().parent / "images"
+
+    print(f"credentials: {env_path}")
+
+    duplicates = payload["summary"].get("duplicate_part_numbers") or {}
+    if duplicates:
+        print(f"note: {len(duplicates)} part numbers appear twice in the "
+              f"document and will collapse to one document each on upsert:")
+        for part, rows in list(duplicates.items())[:10]:
+            print(f"    {part} at rows {', '.join(str(r) for r in rows)}")
+
+    # ---- work out which objects to upload
+    uploads, skipped_format = {}, []
+    for product in products:
+        digest = product.get("imageSha256")
+        if not digest:
+            continue
+        extension = (product.get("imageFormat") or "unknown").lower()
+        if extension in UNRENDERABLE and not args.upload_unrenderable:
+            skipped_format.append((product["partNo"], extension))
+            continue
+        path = image_dir / f"{digest}.{extension}"
+        if not path.is_file():
+            raise SystemExit(
+                f"{path} is missing — run clarke_extract.py first. Loading "
+                f"without it would write documents pointing at objects that "
+                f"do not exist, and every other field on them would be right.")
+        uploads[product["partNo"]] = (path, object_key(product["partNo"], extension))
+
+    if skipped_format:
+        print(f"note: {len(skipped_format)} photos are in a format browsers "
+              f"cannot display and were left out "
+              f"({', '.join(sorted({e for _, e in skipped_format}))}); "
+              f"pass --upload-unrenderable to send them anyway")
+
+    print(f"{len(products):,} products, {len(uploads):,} photos to upload")
+
+    if args.dry_run:
+        print("\n--dry-run: nothing was written")
+        for part, (path, key) in list(uploads.items())[:5]:
+            print(f"    would PUT s3://{require(values, 'S3_BUCKET')}/{key} "
+                  f"({path.stat().st_size:,} bytes)")
+        if len(uploads) > 5:
+            print(f"    ... and {len(uploads) - 5:,} more")
+        sample = document(products[0], uploads.get(products[0]["partNo"], (None, None))[1],
+                          source)
+        print(f"    would upsert {len(products):,} documents into "
+              f"{args.collection}, keyed on partNo, e.g.:")
+        print("    " + json.dumps(sample, indent=2).replace("\n", "\n    "))
+        return 0
+
+    import boto3
+    from pymongo import MongoClient, UpdateOne
+
+    # ---- stage one: the photos
+    bucket = require(values, "S3_BUCKET")
+    region = credential(values, "AWS_REGION") or "us-east-1"
+    session = boto3.session.Session(
+        aws_access_key_id=require(values, "AWS_ACCESS_KEY_ID"),
+        aws_secret_access_key=require(values, "AWS_SECRET_ACCESS_KEY"),
+        region_name=region)
+    s3 = session.client("s3")
+
+    existing = {}
+    token = None
+    while True:
+        page = s3.list_objects_v2(**{"Bucket": bucket, "Prefix": PREFIX + "/",
+                                     **({"ContinuationToken": token} if token else {})})
+        for item in page.get("Contents") or []:
+            existing[item["Key"]] = item["Size"]
+        token = page.get("NextContinuationToken")
+        if not token:
+            break
+
+    uploaded = reused = 0
+    failures = []
+    for part, (path, key) in sorted(uploads.items()):
+        size = path.stat().st_size
+        # Idempotent by construction: an object already present at the same
+        # size is the same photo, so a re-run after a parsing fix does not
+        # re-send 238 files.
+        if existing.get(key) == size:
+            reused += 1
+            continue
+        extension = path.suffix.lstrip(".").lower()
+        try:
+            s3.put_object(Bucket=bucket, Key=key, Body=path.read_bytes(),
+                          ContentType=CONTENT_TYPES.get(extension,
+                                                        "application/octet-stream"))
+            uploaded += 1
+        except Exception as error:                        # noqa: BLE001
+            failures.append((part, key, str(error)))
+
+    print(f"s3://{bucket}/{PREFIX}/ — {uploaded:,} uploaded, "
+          f"{reused:,} already present, {len(failures)} failed")
+
+    if failures:
+        for part, key, error in failures[:5]:
+            print(f"    {part}: {error}")
+        raise SystemExit(
+            "uploads failed, so nothing was written to the database. A "
+            "document referencing a missing object is worse than no document "
+            "and is invisible from the database side. Fix and re-run.")
+
+    # ---- stage two: the products
+    uri = require(values, "MONGODB_URI", "MONGO_URI")
+    client = MongoClient(uri, serverSelectionTimeoutMS=20000)
+    database = client.get_default_database()
+    collection = database[args.collection]
+    hosts = ",".join(h[0] for h in client.topology_description.server_descriptions())
+    print(f"mongo: {database.name} on {hosts}, collection {args.collection}")
+
+    # The collision guard. Anything already in the collection under one of our
+    # part numbers but belonging to someone else would be overwritten in
+    # place, leaving no trace of what it was.
+    part_numbers = [p["partNo"] for p in products]
+    clashes = list(collection.find(
+        {"partNo": {"$in": part_numbers}, "supplier": {"$ne": SUPPLIER}},
+        {"partNo": 1, "supplier": 1, "source": 1}).limit(20))
+    if clashes:
+        print(f"\n{len(clashes)} part numbers already belong to another "
+              f"catalogue in {args.collection}:")
+        for clash in clashes[:10]:
+            print(f"    {clash['partNo']} — supplier "
+                  f"{clash.get('supplier') or '(none)'}, "
+                  f"source {clash.get('source') or '(none)'}")
+        raise SystemExit(
+            "refusing to upsert: these would overwrite another catalogue's "
+            "products in place. Decide what should happen to them first — "
+            "prefixing this catalogue's keys is the usual answer.")
+
+    collection.create_index("partNo", unique=True)
+    operations = [
+        UpdateOne({"partNo": product["partNo"]},
+                  {"$set": document(product,
+                                    uploads.get(product["partNo"], (None, None))[1],
+                                    source)},
+                  upsert=True)
+        for product in products
+    ]
+    result = collection.bulk_write(operations, ordered=False)
+    print(f"upserted: {result.upserted_count:,} new, "
+          f"{result.modified_count:,} updated")
+
+    # An acknowledgement is not evidence. Count back through the same
+    # connection the write went through.
+    expected = len({p["partNo"] for p in products})
+    actual = collection.count_documents({"supplier": SUPPLIER})
+    total = collection.estimated_document_count()
+    print(f"read back: {actual:,} {SUPPLIER} documents in {args.collection} "
+          f"(expected {expected:,}); {total:,} documents in total")
+    if actual != expected:
+        print(f"    mismatch of {abs(actual - expected):,} — investigate "
+              f"before treating this run as complete")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/clarke-list-excel/scripts/run_all.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Run the whole Clarke pipeline: extract, then upload and upsert.
+
+    python scripts/run_all.py "<xlsx>" --out-dir <dir>
+    python scripts/run_all.py "<xlsx>" --out-dir <dir> --dry-run
+
+Stops at the first failure. Loading from a half-finished extraction writes
+documents whose every field is right except that their photo is not there, and
+nothing about that looks wrong until someone opens a product page.
+"""
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+
+
+def run(command: list[str]) -> None:
+    print(f"\n$ {' '.join(command)}\n", flush=True)
+    result = subprocess.run(command)
+    if result.returncode != 0:
+        raise SystemExit(f"stage failed ({result.returncode}); nothing further ran")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("workbook")
+    parser.add_argument("--out-dir", default=".")
+    parser.add_argument("--env")
+    parser.add_argument("--sheet")
+    parser.add_argument("--refresh", action="store_true")
+    parser.add_argument("--no-datalab", action="store_true")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+
+    out_dir = Path(args.out_dir).expanduser()
+
+    extract = [sys.executable, str(HERE / "clarke_extract.py"), args.workbook,
+               "--out-dir", str(out_dir)]
+    if args.env:
+        extract += ["--env", args.env]
+    if args.sheet:
+        extract += ["--sheet", args.sheet]
+    if args.refresh:
+        extract.append("--refresh")
+    if args.no_datalab:
+        extract.append("--no-datalab")
+    run(extract)
+
+    load = [sys.executable, str(HERE / "load_clarke.py"),
+            str(out_dir / "clarke-products.json")]
+    if args.env:
+        load += ["--env", args.env]
+    if args.dry_run:
+        load.append("--dry-run")
+    run(load)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/skills/datalab-api/SKILL.md

@@ -0,0 +1,163 @@
+---
+name: datalab-api
+description: >-
+  Extract tables from spreadsheets (xlsx, xls, csv, ods) using the Datalab
+  Convert API at datalab.to — get each table's source cell range in A1
+  notation, headers, and rows. Use this skill whenever the user mentions
+  Datalab, datalab.to, marker, or a DATALAB_API_KEY, and also whenever they
+  want to pull tables, line items, bills of quantities, BOQs, quotes, or
+  priced item lists out of vendor spreadsheets — even if they never name the
+  API. Covers the async submit-and-poll flow, the block tree, the undocumented
+  spreadsheet bbox format, which parameters silently do nothing on
+  spreadsheets, and how to pick the one table that matters out of many.
+---
+
+# Datalab Convert API for spreadsheets
+
+Datalab segments a messy sheet into discrete **table blocks** — a sheet holding a
+title block, a line-item table, and terms-and-conditions comes back as three
+tables, each with the cell range it occupies. That segmentation is the value. If
+you only need cell values, `openpyxl` is free and instant. Reach for Datalab when
+you need to know *where the tables are* in a sheet nobody has described to you.
+
+## Start here
+
+The bundled script does the whole flow — submit, poll, walk the block tree,
+convert bboxes to A1 ranges, cache responses:
+
+```bash
+python scripts/datalab_tables.py <file-or-dir> --json tables.json --preview tables.md
+```
+
+`--json` is a machine-readable index of every table; `--preview` renders each
+table's headers plus first and last rows, which is what you want when a human
+needs to eyeball whether extraction went right. Responses cache under
+`./datalab_raw/` keyed by filename and mode, so re-runs are free — delete it to
+force a fresh call. Needs `DATALAB_API_KEY` in the environment, or in a
+`.env.local` / `.env` in any parent directory of the working directory.
+
+Two references cover the paths the script doesn't:
+
+- `references/parameters-and-payload.md` — the raw request, which parameters are
+  silently no-ops, how to prove it, and why responses are enormous. Read it
+  before setting any flag, especially anything about images or response size.
+- `references/table-selection.md` — automating table choice with a model call.
+
+## Reading the output
+
+`output_format=json` returns a recursive block tree: root → `Page` blocks (one
+per worksheet, named after the sheet) → `Table`, `Picture`, `Text`. Each table
+block carries `id` (a path within *this* response, e.g. `/page/BOH & FLS/Table/2`),
+`bbox`, `html`, and `children`. Blocks nest arbitrarily, so recurse rather than
+assuming depth — `find_tables()` in the script does it. Match `TableOfContents`
+too; Datalab sometimes classifies a plain lead-in table that way.
+
+**Block ids are not stable across runs.** The same workbook has come back as four
+table blocks on one call and five on the next, so today's `Table/2` is tomorrow's
+`Table/3`. The id is a handle within one response, nothing more. Resolve ids to
+A1 ranges as soon as you have them, and report and persist the *range* — that's
+the identifier that stays true to the source file.
+
+Parse `html` with the stdlib `html.parser`; no BeautifulSoup needed, and
+`to_grid()` already does it, padding ragged rows to equal width. Only `json`
+carries `bbox` — markdown and HTML lose table boundaries, which defeats the point.
+
+### bbox is cell coordinates, not pixels
+
+Undocumented, and the single most important thing to know:
+
+```
+bbox = [col_start, row_start, col_end, row_end]   1-indexed, inclusive
+```
+
+So `[1.0, 6.0, 8.0, 13.0]` means **A6:H13**. They arrive as floats but are whole
+numbers. For PDFs and images the same field holds pixel coordinates, so don't
+carry the assumption across file types. `excel_range()` converts, including
+multi-letter columns past Z. This has held on every spreadsheet tested, but it's
+inference, not contract — on a new workbook, open it with `openpyxl` and confirm
+the cells at the reported range are the table you expect.
+
+### Three bbox conventions coexist in one response
+
+The rule above is for `Table` blocks. Mixing these up silently produces wrong
+ranges:
+
+| Block | bbox | Meaning |
+|---|---|---|
+| `Table`, `Text` | `[1, 6, 9, 10]` | 1-indexed inclusive cell range → A6:I10 |
+| `Page` | `[0, 0, 10, 19]` | sheet *extent*: 10 columns × 19 rows, not a range |
+| `Picture` | `[6.29, 7.15, 6.64, 7.86]` | continuous — integer part is the 1-indexed cell, the fraction is the offset within it |
+
+A `Page` bbox always starts `[0, 0, ...]`; that leading zero is the tell, and
+`excel_range()` over it gives a bogus range at row 0. `Picture` bboxes floor to
+the anchoring cell — `[6.29, 7.15, ...]` is a photo in **F7** — which is what lets
+you map images back to rows.
+
+## Picking the right table
+
+A sheet typically yields several tables and you want one. Header rows are often
+blank, merged, or split across two rows, so structural heuristics are brittle;
+describing the target in prose and choosing from short previews works better.
+
+Describe it by columns and content, not size. "Each row is one physical item;
+columns give a category, an item name or description, and a quantity; price cells
+are sometimes blank for vendors to fill" selects reliably. "The biggest table"
+does not — terms-and-conditions blocks are often larger. Naming the likely
+distractors (title blocks, quote metadata, terms, delivery terms) helps more than
+elaborating the target.
+
+Five rows per table is enough signal, since what distinguishes tables is in the
+headers and first rows. Truncate cells to ~40 characters so paragraph-length item
+descriptions don't crowd out other tables. Allow a *list* of tables back — a
+workbook can legitimately hold two line-item tables, and a scalar silently
+discards one.
+
+## Checking extraction fidelity
+
+Datalab finds the right table consistently, but its boundaries are a first draft.
+Two drifts recur:
+
+- **Trailing rows that aren't items** — `Total cost DDP`, `VAT (if applicable)`,
+  `SUBTOTAL - NET`, usually a `SUM()` over the rows above, often with the label in
+  one cell and the rest blank. Datalab pulls them inside the table.
+- **A promoted header** — a sheet title one row above the real header gets
+  absorbed, so the reported range starts a row early.
+
+So open the workbook and check the edges rather than reporting the bbox as-is.
+
+Which range you *lead with* depends on what was asked. "Where does this block come
+from?" wants the block's actual extent — answer A6:I10 and mention the trailer
+separately. "Where is the line-item table?" wants the items — lead with A6:I9. Get
+this backwards and the headline number doesn't answer the question, even though
+both numbers appear somewhere in the reply.
+
+When items are what's wanted, a range excluding the trailers is *more* accurate
+than Datalab's, not a disagreement with it. Report the tighter range and say
+plainly which rows you dropped and why — "rows 11–13 are total/VAT/grand-total,
+not items" is the part the user actually needs, because it's the difference
+between 3 line items and 6. Some rows are genuinely borderline: a `FREIGHT /
+PACKING / DOCS` line with a quantity of 1 is arguably an item and arguably a cost
+trailer. Don't resolve that silently — name it and let the user decide.
+
+Trim rows, not columns. A sparse trailing column looks like padding but often
+isn't: one sheet's last column held only three cells, and they were a note and two
+vendor URLs. Narrowing the range to "tidy it up" deleted real data. Empty columns
+are safe to drop; populated ones need saying out loud. To confirm nothing was
+dropped, count non-empty rows per sheet with `openpyxl` and compare against the
+sum of rows across extracted tables — they should match, or Datalab should have
+slightly more.
+
+## Cost and docs
+
+Billing is per page, where a worksheet counts as a page; small workbooks cost 1–2
+credits. Datalab caches server-side, so re-uploading an identical file is free
+unless you pass `skip_cache=true`. Cache raw responses to disk anyway —
+experiments mean many re-runs over the same files.
+
+`https://documentation.datalab.to/llms.txt` lists every docs page as markdown;
+fetch it rather than guessing URLs, since the docs site 404s on plausible-looking
+paths and appending `.md` to any page returns clean markdown. The OpenAPI spec at
+`https://www.datalab.to/openapi.json` is the most complete parameter list but lags
+the running API — its `extras` enum omits `extract_bookmarks`, which the API's own
+422 message lists. When docs and observed behaviour disagree, trust the behaviour
+and verify with a deliberately invalid value.

package/skills/datalab-api/references/parameters-and-payload.md

@@ -0,0 +1,121 @@
+# Parameters, no-ops, and payload size
+
+Read this when the question is about request parameters, response size, or
+images — e.g. "can we turn off image extraction", "why is the response 3.7 MB",
+"does accurate mode help", or any time you're about to set a flag you haven't
+personally verified on a spreadsheet.
+
+## The raw request
+
+The script covers the normal path; write the request yourself only when you need
+parameters it doesn't expose. Conversion is async: the POST returns a
+`request_check_url` to poll every couple of seconds. Expect 13–25s for a typical
+workbook. Auth is `X-API-Key`, not a bearer token. There is no table-extraction
+endpoint — the old Table Recognition one was retired.
+
+```python
+resp = requests.post("https://www.datalab.to/api/v1/convert",
+    headers={"X-API-Key": key},
+    files={"file": (path.name, fh, mime)},
+    data={"output_format": "json", "mode": "fast"})
+check_url = resp.json()["request_check_url"]
+while (r := requests.get(check_url, headers={"X-API-Key": key}).json())["status"] != "complete":
+    time.sleep(2)
+```
+
+## Only three parameters matter
+
+`file` (multipart, correct MIME type per extension), `output_format=json` (the
+only format carrying `bbox`), and `mode=fast`.
+
+`mode` deserves emphasis because it's the one people reach for first. On
+spreadsheets it changes nothing, so use `fast`. On PDFs and images it's a real
+accuracy/latency tradeoff.
+
+## Most other parameters do nothing on spreadsheets
+
+The API accepts them, returns 200, and bills normally — the output is
+byte-identical to a control. All of these were tested that way on real xlsx
+files:
+
+| Parameter | Tested as |
+|---|---|
+| `mode` | `fast` / `balanced` / `accurate` — identical block trees, same cost and runtime |
+| `add_block_ids` | `true`, with `output_format=json,html` — zero `data-block-id` attributes |
+| `disable_image_extraction` | `true`, `True`, and with `disable_image_captions` — images still returned |
+| `extras=table_cell_bboxes` | with `json` and with `json,html` plus `word_bboxes=true` — zero `data-bbox` |
+| `extras=extract_links,new_block_types` | identical `json` |
+| `word_bboxes` | `true` — zero annotations |
+
+The bbox add-ons bill at $0.30/1K pages each and need `html` output to expose
+attributes, so on spreadsheets you'd pay for nothing.
+
+It's tempting to explain the no-ops as "spreadsheets skip the vision pipeline, so
+vision options are inert." Resist it — that mechanism has not been tested, and a
+PDF control showed `disable_image_extraction` doing nothing there either, which
+the story doesn't predict. Report the measured no-op and leave the cause open. A
+tidy explanation repeated confidently is how an untested guess ends up in an
+answer that reads as measured.
+
+One option does work: `additional_config={"keep_spreadsheet_formatting": true}`
+adds per-cell `data-excel-col`, fill colours, fonts and number formats, roughly
+4.5x-ing the payload. Useful if you need header shading or currency formats as
+signals. Skip it if you just need values — `data-excel-col` counts up
+sequentially from the bbox start, so it's already derivable.
+
+Asking for markdown instead of json doesn't shrink anything either: the json tree
+comes back regardless and can't be suppressed, and it's the json that carries the
+duplicated base64 images.
+
+## Proving a parameter is really a no-op
+
+**An unknown parameter name returns HTTP 200 and is silently ignored**, so a typo
+looks exactly like a feature that doesn't work. A known parameter with a bad
+value errors clearly:
+
+```
+disable_image_extractoin=true   -> 200  (typo, silently dropped)
+disable_image_extraction=banana -> 422  bool_parsing at body.disable_image_extraction
+mode=banana                     -> 400  "Must be one of 'fast', 'balanced', 'accurate'"
+extras=banana                   -> 400  lists all valid extras
+```
+
+So when a parameter seems to do nothing, send a deliberately invalid value first:
+a 400 or 422 naming your field proves it reached the server and the no-op is real
+behaviour, while a 200 means you misspelled it. Then compare against a control
+run with `skip_cache=true` on both sides and diff the actual payloads rather than
+trusting that a flag did what its name suggests. Several documented parameters
+turn out to be no-ops here with no caveat in the docs.
+
+## Images always come back
+
+`disable_image_extraction` has no effect on spreadsheets — images return whether
+you set it `true` or `false`. They arrive twice, under the top-level `images` key
+and inside each `Picture` block, and on image-heavy sheets that is ~98% of the
+payload: a 3.74 MB response can hold 10 KB of table data. Quote that figure
+carefully — it's the decoded size, and the API gzips, so ~3.7 MB decoded moved
+about 2.3 MB over the wire. Stripping images wins on storage, parsing and tokens
+into a model, not on bandwidth.
+
+**First ask whether the images are content.** Compare the `Picture` count to the
+row count of the line-item table. On product and BOQ sheets they often match
+nearly 1:1 — each row has a photo and the table has an `Image` column whose HTML
+cells come back empty, so those images *are* that column's data. One test sheet
+had 49 pictures against 48 item rows, with `Picture` bboxes landing on rows 4–51.
+
+There are two places to strip, and that gate decides which:
+
+- **After receipt** (default). Drop the `images` dict and each block's image
+  payload but keep the `Picture` blocks, so the bboxes survive and row↔photo
+  mapping stays recoverable if the images turn out to matter. The bundled script
+  does this by default; `--keep-images` opts out.
+- **Before upload**, only once you've confirmed the images are decorative — logos,
+  letterheads. Deleting `xl/media/` and `xl/drawings/` from the xlsx zip took one
+  upload from 9.28 MB to 10.9 KB and its response from 3.73 MB to 43 KB, wall time
+  21.3s → 16.4s, text fidelity 0.997. That's the only lever that shrinks the
+  *upload*, but it's destructive: you must also fix the sheet `.rels` and drawing
+  references or you hand Datalab a workbook it may reject, and the row↔photo link
+  is gone for good.
+
+"There's no API-side lever, so nothing can be done" is wrong on both counts — the
+client side is where the whole win is.