@sitar_fiercer4c/skills 0.1.0

package/skills/clarke-list-excel/scripts/clarke_extract.py

@@ -0,0 +1,487 @@
+#!/usr/bin/env python3
+"""Read the Clarke price list into JSON and Markdown, photos mapped to part numbers.
+
+    python scripts/clarke_extract.py "<xlsx>" --out-dir <dir>
+
+Writes `clarke-products.json` (every product, category and image mapping, plus
+a summary that says what did *not* come through) and `clarke-products.md` (the
+same thing rendered to read).
+
+The interesting work is mapping 249 floating photos onto 1,558 products. See
+`assign_images` — the naive version of that mapping loses 41 photos and looks
+perfectly healthy while doing it. A photo's span is read as the set of
+products it illustrates, so a picture drawn across several rows lands on every
+product in those rows.
+"""
+
+import argparse
+import hashlib
+import json
+import re
+import sys
+import warnings
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from clarke_common import (convert, credential, excel_range, load_env,
+                           media_index, picture_rows, table_blocks)
+
+warnings.filterwarnings("ignore", module="openpyxl")
+
+# Column letters as the sheet lays them out. Read by position rather than by
+# matching the printed header, because the header prints "CRTN QTY" and
+# "TRADER Price- Aed" — names that invite a fuzzy match that would just as
+# happily bind to the wrong column on next year's edition.
+COL_PART, COL_IMAGE, COL_DESC, COL_PKT, COL_CTN, COL_PRICE = 1, 2, 3, 4, 5, 6
+
+SUPPLIER = "Clarke"
+CURRENCY = "AED"
+
+# A part number is a code. Prose in the part-number column is a heading that
+# lost its formatting, and loading it makes a product called "Hand Tools Price
+# List- UAE".
+PART_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9./\-+&\" ]{0,29}$")
+
+
+def text(cell) -> str:
+    value = cell.value
+    if value is None:
+        return ""
+    return " ".join(str(value).split())
+
+
+def to_number(value):
+    """A price or quantity as a number, or None if the cell does not hold one.
+
+    openpyxl hands back real numbers for numeric cells, so the string path is
+    only for the ones typed as text — which in this workbook includes prices
+    written "8.00 " and quantities written "1 PC".
+    """
+    if value is None:
+        return None
+    if isinstance(value, (int, float)):
+        return float(value)
+    cleaned = str(value).replace(",", "").strip()
+    match = re.search(r"\d+(?:\.\d+)?", cleaned)
+    return float(match.group()) if match else None
+
+
+def find_header(sheet) -> int | None:
+    """The row printing PART NO in column A. Everything above it is a title."""
+    for row in range(1, min(sheet.max_row, 60) + 1):
+        if text(sheet.cell(row, COL_PART)).upper().replace(" ", "") == "PARTNO":
+            return row
+    return None
+
+
+def category_rows(sheet) -> dict:
+    """Which rows are headings, and at which level.
+
+    A heading and a product look identical from the values alone — both put
+    text in column A — so the level comes from the formatting the sheet
+    actually uses: a top-level category is merged across A:F and set in 14pt,
+    a sub-category is an unmerged 10pt cell. Reading the two apart matters
+    because the breadcrumb is the only grouping this document has; collapse
+    them and every product under "PLIERS" reports its sub-category as its
+    category instead.
+    """
+    merged = {r.min_row for r in sheet.merged_cells.ranges
+              if r.min_row == r.max_row and r.min_col == COL_PART and r.max_col >= 6}
+    levels = {}
+    for row in range(1, sheet.max_row + 1):
+        label = text(sheet.cell(row, COL_PART))
+        if not label or text(sheet.cell(row, COL_DESC)):
+            continue
+        size = sheet.cell(row, COL_PART).font.size or 0
+        levels[row] = "category" if (row in merged or size >= 14) else "subcategory"
+    return levels
+
+
+def read_products(sheet, header_row: int, levels: dict):
+    """Walk the sheet in order, carrying the current breadcrumb."""
+    products, held_back = [], []
+    category = subcategory = None
+
+    for row in range(header_row + 1, sheet.max_row + 1):
+        part = text(sheet.cell(row, COL_PART))
+        description = text(sheet.cell(row, COL_DESC))
+
+        if row in levels:
+            if levels[row] == "category":
+                category, subcategory = part, None
+            else:
+                subcategory = part
+            continue
+
+        if not part and not description:
+            continue
+
+        if not part or not description:
+            held_back.append({"row": row, "reason": "missing part number or description",
+                              "partNo": part, "description": description})
+            continue
+
+        if not PART_RE.match(part):
+            held_back.append({"row": row, "reason": "part number is not code-like",
+                              "partNo": part, "description": description})
+            continue
+
+        price = to_number(sheet.cell(row, COL_PRICE).value)
+        products.append({
+            "partNo": " ".join(part.split()),
+            "partNoRaw": part,
+            "description": description,
+            "category": category,
+            "subCategory": subcategory,
+            "pktQty": to_number(sheet.cell(row, COL_PKT).value),
+            "ctnQty": to_number(sheet.cell(row, COL_CTN).value),
+            "price": price,
+            "priceLabel": "TRADER Price",
+            "currency": CURRENCY,
+            "supplier": SUPPLIER,
+            "sheetRow": row,
+        })
+    return products, held_back
+
+
+def placements(sheet, datalab_spans: dict):
+    """Every photo in the sheet: its anchor row, its span, and its bytes.
+
+    Three things here are not what the obvious API suggests:
+
+    - `image.path` is a template, not a location. openpyxl reports
+      `/xl/media/image1.jpeg` for 244 of this workbook's 249 placements, so
+      keying media on it collapses the whole catalogue onto two pictures.
+      `image._data()` returns the real bytes and is what gets hashed.
+    - These are one-cell anchors, so `anchor.to` is None and openpyxl knows
+      only where a photo *starts*. The span comes from Datalab, whose Picture
+      bbox carries both edges.
+    - The same media file is placed under several part numbers. That is the
+      document repeating a photo, not an extraction fault, so identical bytes
+      are recorded once and referenced many times.
+    """
+    out = []
+    by_row: dict[int, list] = {}
+    for image in sheet._images:
+        anchor = image.anchor._from
+        by_row.setdefault(anchor.row + 1, []).append(image)
+
+    for row in sorted(by_row):
+        spans = list(datalab_spans.get(row, []))
+        for index, image in enumerate(sorted(by_row[row],
+                                             key=lambda i: i.anchor._from.col)):
+            try:
+                data = image._data()
+            except Exception as error:                    # noqa: BLE001
+                out.append({"anchorRow": row, "error": str(error)})
+                continue
+            span_to = spans[index] if index < len(spans) else row
+            out.append({
+                "anchorRow": row,
+                "spanTo": max(row, span_to),
+                "sha256": hashlib.sha256(data).hexdigest(),
+                "format": (image.format or "").lower() or "unknown",
+                "bytes": len(data),
+                "data": data,
+            })
+    return out
+
+
+def assign_images(photos, products, levels):
+    """Attach each photo to every product its span covers, and say how confidently.
+
+    A photo is anchored in the IMAGES column beside its row, but "beside" is
+    approximate: the anchor lands on whatever cell the top-left corner falls
+    in, and these photos are taller than one row. Matching on the anchor row
+    alone attaches 208 of 249 — the other 41 are anchored on a blank spacer
+    row or on the heading above their product, so they are silently dropped
+    while the run reports no error at all.
+
+    Reading the photo's whole span fixes that, and the span is also the
+    document's own statement of which products the picture is *of*: the one
+    photo drawn beside twenty rows of combination spanners illustrates all
+    twenty sizes. So every product row a photo covers receives it, not just
+    the first. Only when a photo covers no data row at all does it fall back
+    to the nearest one, and that case is recorded as `nearest` rather than
+    `span` so a reader can see which attachment was inferred. Photos are
+    searched outward by at most three rows, because the failure that guards
+    against is the opposite one — a generous radius will happily attach the
+    logo in row 1 to the first product forty rows below it.
+
+    Two things bound the fan-out, and both exist because a wrong photo on a
+    right-looking product is the one error nothing downstream can show you:
+
+    - **A span stops at a heading.** Some spans run past a sub-category row
+      into the group below — the wood carving set photo on row 447 reaches row
+      450, a nail chisel — and a heading is the sheet's own statement that
+      what follows is a different product. Counted as
+      `spans_truncated_at_heading`.
+    - **Where spans overlap, the nearest anchor above the row wins.** Taking
+      the first span to reach a row instead lets one tall photo swallow rows
+      that a later photo is anchored directly beside, and costs ten distinct
+      pictures their only home. Eleven placements still lose every row they
+      cover, because the sheet stacks photos over the same rows and a product
+      carries one; they are counted as `images_overridden`.
+    """
+    by_row = {p["sheetRow"]: p for p in products}
+    rows = sorted(by_row)
+    unmatched = []
+    candidates: dict[int, list] = {}
+
+    for photo in photos:
+        if "error" in photo:
+            unmatched.append(photo)
+            continue
+        anchor, span_to = photo["anchorRow"], photo["spanTo"]
+
+        covered, truncated = [], False
+        for row in range(anchor, span_to + 1):
+            if row in levels and covered:
+                truncated = True
+                break
+            if row in by_row:
+                covered.append(row)
+        photo["truncatedAtHeading"] = truncated
+        photo["how"] = "span"
+
+        if not covered:
+            for distance in range(1, 4):
+                for candidate in (anchor - distance, span_to + distance):
+                    if candidate in by_row:
+                        covered, photo["how"] = [candidate], "nearest"
+                        break
+                if covered:
+                    break
+        if not covered:
+            unmatched.append(photo)
+            continue
+
+        photo["covers"] = covered
+        for row in covered:
+            candidates.setdefault(row, []).append(photo)
+
+    # One image per product: the photo anchored nearest above the row, which
+    # is the one the sheet draws beside it. A "span" match beats a "nearest"
+    # one at the same distance, because the second is inference.
+    for photo in photos:
+        photo["attachedRows"] = []
+    for row, contenders in candidates.items():
+        winner = max(contenders,
+                     key=lambda ph: (ph["how"] == "span", ph["anchorRow"]))
+        product = by_row[row]
+        product["imageSha256"] = winner["sha256"]
+        product["imageFormat"] = winner["format"]
+        product["imageMatch"] = winner["how"]
+        product["imageAnchorRow"] = winner["anchorRow"]
+        product["imageSpanRows"] = [winner["anchorRow"], winner["spanTo"]]
+        winner["attachedRows"].append(row)
+
+    # A product sharing its photo with a sibling row is worth being able to
+    # see: it is the difference between "the sheet illustrated this product"
+    # and "the sheet illustrated this product's group".
+    for photo in photos:
+        if len(photo["attachedRows"]) > 1:
+            for target in photo["attachedRows"]:
+                by_row[target]["imageShared"] = True
+
+    return rows, unmatched
+
+
+def render_markdown(payload: dict) -> str:
+    summary = payload["summary"]
+    lines = [f"# {payload['source']['file']}", ""]
+    lines.append(f"{summary['products']:,} products · "
+                 f"{summary['categories']} categories · "
+                 f"{summary['subCategories']} sub-categories · "
+                 f"{summary['products_with_image']:,} with a photo")
+    lines.append("")
+    current = None
+    for product in payload["products"]:
+        breadcrumb = " → ".join(x for x in (product["category"],
+                                            product["subCategory"]) if x)
+        if breadcrumb != current:
+            current = breadcrumb
+            lines += ["", f"## {breadcrumb or 'Uncategorised'}", "",
+                      "| Part No | Description | Pkt | Ctn | Price (AED) | Photo |",
+                      "|---|---|---|---|---|---|"]
+        price = "" if product["price"] is None else f"{product['price']:,.2f}"
+        pkt = "" if product["pktQty"] is None else f"{product['pktQty']:g}"
+        ctn = "" if product["ctnQty"] is None else f"{product['ctnQty']:g}"
+        photo = (product.get("imageSha256") or "")[:8]
+        lines.append(f"| {product['partNo']} | {product['description']} | "
+                     f"{pkt} | {ctn} | {price} | {photo} |")
+    return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("workbook")
+    parser.add_argument("--out-dir", default=".")
+    parser.add_argument("--env")
+    parser.add_argument("--sheet", help="worksheet name (default: the first)")
+    parser.add_argument("--refresh", action="store_true",
+                        help="force a new Datalab conversion")
+    parser.add_argument("--no-datalab", action="store_true",
+                        help="skip Datalab; photo spans and the cross-check are lost")
+    args = parser.parse_args()
+
+    path = Path(args.workbook).expanduser()
+    if not path.is_file():
+        raise SystemExit(f"no such file: {path}")
+    out_dir = Path(args.out_dir).expanduser()
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    import openpyxl
+    book = openpyxl.load_workbook(path)
+    sheet = book[args.sheet] if args.sheet else book.worksheets[0]
+
+    header_row = find_header(sheet)
+    if header_row is None:
+        raise SystemExit("no PART NO header found in the first 60 rows — "
+                         "is this the right sheet? Use --sheet to choose another.")
+
+    # ---- Datalab: photo spans and an independent opinion on where they sit
+    spans: dict[int, list] = {}
+    cross_check = {"ran": False}
+    table_range = None
+    if not args.no_datalab:
+        values, env_path = load_env(args.env)
+        key = credential(values, "DATALAB_API_KEY")
+        if not key:
+            raise SystemExit(
+                "DATALAB_API_KEY not found. Set it, point --env at a file that "
+                "has it, or pass --no-datalab to run on openpyxl alone.")
+        print(f"credentials: {env_path}")
+        response = convert(path, key, out_dir / "datalab_raw", refresh=args.refresh)
+        pictures = picture_rows(response)
+        for picture in pictures:
+            spans.setdefault(picture["row_from"], []).append(picture["row_to"])
+        for row in spans:
+            spans[row].sort()
+        tables = table_blocks(response)
+        if tables:
+            table_range = excel_range(tables[0]["bbox"])
+        cross_check = {
+            "ran": True,
+            "datalab_pictures": len(pictures),
+            "openpyxl_placements": len(sheet._images),
+            "table_range": table_range,
+            "table_blocks": len(tables),
+        }
+
+    levels = category_rows(sheet)
+    products, held_back = read_products(sheet, header_row, levels)
+    photos = placements(sheet, spans)
+    _, unmatched = assign_images(photos, products, levels)
+
+    # The completeness check: two readers, two code paths, one answer. Rows
+    # where they disagree are where a photo is at risk of landing on the wrong
+    # product, and no other signal in the output would show it.
+    if cross_check["ran"]:
+        datalab_anchor_rows = sorted(r for r, v in spans.items() for _ in v)
+        openpyxl_anchor_rows = sorted(i.anchor._from.row + 1 for i in sheet._images)
+        cross_check["anchor_rows_agree"] = datalab_anchor_rows == openpyxl_anchor_rows
+        cross_check["rows_only_in_datalab"] = sorted(
+            set(datalab_anchor_rows) - set(openpyxl_anchor_rows))
+        cross_check["rows_only_in_openpyxl"] = sorted(
+            set(openpyxl_anchor_rows) - set(datalab_anchor_rows))
+
+    media = media_index(path)
+    unsupported = sorted({Path(name).suffix.lstrip(".").lower() for name in media
+                          if Path(name).suffix.lower() in (".emf", ".wmf")})
+
+    duplicate_parts = {}
+    for product in products:
+        duplicate_parts.setdefault(product["partNo"], []).append(product["sheetRow"])
+    duplicates = {k: v for k, v in duplicate_parts.items() if len(v) > 1}
+
+    distinct_images = {p["imageSha256"] for p in products if p.get("imageSha256")}
+    payload = {
+        "source": {"file": path.name, "path": str(path.resolve()),
+                   "sheet": sheet.title, "headerRow": header_row,
+                   "tableRange": table_range,
+                   "rows": sheet.max_row, "columns": sheet.max_column},
+        "summary": {
+            "products": len(products),
+            "categories": sum(1 for v in levels.values() if v == "category"),
+            "subCategories": sum(1 for v in levels.values() if v == "subcategory"),
+            "image_placements": len(photos),
+            "products_with_image": sum(1 for p in products if p.get("imageSha256")),
+            "products_without_image": sum(1 for p in products
+                                          if not p.get("imageSha256")),
+            "distinct_images": len(distinct_images),
+            "media_files_in_workbook": len(media),
+            "images_matched_by_span": sum(1 for p in products
+                                          if p.get("imageMatch") == "span"),
+            "images_matched_by_nearest": sum(1 for p in products
+                                             if p.get("imageMatch") == "nearest"),
+            # A photo whose span covers several product rows is attached to
+            # every one of them, so these two say how much of the coverage
+            # above comes from spans rather than from single rows.
+            "images_spanning_multiple_products": sum(
+                1 for ph in photos if len(ph.get("attachedRows") or []) > 1),
+            "products_sharing_a_spanned_image": sum(
+                1 for p in products if p.get("imageShared")),
+            # Spans that ran into the next group and were cut at its heading.
+            "spans_truncated_at_heading": sum(
+                1 for ph in photos if ph.get("truncatedAtHeading")),
+            # Placements every one of whose rows a nearer-anchored photo won.
+            "images_overridden": sum(
+                1 for ph in photos
+                if ph.get("covers") and not ph["attachedRows"]),
+            "images_unmatched": len(unmatched),
+            "unsupported_image_formats": unsupported,
+            "products_without_price": sum(1 for p in products
+                                          if p["price"] is None),
+            "rows_held_back": len(held_back),
+            "duplicate_part_numbers": duplicates,
+            "cross_check": cross_check,
+        },
+        "products": products,
+        "rowsHeldBack": held_back,
+    }
+
+    json_path = out_dir / "clarke-products.json"
+    json_path.write_text(json.dumps(payload, indent=2))
+    md_path = out_dir / "clarke-products.md"
+    md_path.write_text(render_markdown(payload))
+
+    # Photo bytes go beside the JSON so the upload stage does not reopen the
+    # workbook — and so a human can look at them.
+    image_dir = out_dir / "images"
+    image_dir.mkdir(exist_ok=True)
+    written = set()
+    for photo in photos:
+        if "error" in photo or photo["sha256"] in written:
+            continue
+        extension = photo["format"] if photo["format"] != "unknown" else "bin"
+        (image_dir / f"{photo['sha256']}.{extension}").write_bytes(photo["data"])
+        written.add(photo["sha256"])
+
+    summary = payload["summary"]
+    print(f"{summary['products']:,} products, "
+          f"{summary['categories']} categories, "
+          f"{summary['subCategories']} sub-categories")
+    print(f"{summary['products_with_image']:,} products carry a photo "
+          f"({summary['images_matched_by_span']} by span, "
+          f"{summary['images_matched_by_nearest']} by nearest); "
+          f"{summary['images_unmatched']} placements unattached")
+    print(f"{summary['images_spanning_multiple_products']} photos span more "
+          f"than one product row and are attached to every product in the "
+          f"span ({summary['products_sharing_a_spanned_image']:,} products "
+          f"share a photo that way)")
+    if cross_check.get("ran"):
+        verdict = "agree" if cross_check["anchor_rows_agree"] else "DISAGREE"
+        print(f"cross-check: Datalab and openpyxl {verdict} on all "
+              f"{cross_check['datalab_pictures']} photo anchor rows")
+    if unsupported:
+        print(f"note: workbook holds {', '.join(unsupported)} media, "
+              f"which browsers cannot display")
+    print(f"wrote {json_path}")
+    print(f"wrote {md_path}")
+    print(f"wrote {len(written)} image files to {image_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())