@topy-ai/maggie 0.7.20 → 0.7.22

package/bundled-tools/clis/maggie_service_booking.py

@@ -17,6 +17,8 @@ from booking_capabilities import audit_matrix  # noqa: E402
 NOW = lambda: datetime.now(timezone.utc).isoformat()
 MONEY = re.compile(r"(?:£|GBP\s*)\s*([0-9]+(?:[.,][0-9]{1,2})?)", re.I)
 DURATION = re.compile(r"(\d{2,3})\s*(?:min|mins|minutes?)", re.I)
+SUPPLY_STATES = {"live", "withdrawn"}
+DISPLAY_STATES = {"published", "hidden", "retired"}
 
 def slug(value):
     value = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
@@ -57,33 +59,37 @@ def flatten(value):
 
 def parse(source, provider):
     raw=fetch(source); parser=Extractor(); parser.feed(raw); records=[]
+    embedded_marker = provider == "fresha" and re.search(r'"services"\s*:\s*\[', raw)
     if provider == "fresha":
         records.extend(parse_fresha_embedded(raw, source))
-    for obj in json_objects(parser.scripts):
-        for item in flatten(obj):
-            typ=item.get("@type", "")
-            types=set(typ) if isinstance(typ, list) else {typ}
-            if not types.intersection({"Service","Product","Offer","ListItem"}) and not any(k in item for k in ("duration","price","providerServiceId")): continue
-            name=item.get("name") or item.get("title") or (item.get("item") or {}).get("name") if isinstance(item.get("item"),dict) else item.get("name")
-            if not name or len(str(name)) < 3: continue
-            offers=item.get("offers") if isinstance(item.get("offers"),list) else [item.get("offers", item)]
-            variants=[]
-            for offer in offers:
-                if not isinstance(offer,dict): offer={}
-                text=" ".join(str(offer.get(k,"")) for k in ("name","description","duration"))
-                dm=DURATION.search(text); price=offer.get("price")
-                if price is None:
-                    match=MONEY.search(text); price=match.group(1).replace(",","") if match else None
-                currency=(offer.get("priceCurrency") or ("GBP" if price and ("£" in text or provider=="fresha") else None))
-                if price is None and dm is None: continue
-                amount=int(round(float(str(price).replace(",",""))*100)) if price is not None else None
-                provider_variant_id = offer.get("sku") or offer.get("@id") or offer.get("id")
-                variants.append({"id":str(provider_variant_id or f"{slug(name)}:{dm.group(1) if dm else 'default'}"),"idSource":"provider" if provider_variant_id else "derived","variantSource":"native" if provider_variant_id else "derived-offers","title":offer.get("name") or (f"{dm.group(1)} minutes" if dm else "Standard"),"durationMinutes":int(dm.group(1)) if dm else None,"price":{"amountMinor":amount,"currency":currency,"display":f"£{amount/100:.2f}" if amount is not None and currency=="GBP" else None}})
-            url=item.get("url") or item.get("sameAs")
-            if not variants and not url: continue
-            pid=str(item.get("providerServiceId") or item.get("productID") or item.get("sku") or slug(name))
-            records.append(make_record(provider, source, pid, str(name), str(item.get("description") or ""), variants, url, parser.links))
-    if not records:
+    # A provider-specific authoritative catalogue is the only service source.
+    # Generic JSON-LD may include marketplace entities for another business.
+    if not embedded_marker:
+        for obj in json_objects(parser.scripts):
+            for item in flatten(obj):
+                typ=item.get("@type", "")
+                types=set(typ) if isinstance(typ, list) else {typ}
+                if not types.intersection({"Service","Product","Offer","ListItem"}) and not any(k in item for k in ("providerServiceId","productID","sku")): continue
+                name=item.get("name") or item.get("title") or (item.get("item") or {}).get("name") if isinstance(item.get("item"),dict) else item.get("name")
+                if not name or len(str(name)) < 3: continue
+                offers=item.get("offers") if isinstance(item.get("offers"),list) else [item.get("offers", item)]
+                variants=[]
+                for offer in offers:
+                    if not isinstance(offer,dict): offer={}
+                    text=" ".join(str(offer.get(k,"")) for k in ("name","description","duration"))
+                    dm=DURATION.search(text); price=offer.get("price")
+                    if price is None:
+                        match=MONEY.search(text); price=match.group(1).replace(",","") if match else None
+                    currency=(offer.get("priceCurrency") or ("GBP" if price and ("£" in text or provider=="fresha") else None))
+                    if price is None and dm is None: continue
+                    amount=int(round(float(str(price).replace(",",""))*100)) if price is not None else None
+                    provider_variant_id = offer.get("sku") or offer.get("@id") or offer.get("id")
+                    variants.append({"id":str(provider_variant_id or f"{slug(name)}:{dm.group(1) if dm else 'default'}"),"idSource":"provider" if provider_variant_id else "derived","variantSource":"native" if provider_variant_id else "derived-offers","title":offer.get("name") or (f"{dm.group(1)} minutes" if dm else "Standard"),"durationMinutes":int(dm.group(1)) if dm else None,"price":{"amountMinor":amount,"currency":currency,"display":f"£{amount/100:.2f}" if amount is not None and currency=="GBP" else None}})
+                url=item.get("url") or item.get("sameAs")
+                if not variants and not url: continue
+                pid=str(item.get("providerServiceId") or item.get("productID") or item.get("sku") or slug(name))
+                records.append(make_record(provider, source, pid, str(name), str(item.get("description") or ""), variants, url, parser.links))
+    if not records and not embedded_marker:
         for _, pieces in parser.headings:
             name=" ".join(pieces).strip()
             if len(name)>3: records.append(make_record(provider, source, slug(name), name, "", [], None, parser.links))
@@ -97,10 +103,10 @@ def parse(source, provider):
 def parse_fresha_embedded(raw, source):
     """Read the public service catalogue embedded in Fresha's Next data."""
     records=[]
-    marker='"services":['; start=raw.find(marker)
-    if start >= 0:
+    marker=re.search(r'"services"\s*:\s*\[', raw); start=marker.start() if marker else -1
+    if marker:
         try:
-            groups=json.JSONDecoder().raw_decode(raw[start+len('"services":'):])[0]
+            groups=json.JSONDecoder().raw_decode(raw[marker.end()-1:])[0]
         except (ValueError, TypeError): groups=[]
         for group in groups if isinstance(groups,list) else []:
             level1=str(group.get("name") or "Uncategorised").strip()
@@ -182,7 +188,7 @@ def make_record(provider, source, pid, name, description, variants, booking, lin
     if not booking:
         for href,_ in links:
             if any(x in href.lower() for x in ("book","appointment","checkout")): booking=urljoin(source,href); break
-    return {"id":f"{provider}:{pid}","provider":provider,"providerServiceId":pid,"slug":slug(name),"title":name,"description":description.strip(),"sourceUrl":source,"category":{"level1":"Uncategorised","level2":None},"variants":variants,"bookingUrl":booking,"paymentUrl":None,"status":"active","firstSeenAt":NOW(),"lastSeenAt":NOW()}
+    return {"id":f"{provider}:{pid}","provider":provider,"providerServiceId":pid,"slug":slug(name),"title":name,"description":description.strip(),"sourceUrl":source,"category":{"level1":"Uncategorised","level2":None},"variants":variants,"bookingUrl":booking,"paymentUrl":None,"status":"active","supplyState":"live","displayState":"published","firstSeenAt":NOW(),"lastSeenAt":NOW()}
 
 def root(args): return Path(args.project).resolve()
 def path(project): return project/".maggie"/"booking"/"services.json"
@@ -199,11 +205,15 @@ def validate(data):
             if not s.get(key): errors.append(f"{s.get('id','unknown')}: missing {key}")
         if s.get("id") in seen: errors.append(f"duplicate service id: {s['id']}")
         seen.add(s.get("id"));
+        supply=s.get("supplyState") or ("withdrawn" if s.get("status") in {"archived", "removed"} else "live")
+        display=s.get("displayState") or ("retired" if s.get("status") == "archived" else "published")
+        if supply not in SUPPLY_STATES: errors.append(f"{s.get('id')}: invalid supplyState")
+        if display not in DISPLAY_STATES: errors.append(f"{s.get('id')}: invalid displayState")
         if any(x.get("slug")==s.get("slug") for x in data.get("services",[]) if x is not s): errors.append(f"duplicate service slug: {s.get('slug')}")
         pages=s.get("pages",[])
         if not isinstance(pages,list): errors.append(f"{s.get('id')}: pages must be an array"); pages=[]
         if sum(p.get("role")=="canonical" for p in pages if isinstance(p,dict))>1: errors.append(f"{s.get('id')}: multiple canonical pages")
-        if s.get("status")=="active" and not s.get("variants"): errors.append(f"{s.get('id')}: no variants")
+        if supply == "live" and not s.get("variants"): errors.append(f"{s.get('id')}: no variants")
         for v in s.get("variants",[]):
             if not v.get("durationMinutes") or not v.get("price",{}).get("currency"): errors.append(f"{s.get('id')}: incomplete variant {v.get('id')}")
         for key in ("bookingUrl","paymentUrl"):
@@ -312,33 +322,172 @@ def cmd_fact_audit(args):
     print(json.dumps({"status": "passed" if report["passed"] else "failed", "report": str(out), "reviewQueue": str(queue_path), "services": len(services), "variants": len(seen_variants), "sourceUrlsBackfilled": changed, "errors": errors[:20], "errorCount": len(errors), "publish": False}, indent=2, ensure_ascii=False))
     return 0 if report["passed"] else 1
 
+def cmd_catalogue_check(args):
+    """Answer treatment-presence questions using the import/sync parser."""
+    try:
+        data = parse(args.source, args.provider)
+    except (OSError, ValueError, UnicodeError):
+        print(json.dumps({"status": "failed", "error": "source could not be parsed"}, indent=2))
+        return 1
+    services = data.get("services", [])
+    results = []
+    for query in args.treatment:
+        query_tokens = page_tokens(query)
+        matches = [
+            {"id": service.get("id"), "title": service.get("title"), "category": service.get("category", {}).get("level1")}
+            for service in services
+            if query_tokens and query_tokens <= page_tokens(service.get("title", ""))
+        ]
+        results.append({"treatment": query, "status": "found" if matches else "not-found", "matches": matches})
+    passed = all(item["status"] == "found" for item in results)
+    print(json.dumps({"status": "passed" if passed else "failed", "provider": args.provider, "serviceCount": len(services), "results": results}, indent=2, ensure_ascii=False))
+    return 0 if passed else 1
+
+RETIREMENT_SCHEMA = "maggie-service-retirement-evidence.v1"
+RETIREMENT_ENDINGS = {"pending", "redirect", "tombstone", "gone"}
+
+def cmd_retirement_audit(args):
+    """Validate sanitized runtime evidence for withdrawn service URLs."""
+    project=root(args); catalogue_path=Path(args.catalogue); catalogue_path=catalogue_path if catalogue_path.is_absolute() else project/catalogue_path
+    evidence_path=Path(args.evidence); evidence_path=evidence_path if evidence_path.is_absolute() else project/evidence_path
+    try:
+        catalogue=json.loads(catalogue_path.read_text(encoding="utf-8")); evidence=json.loads(evidence_path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        print(json.dumps({"status":"failed","error":"catalogue or retirement evidence is invalid"},indent=2)); return 1
+    services={str(item.get("id")):item for item in catalogue.get("services",[]) if isinstance(item,dict) and item.get("id")}
+    rows=evidence.get("services") if isinstance(evidence,dict) else None
+    errors=[]; results=[]
+    if not isinstance(evidence,dict) or evidence.get("schemaVersion") != RETIREMENT_SCHEMA:
+        errors.append(f"evidence schemaVersion must be {RETIREMENT_SCHEMA}")
+    if not isinstance(rows,list):
+        errors.append("evidence.services must be a non-empty array")
+        rows=[]
+    if isinstance(rows,list) and not rows: errors.append("evidence.services must be a non-empty array")
+    seen=set()
+    for index,row in enumerate(rows):
+        item_errors=[]
+        if not isinstance(row,dict):
+            results.append({"index":index,"passed":False,"errors":["evidence row must be an object"]}); continue
+        service_id=str(row.get("serviceId") or "")
+        if not service_id or service_id in seen: item_errors.append("serviceId is missing or duplicated")
+        seen.add(service_id)
+        service=services.get(service_id)
+        if not service: item_errors.append("service is not in the catalogue")
+        supply=(service or {}).get("supplyState") or ("withdrawn" if (service or {}).get("status") in {"archived","removed"} else "live")
+        if supply != "withdrawn": item_errors.append("retirement evidence requires a withdrawn service")
+        ending=str(row.get("ending") or "")
+        if ending not in RETIREMENT_ENDINGS: item_errors.append("ending must be pending, redirect, tombstone, or gone")
+        expected_ending=((service or {}).get("retirement") or {}).get("ending")
+        if expected_ending and expected_ending != ending: item_errors.append("evidence ending does not match the catalogue decision")
+        status=row.get("httpStatus")
+        if not isinstance(status,int) or isinstance(status,bool): item_errors.append("httpStatus must be an integer")
+        if row.get("inSitemap") is not False: item_errors.append("withdrawn service must be absent from the sitemap")
+        if row.get("bookingSuppressed") is not True: item_errors.append("bookingSuppressed must be true")
+        if ending in {"pending","tombstone"}:
+            if status != 200: item_errors.append(f"{ending} must return HTTP 200")
+            if row.get("unavailableNotice") is not True: item_errors.append("unavailableNotice must be true")
+            if row.get("noindex") is not True: item_errors.append("noindex must be true")
+        elif ending == "redirect":
+            if status not in {301,308}: item_errors.append("redirect must return HTTP 301 or 308")
+            location=str(row.get("location") or "")
+            route=str(row.get("route") or "")
+            if not location or not location.startswith("/") or location == route: item_errors.append("redirect needs a different same-site location")
+        elif ending == "gone" and status != 410:
+            item_errors.append("gone must return HTTP 410")
+        results.append({"serviceId":service_id,"ending":ending,"passed":not item_errors,"errors":item_errors})
+    passed=not errors and bool(results) and all(item["passed"] for item in results)
+    report={"schemaVersion":"maggie-service-retirement-audit.v1","evidenceSchema":RETIREMENT_SCHEMA,"checked":len(results),"services":results,"errors":errors,"passed":passed}
+    output=Path(args.output).expanduser(); output=output if output.is_absolute() else project/output; output.parent.mkdir(parents=True,exist_ok=True); output.write_text(json.dumps(report,indent=2,ensure_ascii=False)+"\n",encoding="utf-8")
+    print(json.dumps({"status":"passed" if passed else "failed","report":str(output.resolve()),"checked":len(results),"errors":errors},indent=2)); return 0 if passed else 1
+
 def cmd_import(args):
     project=root(args); data=parse(args.source,args.provider); old=load(project) if path(project).exists() else {"services":[]}; previous={s["id"]:s for s in old.get("services",[])}
     for service in data.get("services",[]):
         if service["id"] in previous:
-            service["pages"]=previous[service["id"]].get("pages",[]); service["additional"]={**previous[service["id"]].get("additional",{}),**service.get("additional",{})}
+            old_service=previous[service["id"]]
+            preserve_site_fields(service, old_service)
+            service["displayState"]=old_service.get("displayState") or ("retired" if old_service.get("status") == "archived" else "published")
+        service.setdefault("supplyState", "live"); service.setdefault("displayState", "published")
     errors=validate(data)
     if errors: print(json.dumps({"status":"failed","errors":errors},indent=2)); return 1
     save(project,data); print(json.dumps({"status":"imported","path":str(path(project)),"reviewPath":str(project/"docs"/"services.json"),"services":len(data["services"])},indent=2)); return 0
+
+SERVICE_DIFF_FIELDS = ("provider", "providerServiceId", "slug", "title", "description", "category", "bookingUrl", "paymentUrl", "status", "supplyState", "displayState", "retirement")
+
+def preserve_site_fields(service, old_service):
+    proposal=None
+    old_slug=str(old_service.get("slug") or "")
+    new_slug=str(service.get("slug") or "")
+    if old_slug and new_slug and old_slug != new_slug:
+        proposal={"status":"pending","from":old_slug,"to":new_slug,"reason":"provider name implies a different public slug"}
+        service["slug"]=old_slug
+    service["pages"] = old_service.get("pages", [])
+    service["additional"] = {**old_service.get("additional", {}), **service.get("additional", {})}
+    return proposal
+
+def service_diff(before, after):
+    fields=[]
+    for field in SERVICE_DIFF_FIELDS:
+        old_value = before.get(field) if isinstance(before, dict) else None
+        new_value = after.get(field) if isinstance(after, dict) else None
+        if old_value != new_value:
+            fields.append({"field": field, "before": old_value, "after": new_value})
+    old_variants={str(item.get("id")): item for item in (before or {}).get("variants", []) if isinstance(item, dict) and item.get("id")}
+    new_variants={str(item.get("id")): item for item in (after or {}).get("variants", []) if isinstance(item, dict) and item.get("id")}
+    variants=[]
+    for variant_id in sorted(set(old_variants) | set(new_variants)):
+        old_variant=old_variants.get(variant_id); new_variant=new_variants.get(variant_id)
+        if old_variant is None:
+            variants.append({"id": variant_id, "change": "added", "before": None, "after": new_variant})
+        elif new_variant is None:
+            variants.append({"id": variant_id, "change": "removed", "before": old_variant, "after": None})
+        elif old_variant != new_variant:
+            variants.append({"id": variant_id, "change": "updated", "before": old_variant, "after": new_variant})
+    return {"fields": fields, "variantChanges": variants}
+
 def cmd_sync(args):
     project=root(args); old=load(project) if path(project).exists() else {"services":[]}; fresh=parse(args.source,args.provider); previous={s["id"]:s for s in old.get("services",[])}; current={s["id"]:s for s in fresh.get("services",[])}; changes=[]
-    def comparable(service):
-        value=dict(service)
-        value.pop("firstSeenAt", None)
-        value.pop("lastSeenAt", None)
-        return value
     for sid, service in current.items():
         old_service = previous.get(sid)
+        slug_proposal=None
         if old_service:
-            service["pages"] = old_service.get("pages", [])
-            service["additional"] = {**old_service.get("additional", {}), **service.get("additional", {})}
-        changes.append({"id":sid,"change":"added" if sid not in previous else ("updated" if json.dumps(comparable(service),sort_keys=True,ensure_ascii=False)!=json.dumps(comparable(old_service),sort_keys=True,ensure_ascii=False) else "unchanged")})
+            slug_proposal=preserve_site_fields(service, old_service)
+            # Provider sync owns supply; a person's display decision survives
+            # the run and is never reset by a newly fetched row.
+            service["displayState"] = old_service.get("displayState") or ("retired" if old_service.get("status") == "archived" else "published")
+        service["supplyState"] = "live"
+        diff=service_diff(old_service, service)
+        change="added" if sid not in previous else ("updated" if diff["fields"] or diff["variantChanges"] else "unchanged")
+        if slug_proposal:
+            diff["slugProposal"]=slug_proposal
+            diff["reviewRequired"]=True
+            change="updated"
+        changes.append({"id":sid,"change":change,**diff})
     for sid, service in previous.items():
-        if sid not in current: archived={**service,"status":"archived","removedAt":NOW()}; current[sid]=archived; changes.append({"id":sid,"change":"removed"})
+        if sid not in current:
+            archived={**service,"status":"archived","supplyState":"withdrawn","displayState":service.get("displayState") or ("retired" if service.get("status") == "archived" else "published"),"retirement":service.get("retirement") or {"ending":"pending"},"removedAt":NOW()}; current[sid]=archived
+            changes.append({"id":sid,"change":"removed","fields":[{"field":"status","before":service.get("status"),"after":"archived"}],"variantChanges":[]})
     fresh["services"]=list(current.values()); save(project,fresh)
-    run_id=datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
-    report={"schemaVersion":"1.0","runId":run_id,"syncedAt":fresh["syncedAt"],"sourceUrl":args.source,"provider":args.provider,"changes":changes,"summary":{"total":len(changes),"added":sum(c["change"]=="added" for c in changes),"updated":sum(c["change"]=="updated" for c in changes),"removed":sum(c["change"]=="removed" for c in changes),"unchanged":sum(c["change"]=="unchanged" for c in changes)},"checkpoint":{"phase":"reconciled","processed":len(changes),"total":len(changes),"resumable":True,"status":"complete"}}
+    run_id=datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
+    report={"schemaVersion":"maggie-service-sync-report.v1","runId":run_id,"syncedAt":fresh["syncedAt"],"sourceUrl":args.source,"provider":args.provider,"changes":changes,"summary":{"total":len(changes),"added":sum(c["change"]=="added" for c in changes),"updated":sum(c["change"]=="updated" for c in changes),"removed":sum(c["change"]=="removed" for c in changes),"unchanged":sum(c["change"]=="unchanged" for c in changes)},"checkpoint":{"phase":"reconciled","processed":len(changes),"total":len(changes),"resumable":True,"status":"complete"}}
     sync_dir=path(project).parent/"sync-runs"; sync_dir.mkdir(parents=True,exist_ok=True); (sync_dir/f"{run_id}.json").write_text(json.dumps(report,indent=2,ensure_ascii=False)+"\n",encoding="utf-8"); (path(project).parent/"sync-checkpoint.json").write_text(json.dumps(report["checkpoint"]|{"runId":run_id,"sourceUrl":args.source},indent=2,ensure_ascii=False)+"\n",encoding="utf-8"); (path(project).parent/"last-sync.json").write_text(json.dumps(report,indent=2,ensure_ascii=False)+"\n",encoding="utf-8"); print(json.dumps(report,indent=2,ensure_ascii=False)); return 0 if not validate(fresh) else 1
+
+def cmd_sync_report(args):
+    project=root(args); directory=path(project).parent/"sync-runs"
+    candidates=sorted(directory.glob("*.json")) if directory.exists() else []
+    if args.run_id:
+        candidates=[directory/f"{args.run_id}.json"]
+    if not candidates or not candidates[-1].exists():
+        print(json.dumps({"status":"failed","error":"sync report not found"},indent=2)); return 1
+    try:
+        report=json.loads(candidates[-1].read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        print(json.dumps({"status":"failed","error":"sync report is invalid"},indent=2)); return 1
+    if report.get("schemaVersion") != "maggie-service-sync-report.v1":
+        print(json.dumps({"status":"failed","error":"unsupported sync report schema"},indent=2)); return 1
+    if args.output:
+        output=Path(args.output).expanduser(); output=output if output.is_absolute() else project/output; output.parent.mkdir(parents=True,exist_ok=True); output.write_text(json.dumps(report,indent=2,ensure_ascii=False)+"\n",encoding="utf-8")
+    print(json.dumps(report,indent=2,ensure_ascii=False)); return 0
 def cmd_generate(args):
     print(json.dumps({"status":"blocked","error":"Public copy generation is agent-owned","required":"The AI agent must create and review the copy artifact, then the project renderer must consume it. This deterministic CLI never synthesizes public hero, section, CTA, title, or FAQ copy.","copyData":getattr(args, "copy_data", None)},indent=2)); return 1
     project=root(args); data=load(project); errors=validate(data)
@@ -1004,6 +1153,7 @@ def main():
     for name in ("import","sync","run"):
         q=sub.add_parser(name); q.add_argument("source"); q.add_argument("--project",default="."); q.add_argument("--provider",default="fresha",choices=["fresha"])
         if name == "run": q.add_argument("--copy-data", help="AI-authored service copy artifact; required before public page generation")
+    q=sub.add_parser("sync-report"); q.add_argument("--project",default="."); q.add_argument("--run-id"); q.add_argument("--output")
     q=sub.add_parser("convert-page"); q.add_argument("page"); q.add_argument("--project",default=".")
     for name in ("polish","validate-polish"):
         q=sub.add_parser(name); q.add_argument("--service-id",required=True); q.add_argument("--page",required=True); q.add_argument("--project",default="."); q.add_argument("--rendered",help="rendered HTML file or URL for runtime evidence"); q.add_argument("--copy-data",required=True,help="AI-authored copy JSON with provenance and completed review statuses")
@@ -1013,6 +1163,8 @@ def main():
     q=sub.add_parser("category-editorial-review"); q.add_argument("--project",default="."); q.add_argument("--copy-data",default="docs/category-page-copy.json"); q.add_argument("--category",action="append",help="category to review; defaults to all seven launch categories"); q.add_argument("--reviewer"); q.add_argument("--evidence",action="append",default=[],help="review evidence path or URL; repeatable"); q.add_argument("--apply",action="store_true",help="write explicit editorial approval metadata"); q.add_argument("--confirm",action="store_true",help="confirm that the selected records were actually reviewed by a human")
     q=sub.add_parser("category-hash"); q.add_argument("--project",default="."); q.add_argument("--copy-data",required=True); q.add_argument("--apply",action="store_true",help="backfill deterministic provenance hashes in this generated artifact")
     q=sub.add_parser("fact-audit"); q.add_argument("--project",default="."); q.add_argument("--backfill-source",action="store_true",help="copy the catalogue sourceUrl into records that have no sourceUrl"); q.add_argument("--overrides",help="JSON of human-approved, evidenced descriptions for provider gaps"); q.add_argument("--apply-overrides",action="store_true",help="apply only approved fact overrides to the draft catalogue")
+    q=sub.add_parser("catalogue-check"); q.add_argument("source"); q.add_argument("--project",default="."); q.add_argument("--provider",default="fresha",choices=["fresha"]); q.add_argument("--treatment",action="append",required=True,help="provider treatment name; repeat for multiple checks")
+    q=sub.add_parser("retirement-audit"); q.add_argument("--project",default="."); q.add_argument("--catalogue",default=".maggie/booking/services.json"); q.add_argument("--evidence",required=True,help="sanitized runtime retirement evidence JSON"); q.add_argument("--output",default="docs/service-retirement-audit.json")
     q=sub.add_parser("capability-audit", help="validate provider variant declarations and fixture evidence"); q.add_argument("--project",default="."); q.add_argument("--provider"); q.add_argument("--catalogue",default=".maggie/booking/services.json"); q.add_argument("--capabilities-file",default=".maggie/booking/provider-capabilities.json"); q.add_argument("--fixture",help="sanitized provider fixture JSON"); q.add_argument("--output")
     q=sub.add_parser("category-context"); q.add_argument("--project",default="."); q.add_argument("--output")
     q=sub.add_parser("validate-copy"); q.add_argument("--project",default="."); q.add_argument("--service-id",required=True); q.add_argument("--copy-data",required=True,help="AI-authored service copy JSON")
@@ -1026,6 +1178,7 @@ def main():
     a=p.parse_args(); project=root(a)
     if a.command=="import": return cmd_import(a)
     if a.command=="sync": return cmd_sync(a)
+    if a.command=="sync-report": return cmd_sync_report(a)
     if a.command=="run": return cmd_run(a)
     if a.command=="convert-page": return cmd_convert_page(a)
     if a.command in {"polish","validate-polish"}: return cmd_polish(a)
@@ -1035,6 +1188,8 @@ def main():
     if a.command=="category-editorial-review": return cmd_category_editorial_review(a)
     if a.command=="category-hash": return cmd_category_hash(a)
     if a.command=="fact-audit": return cmd_fact_audit(a)
+    if a.command=="catalogue-check": return cmd_catalogue_check(a)
+    if a.command=="retirement-audit": return cmd_retirement_audit(a)
     if a.command=="capability-audit": return cmd_capability_audit(a)
     if a.command=="category-context": return cmd_category_context(a)
     if a.command=="validate-copy": return cmd_validate_copy(a)

package/bundled-tools/runtime/content_localization.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections import Counter
 import re
 from typing import Any, Iterable
 
@@ -17,6 +18,10 @@ MARKETS = {"global", "uk", "us"}
 STATUSES = {"missing", "draft", "machine_translated", "needs_review", "approved", "published", "archived"}
 OPERATIONS = {"translate", "polish", "rewrite", "localise", "rebrand", "manual_edit"}
 PROTECTED_FIELDS = {"price", "currency", "rating", "provider", "providerId", "bookingUrl", "paymentUrl", "legalClaims", "healthClaims", "contentId", "slug", "canonicalUrl", "translationGroupId"}
+QUALITY_OPERATIONS = {"polish", "rewrite"}
+QUALITY_META_FIELDS = {"identity", "contentId", "contentType", "sourceRevision", "translationStatus", "isIndexable", "provenance", "sourceStringIds", "qualityEvidence", "humanReviewStatus"}
+FACT_TOKEN_PATTERN = re.compile(r"https?://[^\s<>]+|[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}|(?:£|\$|€|¥|₹)\s*\d[\d,.]*|\b\d[\d,.]*%|\b\d[\d,.]*\b", re.UNICODE)
+PLACEHOLDER_PATTERN = re.compile(r"\{\{[^}]+\}\}|\$\{[^}]+\}|\b(?:TODO|TBD|lorem ipsum)\b", re.IGNORECASE)
 
 
 class TranslationIndex:
@@ -162,3 +167,135 @@ def protected_field_changes(source: Any, translation: Any) -> list[str]:
     if not isinstance(source, dict) or not isinstance(translation, dict):
         return []
     return [field for field in sorted(PROTECTED_FIELDS) if field in source and field in translation and source[field] != translation[field]]
+
+
+def _copy_payload(value: Any) -> Any:
+    """Remove identity/approval metadata before comparing editorial structure."""
+    if isinstance(value, dict):
+        return {
+            key: _copy_payload(item)
+            for key, item in value.items()
+            if key not in QUALITY_META_FIELDS and key not in PROTECTED_FIELDS
+        }
+    if isinstance(value, list):
+        return [_copy_payload(item) for item in value]
+    return value
+
+
+def _copy_text(value: Any) -> str:
+    if isinstance(value, dict):
+        return "\n".join(part for part in (_copy_text(item) for item in value.values()) if part)
+    if isinstance(value, list):
+        return "\n".join(part for part in (_copy_text(item) for item in value) if part)
+    return str(value).strip() if isinstance(value, str) else ""
+
+
+def _copy_metrics(text: str) -> dict[str, int | bool]:
+    normalized = text.strip()
+    sentences = [item for item in re.split(r"[.!?。！？]+", normalized) if item.strip()]
+    words = re.findall(r"\w+(?:['’\-]\w+)*", normalized, re.UNICODE)
+    sentence_lengths = [len(re.findall(r"\w+", item, re.UNICODE)) for item in sentences]
+    return {
+        "characters": len(normalized),
+        "words": len(words),
+        "sentences": len(sentences),
+        "paragraphs": len([item for item in re.split(r"\n\s*\n", normalized) if item.strip()]),
+        "headings": len(re.findall(r"(?im)^\s{0,3}(?:#{1,6}\s+|<h[1-6]\b)", normalized)),
+        "listItems": len(re.findall(r"(?im)^\s*(?:[-*+]\s+|\d+[.)]\s+|<li\b)", normalized)),
+        "maxSentenceWords": max(sentence_lengths, default=0),
+        "placeholders": len(PLACEHOLDER_PATTERN.findall(normalized)),
+        "repeatedWhitespace": len(re.findall(r"[ \t]{2,}", normalized)),
+    }
+
+
+def _fact_tokens(text: str) -> Counter[str]:
+    return Counter(token.casefold().rstrip(".,;:)") for token in FACT_TOKEN_PATTERN.findall(text))
+
+
+def _structure_signature(value: Any) -> tuple[Any, ...]:
+    if isinstance(value, dict):
+        return ("object", tuple((key, _structure_signature(item)) for key, item in sorted(value.items())))
+    if isinstance(value, list):
+        return ("list", len(value), tuple(_structure_signature(item) for item in value))
+    if isinstance(value, str):
+        metrics = _copy_metrics(value)
+        return ("text", metrics["paragraphs"], metrics["headings"], metrics["listItems"], metrics["sentences"])
+    if value is None:
+        return ("null",)
+    return (type(value).__name__,)
+
+
+def validate_operation_quality(source: Any, translation: Any, operation: str, review: Any = None) -> tuple[list[str], dict[str, Any]]:
+    """Run conservative, dependency-free checks for polish and rewrite jobs.
+
+    These checks cover observable copy structure and protected fact tokens. They
+    intentionally do not claim semantic equivalence; every applicable result
+    carries a mandatory human-review signal.
+    """
+    report: dict[str, Any] = {
+        "operation": operation,
+        "applicable": operation in QUALITY_OPERATIONS,
+        "method": "deterministic-copy-heuristics",
+    }
+    if operation not in QUALITY_OPERATIONS:
+        return [], report
+    source_copy = _copy_payload(source)
+    translation_copy = _copy_payload(translation)
+    source_text = _copy_text(source_copy)
+    translation_text = _copy_text(translation_copy)
+    source_metrics = _copy_metrics(source_text)
+    translation_metrics = _copy_metrics(translation_text)
+    source_facts = _fact_tokens(source_text)
+    translation_facts = _fact_tokens(translation_text)
+    changed_fact_count = sum((source_facts - translation_facts).values()) + sum((translation_facts - source_facts).values())
+    normalized_source = re.sub(r"\s+", " ", source_text).strip().casefold()
+    normalized_translation = re.sub(r"\s+", " ", translation_text).strip().casefold()
+    structure_changed = _structure_signature(source_copy) != _structure_signature(translation_copy)
+    clarity_signal = (
+        translation_metrics["maxSentenceWords"] < source_metrics["maxSentenceWords"]
+        or translation_metrics["sentences"] > source_metrics["sentences"]
+        or translation_metrics["repeatedWhitespace"] < source_metrics["repeatedWhitespace"]
+        or translation_metrics["placeholders"] < source_metrics["placeholders"]
+    )
+    declared_clarity = isinstance(translation, dict) and isinstance(translation.get("qualityEvidence"), dict) and translation["qualityEvidence"].get("clarityImproved") is True
+    human_review = review if isinstance(review, dict) else {}
+    review_approved = human_review.get("decision") == "approve" and bool(human_review.get("reviewer"))
+    errors: list[str] = []
+    if not source_text:
+        errors.append(f"{operation} quality requires source copy text")
+    if not translation_text:
+        errors.append(f"{operation} quality requires generated copy text")
+    if changed_fact_count:
+        errors.append(f"{operation} output changed {changed_fact_count} protected fact token(s)")
+    if operation == "polish":
+        if normalized_source and normalized_source == normalized_translation:
+            errors.append("polish output is unchanged")
+        if translation_metrics["placeholders"] > source_metrics["placeholders"]:
+            errors.append("polish output introduced unresolved placeholders")
+        if source_text and translation_text and not (clarity_signal or declared_clarity):
+            errors.append("polish output has no deterministic clarity-improvement signal")
+    if operation == "rewrite":
+        if normalized_source and normalized_source == normalized_translation:
+            errors.append("rewrite output is unchanged")
+        if source_text and translation_text and not structure_changed:
+            errors.append("rewrite output did not change copy structure")
+    report.update({
+        "source": source_metrics,
+        "output": translation_metrics,
+        "checks": {
+            "nonEmpty": bool(source_text and translation_text),
+            "changed": bool(normalized_source != normalized_translation),
+            "protectedFactsPreserved": changed_fact_count == 0,
+            "structureChanged": structure_changed,
+            "clarityImproved": bool(clarity_signal or declared_clarity) if operation == "polish" else None,
+            "clarityEvidence": "deterministic" if clarity_signal else "declared" if declared_clarity else "missing",
+        },
+        "humanReview": {
+            "required": True,
+            "status": "approved" if review_approved else "required",
+            "semanticEquivalence": "not-certified",
+            "reason": "heuristics do not replace human meaning review",
+        },
+        "status": "passed" if not errors else "failed",
+    })
+    return errors, report

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@topy-ai/maggie",
-  "version": "0.7.20",
+  "version": "0.7.22",
   "description": "Install and manage Maggie Skills for AI coding agents",
   "license": "MIT",
   "type": "module",

package/references/content-localization-contract.md

@@ -61,6 +61,15 @@ Supported operations are distinct: `translate`, `polish`, `rewrite`,
 source. A source revision change marks dependent translations stale. Fallback
 content is never indexable.
 
+`polish` and `rewrite` validation includes deterministic output checks. Polish
+must keep the source language, change the copy, preserve protected fact tokens,
+and show a clarity signal (shorter maximum sentence, clearer sentence
+segmentation, placeholder reduction, or explicit quality evidence). Rewrite
+must change the copy and its observable structure, while preserving protected
+fact tokens. These are conservative heuristics, not semantic equivalence
+proof: every result contains `quality.humanReview.required: true` and marks
+semantic equivalence as `not-certified` until a named reviewer approves it.
+
 The following are protected by default: price, currency, rating, provider
 facts, booking URL, legal/health claims, content ID, slug, canonical owner,
 and translation group. Changes require structured approval and evidence.

package/references/universal-booking-adapter.md

@@ -57,7 +57,7 @@ The stable model is deliberately small and provider-neutral:
 - **category**: `level1` and `level2`; a service may belong to more than one
   provider category through `additional.provider.categories`;
 - **service**: identity, slug, title, description, active/archived status,
-  timestamps, and variants;
+  timestamps, supply/display state, and variants;
 - **variant**: identity, display name, duration, price, currency, discount or
   price range when the provider exposes them;
 - **booking actions**: booking URL, payment URL, and optional action metadata;
@@ -67,6 +67,44 @@ The stable model is deliberately small and provider-neutral:
 - **sync**: source, fetched time, content hash, parser version, and change
   status.
 
+The portable lifecycle model keeps two owners separate:
+
+- `supplyState`: provider sync state, either `live` or `withdrawn`;
+- `displayState`: human publication decision, either `published`, `hidden`, or
+  `retired`.
+
+Sync may change `supplyState` when the provider adds or removes a service, but
+must preserve `displayState`. A withdrawn/published record is an explicit
+interim state: keep the URL answerable, suppress booking and indexing, and
+open a human retirement decision. The legacy `status` field may remain for
+backwards compatibility, but it must not be used as both provider availability
+and display policy.
+
+The `slug` is site-owned. A provider rename may produce a `slugProposal` in a
+sync report, but the stored slug remains unchanged until a person accepts the
+proposal and the host writes the new slug plus its redirect in one transaction.
+
+## Withdrawal and retirement contract
+
+Provider removal is not permission to delete a public route. The adapter keeps
+the record with `supplyState: "withdrawn"` and the host chooses a reviewed
+`retirement.ending`:
+
+| Ending | Required host response | Required safety signals |
+|---|---|---|
+| `pending` | HTTP 200 interim page | unavailable notice, no booking action, `noindex`, absent from sitemap |
+| `redirect` | HTTP 301 or 308 to a different same-site route | no booking action, absent from sitemap |
+| `tombstone` | HTTP 200 unavailable page | unavailable notice, no booking action, `noindex`, absent from sitemap |
+| `gone` | HTTP 410 | no booking action, absent from sitemap |
+
+The host must preserve the old route long enough to apply its chosen ending,
+and must not return a generic 404 as a substitute for the reviewed contract.
+`maggie service retirement-audit` validates sanitized runtime evidence against
+the catalogue and records only service IDs, ending decisions, statuses, and
+safe pass/fail metadata. It does not fetch private routes, store response
+bodies, or invent redirect targets. A person must review redirect destination,
+copy, and accessibility before publication.
+
 The service page must render only canonical fields. `additional` is an
 explicit extension point for provider-specific or future fields and must be
 namespaced by concern (`provider`, `location`, `presentation`, `compliance`,
@@ -74,6 +112,21 @@ namespaced by concern (`provider`, `location`, `presentation`, `compliance`,
 nullable-safe, and must never override canonical fields. Unknown data stays in
 `additional`; it is not guessed into the stable model.
 
+## Source ownership and onboarding precedence
+
+When a host onboarding flow collects a value directly from the merchant, that
+explicit value is authoritative for the project. Provider research, imported
+profiles, search results, and inferred metadata may fill an empty field only;
+they must never replace a non-empty merchant entry. Persist the source of each
+value when the host supports it, and show a conflict for human review when two
+non-empty sources disagree. This rule applies to the website URL as well as
+business name, address, phone, category, and booking-provider identity.
+
+The shared service-booking parser does not implement a host's onboarding UI.
+Adapters must enforce this precedence before passing project identity into
+provider research or sync. A provider URL is evidence about the provider
+catalogue, not permission to overwrite the merchant's website URL.
+
 Pages can also enter the model without a provider. A normal page conversion
 creates a `draft` service with `additional.conversion.sourcePage` and no
 invented price, duration, or booking URL. It becomes `active` only after the
@@ -104,6 +157,14 @@ silent conversion, idempotent add/update/archive synchronisation, source
 snapshots/checksums, import-run audit records and generated service pages with
 provider booking CTAs.
 
+When a provider exposes an authoritative venue-owned catalogue, use that
+adapter extraction for sync decisions and treatment-presence checks. Generic
+JSON-LD, navigation links, and page-wide text can contain marketplace or other
+business entities and are not proof that the connected venue still sells a
+treatment. A read-only catalogue query should call the same parser as import
+and sync so an operator can review a reproducible answer before applying a
+withdrawal.
+
 It must not claim real-time availability, booking creation, cancellation sync
 or webhooks unless an official Fresha partner/API capability is available for
 that account. Public booking links remain the safe fallback. The paid Fresha