@topy-ai/maggie 0.7.12 → 0.7.14

package/bundled-tools/runtime/maggie_quality.py

@@ -0,0 +1,231 @@
+"""Provider-neutral quality contracts for variants, media, inventory, and bindings."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections import Counter, defaultdict
+from typing import Any, Iterable
+
+
+VARIANT_SCHEMA = "maggie-variant-copy.v1"
+MEDIA_SCHEMA = "maggie-media-uniqueness.v1"
+INVENTORY_SCHEMA = "maggie-page-inventory.v1"
+BINDING_SCHEMA = "maggie-section-bindings.v1"
+IDEMPOTENCY_SCHEMA = "maggie-reconcile-contract.v1"
+
+
+def _items(value: object, *keys: str) -> list[dict[str, Any]]:
+    if isinstance(value, list):
+        return [item for item in value if isinstance(item, dict)]
+    if isinstance(value, dict):
+        for key in keys:
+            candidate = value.get(key)
+            if isinstance(candidate, list):
+                return [item for item in candidate if isinstance(item, dict)]
+    return []
+
+
+def _normalise(value: object) -> object:
+    if isinstance(value, dict):
+        return {str(key): _normalise(value[key]) for key in sorted(value)}
+    if isinstance(value, list):
+        values = [_normalise(item) for item in value]
+        return sorted(values, key=lambda item: json.dumps(item, sort_keys=True, ensure_ascii=False))
+    if isinstance(value, str):
+        return re.sub(r"\s+", " ", value).strip().casefold()
+    return value
+
+
+def _field_value(page: dict[str, Any], field: str) -> object:
+    if field in page:
+        return page[field]
+    for section in page.get("sections", []) if isinstance(page.get("sections"), list) else []:
+        if isinstance(section, dict) and (section.get("type") == field or section.get("id") == field):
+            copy = {key: value for key, value in section.items() if key not in {"id", "type"}}
+            return copy
+    return None
+
+
+def validate_variant_copy(value: object, fields: Iterable[str] = ("faq",)) -> dict[str, Any]:
+    pages = _items(value, "pages", "variants")
+    errors: list[str] = []
+    duplicate_groups: list[dict[str, Any]] = []
+    grouped: dict[tuple[str, str, str], list[dict[str, Any]]] = defaultdict(list)
+    field_list = [str(field) for field in fields if str(field)]
+    for index, page in enumerate(pages):
+        page_id = str(page.get("id") or page.get("variantId") or f"item-{index}")
+        family = str(page.get("family") or page.get("variantFamily") or page.get("sourceVariantKey") or page.get("canonicalVariantId") or "")
+        locale = str(page.get("locale") or "default")
+        if not family:
+            errors.append(f"{page_id}: variant family is required")
+            continue
+        for field in field_list:
+            selected = _field_value(page, field)
+            if selected is not None:
+                digest = hashlib.sha256(json.dumps(_normalise(selected), sort_keys=True, ensure_ascii=False).encode()).hexdigest()
+                grouped[(family, locale, field)].append({"id": page_id, "digest": digest})
+    for (family, locale, field), records in sorted(grouped.items()):
+        by_digest: dict[str, list[str]] = defaultdict(list)
+        for record in records:
+            by_digest[record["digest"]].append(record["id"])
+        for digest, ids in sorted(by_digest.items()):
+            if len(ids) > 1:
+                duplicate_groups.append({"family": family, "locale": locale, "field": field, "variantIds": sorted(ids), "digest": digest})
+    return {"schemaVersion": VARIANT_SCHEMA, "passed": not errors and not duplicate_groups, "errors": errors, "duplicates": duplicate_groups, "pages": len(pages), "fields": field_list}
+
+
+def _media_sources(value: object, path: str = "") -> list[dict[str, str]]:
+    media_keys = {"image", "imageurl", "image_url", "src", "srcset", "poster", "media", "mediaurl", "media_url"}
+    result: list[dict[str, str]] = []
+    if isinstance(value, dict):
+        for key, child in value.items():
+            child_path = f"{path}.{key}" if path else str(key)
+            if str(key).casefold() in media_keys and isinstance(child, str) and child.strip():
+                result.append({"source": child.strip(), "path": child_path})
+            else:
+                result.extend(_media_sources(child, child_path))
+    elif isinstance(value, list):
+        for index, child in enumerate(value):
+            result.extend(_media_sources(child, f"{path}.{index}"))
+    return result
+
+
+def validate_media_uniqueness(value: object, *, across_siblings: bool = False) -> dict[str, Any]:
+    pages = _items(value, "pages", "variants")
+    errors: list[dict[str, Any]] = []
+    cross: dict[tuple[str, str], list[str]] = defaultdict(list)
+    for index, page in enumerate(pages):
+        page_id = str(page.get("id") or page.get("variantId") or f"item-{index}")
+        sources = _media_sources(page.get("sections", page))
+        by_source: dict[str, list[str]] = defaultdict(list)
+        for item in sources:
+            by_source[item["source"]].append(item["path"])
+        for source, paths in sorted(by_source.items()):
+            if len(paths) > 1:
+                errors.append({"scope": "page", "pageId": page_id, "source": source, "paths": paths})
+        if across_siblings:
+            family = str(page.get("family") or page.get("variantFamily") or page.get("sourceVariantKey") or page_id)
+            for source in by_source:
+                cross[(family, source)].append(page_id)
+    if across_siblings:
+        for (family, source), ids in sorted(cross.items()):
+            if len(set(ids)) > 1:
+                errors.append({"scope": "siblings", "family": family, "source": source, "pageIds": sorted(set(ids))})
+    return {"schemaVersion": MEDIA_SCHEMA, "passed": not errors, "duplicates": errors, "pages": len(pages), "acrossSiblings": across_siblings}
+
+
+def classify_inventory(value: object) -> dict[str, Any]:
+    pages = _items(value, "pages", "items", "routes")
+    allowed = {"band", "code-rendered", "redirect", "asset", "unknown"}
+    records: list[dict[str, Any]] = []
+    errors: list[str] = []
+    seen: set[str] = set()
+    for index, page in enumerate(pages):
+        path = str(page.get("path") or page.get("url") or page.get("route") or f"item-{index}")
+        if path in seen:
+            errors.append(f"duplicate inventory path: {path}")
+        seen.add(path)
+        explicit = str(page.get("kind") or page.get("classification") or "").casefold()
+        if explicit in {"code", "code-rendered", "component", "source"}:
+            kind = "code-rendered"
+        elif explicit in {"band", "section", "sections"}:
+            kind = "band"
+        elif explicit in {"redirect", "asset"}:
+            kind = explicit
+        elif isinstance(page.get("sections"), list) or isinstance(page.get("sectionTypes"), list):
+            kind = "band"
+        elif page.get("renderedBy") or re.search(r"\.(astro|tsx|jsx|vue|svelte)$", str(page.get("source") or page.get("file") or path), re.I):
+            kind = "code-rendered"
+        else:
+            kind = "unknown"
+        records.append({"path": path, "kind": kind})
+        if kind not in allowed or kind == "unknown":
+            errors.append(f"{path}: cannot classify published page")
+    counts = dict(sorted(Counter(item["kind"] for item in records).items()))
+    return {"schemaVersion": INVENTORY_SCHEMA, "passed": not errors and len(records) == len(seen), "records": records, "counts": counts, "bandCount": counts.get("band", 0), "codeRenderedCount": counts.get("code-rendered", 0), "errors": errors, "disjoint": True}
+
+
+def _reference_values(references: object) -> dict[str, set[str]]:
+    source = references if isinstance(references, dict) else {}
+    result: dict[str, set[str]] = {}
+    for key, values in source.items():
+        entries = values if isinstance(values, list) else [values]
+        allowed: set[str] = set()
+        for item in entries:
+            if isinstance(item, dict):
+                for candidate in (item.get("id"), item.get("slug"), item.get("name"), item.get("title"), item.get("value")):
+                    if candidate is not None and str(candidate).strip():
+                        allowed.add(str(candidate).strip())
+            elif isinstance(item, (str, int)):
+                allowed.add(str(item).strip())
+        result[str(key).casefold()] = allowed
+    return result
+
+
+def validate_bindings(sections: object, references: object) -> dict[str, Any]:
+    allowed = _reference_values(references)
+    unresolved: list[dict[str, str]] = []
+    checked: list[dict[str, str]] = []
+    suffixes = ("id", "slug", "ref", "reference")
+    values = sections if isinstance(sections, list) else sections.get("sections", []) if isinstance(sections, dict) else []
+    for index, section in enumerate(values):
+        if not isinstance(section, dict):
+            continue
+        section_id = str(section.get("id") or index)
+        for key, value in section.items():
+            if not isinstance(value, (str, int)) or not str(value).strip():
+                continue
+            key_lower = str(key).casefold()
+            base = key_lower
+            for suffix in suffixes:
+                if base.endswith(suffix) and len(base) > len(suffix):
+                    base = base[: -len(suffix)]
+                    break
+            if base not in allowed:
+                continue
+            record = {"sectionId": section_id, "field": str(key), "value": str(value)}
+            checked.append(record)
+            if str(value) not in allowed[base]:
+                unresolved.append(record)
+        binding = section.get("binding")
+        if isinstance(binding, dict):
+            binding_type = str(binding.get("type") or "").casefold()
+            binding_value = binding.get("value", binding.get("slug", binding.get("id", binding.get("ref"))))
+            if binding_type and binding_value is not None and binding_type in allowed:
+                record = {"sectionId": section_id, "field": "binding", "value": str(binding_value)}
+                checked.append(record)
+                if str(binding_value) not in allowed[binding_type]:
+                    unresolved.append(record)
+    return {"schemaVersion": BINDING_SCHEMA, "passed": not unresolved, "checked": checked, "unresolved": unresolved, "referenceTypes": sorted(allowed)}
+
+
+def validate_reconcile_contract(contract: object) -> dict[str, Any]:
+    errors: list[str] = []
+    if not isinstance(contract, dict):
+        return {"schemaVersion": IDEMPOTENCY_SCHEMA, "passed": False, "errors": ["reconcile contract must be an object"]}
+    if contract.get("schemaVersion") not in {None, IDEMPOTENCY_SCHEMA}:
+        errors.append(f"schemaVersion must be {IDEMPOTENCY_SCHEMA}")
+    candidate = contract.get("candidateSelection") if isinstance(contract.get("candidateSelection"), dict) else {}
+    strategy = str(candidate.get("strategy") or "")
+    if strategy not in {"all-candidates", "all_rows", "all-rows"}:
+        errors.append("candidateSelection.strategy must select all candidates")
+    if candidate.get("filtersOnlyUnconverted") is True:
+        errors.append("candidate selection cannot filter only rows that look converted")
+    query = str(candidate.get("query") or "").casefold()
+    if query and re.search(r"(is\s+null|=\s*'?(empty|converted|none)|jsonb_array_length\s*\([^)]*\)\s*=\s*0)", query):
+        errors.append("candidate selection query looks limited to already-unconverted rows")
+    compare_fields = contract.get("compareFields")
+    if not isinstance(compare_fields, list) or not compare_fields:
+        errors.append("compareFields must list every repaired field")
+    if contract.get("writesOnlyWhenChanged") is not True:
+        errors.append("writesOnlyWhenChanged must be true")
+    report = contract.get("report") if isinstance(contract.get("report"), dict) else {}
+    for key in ("changed", "unchanged"):
+        if report.get(key) is not True:
+            errors.append(f"report.{key} evidence is required")
+    second = contract.get("secondRun") if isinstance(contract.get("secondRun"), dict) else {}
+    if second.get("convergent") is not True:
+        errors.append("secondRun.convergent must be true")
+    return {"schemaVersion": IDEMPOTENCY_SCHEMA, "passed": not errors, "errors": errors, "mutation": "not executed", "contract": {"candidateStrategy": strategy, "compareFields": compare_fields if isinstance(compare_fields, list) else [], "writesOnlyWhenChanged": contract.get("writesOnlyWhenChanged"), "report": report, "secondRun": second}}

package/bundled-tools/runtime/maggie_sections.py

@@ -193,31 +193,79 @@ def validate_values(registry: dict[str, Any], sections: object) -> dict[str, Any
     return {"passed": not errors, "errors": errors}
 
 
-def translation_key_paths(page_id: str, sections: object, field_names: tuple[str, ...] = ("imageAlt",)) -> list[str]:
-    """Find stored section fields that need a locale sidecar value."""
+def _registry_section_fields(registry: dict[str, Any] | None, section_type: str) -> list[dict[str, Any]]:
+    if not isinstance(registry, dict):
+        return []
+    for section in registry.get("sections", []):
+        if isinstance(section, dict) and str(section.get("type")) == section_type:
+            return [field for field in section.get("fields", []) if isinstance(field, dict)]
+    return []
+
+
+def _field_translatable(field: dict[str, Any]) -> bool:
+    """Return the explicit copy policy, defaulting to translatable for text."""
+    return field.get("translatable") is not False
+
+
+def translation_key_paths(page_id: str, sections: object, field_names: tuple[str, ...] | None = ("imageAlt",), registry: dict[str, Any] | None = None) -> list[str]:
+    """Find stored section fields that need a locale sidecar value.
+
+    Older callers can continue to pass ``field_names`` and receive the legacy
+    image-alt contract. New callers pass a registry, which makes every field
+    explicitly marked ``translatable`` part of the locale contract, including
+    repeated item fields.
+    """
     if not isinstance(sections, list):
         return []
     paths: list[str] = []
 
-    def walk(value: object, path: str) -> None:
+    def walk_legacy(value: object, path: str) -> None:
         if isinstance(value, dict):
             for key, child in value.items():
                 child_path = f"{path}.{key}"
-                if key in field_names and isinstance(child, str) and child.strip():
+                if field_names and key in field_names and isinstance(child, str) and child.strip():
                     paths.append(child_path)
-                walk(child, child_path)
+                walk_legacy(child, child_path)
         elif isinstance(value, list):
             for index, child in enumerate(value):
-                walk(child, f"{path}.{index}")
+                walk_legacy(child, f"{path}.{index}")
+
+    def walk_declared(value: object, fields: list[dict[str, Any]], path: str) -> None:
+        if not isinstance(value, dict):
+            return
+        for field in fields:
+            name = str(field.get("name") or "")
+            if not name:
+                continue
+            child = value.get(name)
+            child_path = f"{path}.{name}"
+            if _field_translatable(field) and isinstance(child, str) and child.strip():
+                paths.append(child_path)
+            repeats = field.get("repeats")
+            if not isinstance(repeats, dict) or not isinstance(child, list):
+                continue
+            nested = [item for item in repeats.get("of", []) if isinstance(item, dict)]
+            for index, item in enumerate(child):
+                item_path = f"{child_path}.{index}"
+                if isinstance(item, dict):
+                    walk_declared(item, nested, item_path)
+                elif len(nested) == 1 and _field_translatable(nested[0]) and isinstance(item, str) and item.strip():
+                    paths.append(item_path)
 
     for index, section in enumerate(sections):
-        walk(section, f"page.{page_id}.sections.{section_key(section, index)}")
+        path = f"page.{page_id}.sections.{section_key(section, index)}"
+        if registry and isinstance(section, dict):
+            declared = _registry_section_fields(registry, str(section.get("type") or ""))
+            if declared:
+                walk_declared(section, declared, path)
+                continue
+        walk_legacy(section, path)
     return paths
 
 
-def validate_locale_coverage(page_id: str, sections: object, translations: object, locales: Iterable[str]) -> dict[str, Any]:
+def validate_locale_coverage(page_id: str, sections: object, translations: object, locales: Iterable[str], registry: dict[str, Any] | None = None) -> dict[str, Any]:
     """Require non-default locale rows for every declared translatable field."""
-    paths = translation_key_paths(page_id, sections)
+    paths = translation_key_paths(page_id, sections, registry=registry)
     values = translations if isinstance(translations, dict) else {}
     missing = [{"locale": locale, "key": key} for locale in locales for key in paths if not isinstance(values.get(locale), dict) or not str(values[locale].get(key) or "").strip()]
     return {"schemaVersion": LOCALE_SCHEMA, "passed": not missing, "pageId": page_id, "requiredKeys": paths, "missing": missing, "writePolicy": "section row and locale rows must be committed in one transaction"}
@@ -309,12 +357,64 @@ def validate_fanout(registry: dict[str, Any], fanout: object) -> dict[str, Any]:
     types = {str(item.get("type")) for item in registry.get("sections", []) if isinstance(item, dict)}
     surfaces = ("typeUnion", "sectionSchema", "blank", "validation", "textExtraction", "renderer", "editor")
     missing: list[dict[str, str]] = []
+
+    def field_paths(section: dict[str, Any]) -> set[str]:
+        result: set[str] = set()
+        for field in section.get("fields", []):
+            if not isinstance(field, dict) or not field.get("name"):
+                continue
+            name = str(field["name"])
+            result.add(name)
+            repeats = field.get("repeats")
+            if isinstance(repeats, dict):
+                for nested in repeats.get("of", []):
+                    if isinstance(nested, dict) and nested.get("name"):
+                        result.add(f"{name}.{nested['name']}")
+        return result
+
+    required_fields = {
+        str(section.get("type")): field_paths(section)
+        for section in registry.get("sections", [])
+        if isinstance(section, dict) and section.get("type")
+    }
+
+    def declared(values: object) -> tuple[set[str], dict[str, set[str]]]:
+        if isinstance(values, list):
+            return set(map(str, values)), {}
+        if not isinstance(values, dict):
+            return set(), {}
+        types = set(map(str, values.keys()))
+        fields: dict[str, set[str]] = {}
+        for section_type, entry in values.items():
+            if isinstance(entry, list):
+                fields[str(section_type)] = set(map(str, entry))
+            elif isinstance(entry, dict):
+                candidate = entry.get("fields", [])
+                if isinstance(candidate, list):
+                    fields[str(section_type)] = set(map(str, candidate))
+        return types, fields
+
     for surface in surfaces:
         values = fanout.get(surface)
-        declared = set(values) if isinstance(values, list) else set(values.keys()) if isinstance(values, dict) else set()
-        for section_type in sorted(types - declared):
+        declared_types, declared_fields = declared(values)
+        for section_type in sorted(types - declared_types):
             missing.append({"type": section_type, "surface": surface})
-    return {"schemaVersion": FANOUT_SCHEMA, "passed": not missing, "errors": [f"{item['type']} missing {item['surface']} fan-out" for item in missing], "sectionTypes": sorted(types), "missing": missing}
+        for section_type in sorted(types & declared_types):
+            for field in sorted(required_fields.get(section_type, set()) - declared_fields.get(section_type, set())):
+                missing.append({"type": section_type, "surface": f"{surface}.{field}"})
+
+    editor_screens = fanout.get("editorScreens")
+    if not isinstance(editor_screens, dict) or len(editor_screens) < 2:
+        missing.append({"type": "editor", "surface": "editorScreens (at least two screens required)"})
+    elif isinstance(editor_screens, dict):
+        for screen, values in editor_screens.items():
+            screen_types, screen_fields = declared(values)
+            for section_type in sorted(types - screen_types):
+                missing.append({"type": section_type, "surface": f"editorScreens.{screen}"})
+            for section_type in sorted(types & screen_types):
+                for field in sorted(required_fields.get(section_type, set()) - screen_fields.get(section_type, set())):
+                    missing.append({"type": section_type, "surface": f"editorScreens.{screen}.{field}"})
+    return {"schemaVersion": FANOUT_SCHEMA, "passed": not missing, "errors": [f"{item['type']} missing {item['surface']} fan-out" for item in missing], "sectionTypes": sorted(types), "missing": missing, "fieldAware": True, "editorScreenCount": len(editor_screens) if isinstance(editor_screens, dict) else 0}
 
 
 def reconcile_fields(current: object, desired: object) -> dict[str, Any]:

package/bundled-tools/runtime/maggie_sitemap.py

@@ -14,6 +14,7 @@ HARD_URL_LIMIT = 50_000
 HARD_BYTES_LIMIT = 52_428_800
 DEFAULT_CHUNK_TARGET = 500
 EXCLUDED_PATHS = re.compile(r"/(search|find|login|draft|preview)(/|$)", re.I)
+W3C_UTC_DATETIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
 
 
 def absolute_url(url: str, origin: str) -> bool:
@@ -121,7 +122,7 @@ def build_plan(routes: list[dict[str, str]], origin: str, content_types: set[str
     current = set(sitemap_urls)
     old = set((previous or {}).get("sitemapUrls", []))
     redirects = [{"from": url, "to": origin.rstrip("/") + "/sitemap.xml", "reason": "previously advertised sitemap removed"} for url in sorted(old - current)]
-    plan = {"schemaVersion": "maggie-seo-sitemap-plan.v1", "planId": "sitemap-" + datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"), "origin": origin.rstrip("/"), "groups": group_plans, "index": {"url": origin.rstrip("/") + "/sitemap.xml", "bytes": len(index_xml.encode()), "xml": index_xml, "sitemapUrls": sitemap_urls, "responseHeaders": {"content-type": "text/xml; charset=utf-8", "x-robots-tag": "all"}}, "redirects": redirects, "excluded": excluded}
+    plan = {"schemaVersion": "maggie-seo-sitemap-plan.v1", "planId": "sitemap-" + datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"), "origin": origin.rstrip("/"), "groups": group_plans, "index": {"url": origin.rstrip("/") + "/sitemap.xml", "bytes": len(index_xml.encode()), "xml": index_xml, "sitemapUrls": sitemap_urls, "responseHeaders": {"content-type": "text/xml; charset=utf-8", "x-robots-tag": "noindex"}}, "redirects": redirects, "excluded": excluded}
     plan["validation"] = validate_plan_data(plan)
     return plan
 
@@ -163,9 +164,16 @@ def validate_plan_data(plan: dict, strict_semantic: bool = False) -> dict:
                 for link in root.iter():
                     if link.tag.endswith("link") and link.get("rel") == "alternate" and not absolute_url(link.get("href", ""), origin):
                         errors.append(f"chunk contains a relative or off-origin alternate: {chunk.get('filename')}")
+                    if link.tag.endswith("lastmod") and not W3C_UTC_DATETIME.fullmatch((link.text or "").strip()):
+                        errors.append(f"lastmod must be a full W3C UTC datetime: {chunk.get('filename')}")
             except ElementTree.ParseError:
                 errors.append(f"chunk is not valid XML: {chunk.get('filename')}")
     index = plan.get("index", {})
+    headers = index.get("responseHeaders") if isinstance(index.get("responseHeaders"), dict) else {}
+    if headers.get("content-type") != "text/xml; charset=utf-8":
+        errors.append("sitemap response content-type must be text/xml; charset=utf-8")
+    if str(headers.get("x-robots-tag") or "").casefold() != "noindex":
+        errors.append("sitemap response x-robots-tag must be noindex")
     if index.get("bytes", 0) > HARD_BYTES_LIMIT:
         errors.append("sitemap index exceeds byte limit")
     try:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@topy-ai/maggie",
-  "version": "0.7.12",
+  "version": "0.7.14",
   "description": "Install and manage Maggie Skills for AI coding agents",
   "license": "MIT",
   "type": "module",