@topy-ai/maggie 0.7.13 → 0.7.15

package/bundled-tools/runtime/maggie_indexnow.py

@@ -0,0 +1,128 @@
+"""Provider-neutral, change-driven IndexNow planning and submission state."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any
+from urllib.error import HTTPError, URLError
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
+
+
+SCHEMA = "maggie-indexnow.v1"
+DEFAULT_GUARD_HOURS = 24
+ACCEPTED = {200, 202}
+
+
+def _now(value: str | None = None) -> datetime:
+    if value:
+        return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
+    return datetime.now(timezone.utc)
+
+
+def _iso(value: datetime) -> str:
+    return value.astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
+
+
+def normalise_urls(origin: str, urls: list[object]) -> tuple[list[str], list[str]]:
+    """Return valid same-origin URLs and explicit rejects."""
+    base = origin.rstrip("/")
+    origin_host = urlparse(base).netloc
+    accepted: set[str] = set()
+    rejected: list[str] = []
+    for raw in urls:
+        value = str(raw or "").strip()
+        if not value:
+            continue
+        candidate = value if value.startswith(("http://", "https://")) else f"{base}/{value.lstrip('/')}"
+        parsed = urlparse(candidate)
+        if parsed.scheme not in {"http", "https"} or parsed.netloc != origin_host or parsed.fragment:
+            rejected.append(value)
+            continue
+        accepted.add(candidate)
+    return sorted(accepted), rejected
+
+
+def _state_entries(state: object) -> list[dict[str, Any]]:
+    if not isinstance(state, dict) or not isinstance(state.get("submissions"), list):
+        return []
+    return [item for item in state["submissions"] if isinstance(item, dict)]
+
+
+def plan_indexnow(origin: str, urls: list[object], state: object | None = None,
+                  *, key: str, key_location: str | None = None,
+                  guard_hours: int = DEFAULT_GUARD_HOURS, now: str | None = None) -> dict[str, Any]:
+    if not key.strip():
+        raise ValueError("IndexNow key is required; it is a public ownership key, not a provider secret")
+    eligible, rejected = normalise_urls(origin, urls)
+    cutoff = _now(now) - timedelta(hours=guard_hours)
+    accepted_at: dict[str, datetime] = {}
+    for entry in _state_entries(state):
+        if entry.get("statusCode") not in ACCEPTED or not entry.get("acceptedAt"):
+            continue
+        try:
+            stamp = _now(str(entry["acceptedAt"]))
+        except ValueError:
+            continue
+        if stamp >= cutoff:
+            accepted_at[str(entry.get("url"))] = stamp
+    held_back = [{"url": url, "acceptedAt": _iso(accepted_at[url]), "reason": f"accepted within {guard_hours} hours"}
+                 for url in eligible if url in accepted_at]
+    held_set = {item["url"] for item in held_back}
+    return {
+        "schemaVersion": SCHEMA,
+        "origin": origin.rstrip("/"),
+        "key": key.strip(),
+        "keyLocation": key_location or f"{origin.rstrip('/')}/{key.strip()}.txt",
+        "guardHours": guard_hours,
+        "eligible": [url for url in eligible if url not in held_set],
+        "heldBack": held_back,
+        "rejected": rejected,
+        "verification": {"required": True, "status": "not-checked"},
+        "plannedAt": _iso(_now(now)),
+        "mutation": False,
+    }
+
+
+def check_key_file(public_dir: Path, key_file: str, key: str) -> dict[str, Any]:
+    if not key.strip():
+        raise ValueError("key is required")
+    relative = Path(key_file)
+    if relative.is_absolute() or ".." in relative.parts or len(relative.parts) != 1:
+        raise ValueError("key file must be a single filename inside public-dir")
+    target = public_dir / relative
+    served = target.read_text(encoding="utf-8").strip() if target.exists() else None
+    return {"schemaVersion": SCHEMA, "path": str(relative), "exists": target.exists(),
+            "matches": served == key.strip(), "status": "pass" if served == key.strip() else "fail"}
+
+
+def submit_plan(plan: dict[str, Any], state: dict[str, Any], endpoint: str) -> dict[str, Any]:
+    urls = plan.get("eligible") if isinstance(plan.get("eligible"), list) else []
+    if not urls:
+        return {"schemaVersion": SCHEMA, "status": "no-op", "submitted": 0, "accepted": 0, "retryable": False, "mutation": True}
+    payload = {"host": urlparse(str(plan["origin"])).netloc, "key": str(plan["key"]),
+               "keyLocation": str(plan["keyLocation"]), "urlList": urls}
+    status_code: int | None = None
+    failure = None
+    try:
+        request = Request(endpoint, data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json; charset=utf-8"}, method="POST")
+        with urlopen(request, timeout=30) as response:
+            status_code = int(response.status)
+    except HTTPError as error:
+        status_code = int(error.code)
+        failure = "provider returned HTTP status"
+    except (URLError, TimeoutError, OSError):
+        failure = "provider request failed"
+    accepted = status_code in ACCEPTED
+    retryable = status_code in {403, 429} or (status_code is not None and status_code >= 500) or failure is not None
+    entries = _state_entries(state)
+    if accepted:
+        stamp = _iso(_now())
+        entries.extend({"url": url, "statusCode": status_code, "acceptedAt": stamp, "batchId": hashlib.sha256((stamp + url).encode()).hexdigest()[:12]} for url in urls)
+    state.update({"schemaVersion": SCHEMA, "submissions": entries})
+    return {"schemaVersion": SCHEMA, "status": "accepted" if accepted else "retryable-failure" if retryable else "failed",
+            "submitted": len(urls), "accepted": len(urls) if accepted else 0, "statusCode": status_code,
+            "retryable": retryable, "failure": failure, "mutation": True}

package/bundled-tools/runtime/maggie_quality.py

@@ -14,6 +14,7 @@ MEDIA_SCHEMA = "maggie-media-uniqueness.v1"
 INVENTORY_SCHEMA = "maggie-page-inventory.v1"
 BINDING_SCHEMA = "maggie-section-bindings.v1"
 IDEMPOTENCY_SCHEMA = "maggie-reconcile-contract.v1"
+PAGE_KINDS = {"service", "variant", "category-hub", "ordinary", "blog-topic", "blog-archive", "blog-article", "homepage", "service-index", "service-category", "redirect", "asset"}
 
 
 def _items(value: object, *keys: str) -> list[dict[str, Any]]:
@@ -116,8 +117,10 @@ def validate_media_uniqueness(value: object, *, across_siblings: bool = False) -
     return {"schemaVersion": MEDIA_SCHEMA, "passed": not errors, "duplicates": errors, "pages": len(pages), "acrossSiblings": across_siblings}
 
 
-def classify_inventory(value: object) -> dict[str, Any]:
+def classify_inventory(value: object, *, require_page_kinds: bool = False, require_source_coverage: bool = False) -> dict[str, Any]:
     pages = _items(value, "pages", "items", "routes")
+    if isinstance(value, dict):
+        pages += _items(value, "codeRenderedPages", "codeRenderedRoutes")
     allowed = {"band", "code-rendered", "redirect", "asset", "unknown"}
     records: list[dict[str, Any]] = []
     errors: list[str] = []
@@ -127,7 +130,7 @@ def classify_inventory(value: object) -> dict[str, Any]:
         if path in seen:
             errors.append(f"duplicate inventory path: {path}")
         seen.add(path)
-        explicit = str(page.get("kind") or page.get("classification") or "").casefold()
+        explicit = str(page.get("rendererKind") or page.get("rendering") or page.get("kind") or page.get("classification") or "").casefold()
         if explicit in {"code", "code-rendered", "component", "source"}:
             kind = "code-rendered"
         elif explicit in {"band", "section", "sections"}:
@@ -140,11 +143,25 @@ def classify_inventory(value: object) -> dict[str, Any]:
             kind = "code-rendered"
         else:
             kind = "unknown"
-        records.append({"path": path, "kind": kind})
+        raw_page_kind = page.get("pageKind") or page.get("pageType")
+        if not raw_page_kind and str(page.get("kind") or "").casefold() in PAGE_KINDS:
+            raw_page_kind = page.get("kind")
+        page_kind = str(raw_page_kind or "").casefold().replace("_", "-")
+        records.append({"path": path, "kind": kind, "pageKind": page_kind or None})
         if kind not in allowed or kind == "unknown":
             errors.append(f"{path}: cannot classify published page")
+        if page_kind and page_kind not in PAGE_KINDS:
+            errors.append(f"{path}: unknown pageKind {page_kind}")
+        if require_page_kinds and not page_kind:
+            errors.append(f"{path}: pageKind is required")
     counts = dict(sorted(Counter(item["kind"] for item in records).items()))
-    return {"schemaVersion": INVENTORY_SCHEMA, "passed": not errors and len(records) == len(seen), "records": records, "counts": counts, "bandCount": counts.get("band", 0), "codeRenderedCount": counts.get("code-rendered", 0), "errors": errors, "disjoint": True}
+    page_kind_counts = dict(sorted(Counter(item["pageKind"] for item in records if item["pageKind"]).items()))
+    coverage = value.get("sourceCoverage") if isinstance(value, dict) else None
+    if not isinstance(coverage, dict):
+        coverage = {"status": "unverified", "complete": False, "sources": []}
+    if require_source_coverage and coverage.get("complete") is not True:
+        errors.append("sourceCoverage.complete must be true when source coverage is required")
+    return {"schemaVersion": INVENTORY_SCHEMA, "passed": not errors and len(records) == len(seen), "records": records, "counts": counts, "pageKindCounts": page_kind_counts, "pageKinds": sorted(PAGE_KINDS), "bandCount": counts.get("band", 0), "codeRenderedCount": counts.get("code-rendered", 0), "errors": errors, "disjoint": True, "sourceCoverage": coverage}
 
 
 def _reference_values(references: object) -> dict[str, set[str]]:

package/bundled-tools/runtime/maggie_sitemap.py

@@ -15,6 +15,8 @@ HARD_BYTES_LIMIT = 52_428_800
 DEFAULT_CHUNK_TARGET = 500
 EXCLUDED_PATHS = re.compile(r"/(search|find|login|draft|preview)(/|$)", re.I)
 W3C_UTC_DATETIME = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
+FRESHNESS_MIN_SAMPLE = 10
+FRESHNESS_CONCENTRATION_THRESHOLD = 0.75
 
 
 def absolute_url(url: str, origin: str) -> bool:
@@ -100,6 +102,51 @@ def xml_file(urls: list[dict[str, str]]) -> str:
     return '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">\n' + body + ('\n' if body else '') + '</urlset>\n'
 
 
+def freshness_report(routes: list[dict[str, str]], today: str | None = None) -> dict[str, object]:
+    """Detect a suspiciously uniform lastmod distribution.
+
+    A structurally valid sitemap can still lose crawler trust when a migration
+    stamps nearly every URL with the same operational date. This is a warning
+    by default so unknown or intentionally batched content is not silently
+    rewritten; strict semantic validation promotes it to an error.
+    """
+    dates: list[str] = []
+    for route in routes:
+        value = route.get("lastmod")
+        if not value:
+            continue
+        normalised = conventional_lastmod(str(value))
+        if W3C_UTC_DATETIME.fullmatch(normalised):
+            dates.append(normalised[:10])
+    counts: dict[str, int] = {}
+    for date in dates:
+        counts[date] = counts.get(date, 0) + 1
+    dominant_date, dominant_count = (max(counts.items(), key=lambda item: (item[1], item[0]))
+                                     if counts else (None, 0))
+    sample = len(dates)
+    ratio = dominant_count / sample if sample else 0.0
+    current = today or datetime.now(timezone.utc).date().isoformat()
+    suspicious = bool(sample >= FRESHNESS_MIN_SAMPLE and ratio >= FRESHNESS_CONCENTRATION_THRESHOLD)
+    warnings: list[str] = []
+    if suspicious:
+        date_label = "today's date" if dominant_date == current else str(dominant_date)
+        warnings.append(
+            f"lastmod distribution is concentrated on {date_label} "
+            f"({dominant_count}/{sample}, {ratio:.0%}); verify content-change provenance"
+        )
+    return {
+        "status": "warn" if warnings else "pass",
+        "datedRoutes": sample,
+        "distinctDates": len(counts),
+        "dominantDate": dominant_date,
+        "dominantCount": dominant_count,
+        "dominantRatio": round(ratio, 4),
+        "today": current,
+        "todayCount": counts.get(current, 0),
+        "warnings": warnings,
+    }
+
+
 def build_plan(routes: list[dict[str, str]], origin: str, content_types: set[str], chunk_target: int = DEFAULT_CHUNK_TARGET, previous: dict | None = None) -> dict:
     if chunk_target < 1 or chunk_target > HARD_URL_LIMIT:
         raise ValueError("chunk target must be between 1 and 50000")
@@ -134,6 +181,7 @@ def validate_plan_data(plan: dict, strict_semantic: bool = False) -> dict:
     if not urlparse(origin).scheme or not urlparse(origin).netloc:
         errors.append("origin must be an absolute HTTP(S) URL")
     expected_urls = []
+    all_routes: list[dict[str, str]] = []
     for group in plan.get("groups", []):
         for chunk in group.get("chunks", []):
             expected_urls.append(chunk.get("url"))
@@ -142,6 +190,7 @@ def validate_plan_data(plan: dict, strict_semantic: bool = False) -> dict:
             if chunk.get("bytes", 0) > HARD_BYTES_LIMIT:
                 errors.append(f"chunk exceeds byte limit: {chunk.get('filename')}")
             for route in chunk.get("routes", []):
+                all_routes.append(route)
                 if route.get("indexable") is False or route.get("searchable") is False:
                     errors.append(f"non-indexable/searchable route was emitted: {route.get('url')}")
                 if route.get("canonicalUrl") and route.get("canonicalUrl") != route.get("url"):
@@ -187,6 +236,8 @@ def validate_plan_data(plan: dict, strict_semantic: bool = False) -> dict:
         errors.append("sitemap index is not valid XML")
     if not expected_urls:
         warnings.append("no sitemap chunks generated; empty content types are not advertised")
+    freshness = freshness_report(all_routes)
+    warnings.extend(freshness["warnings"])
     if strict_semantic:
         for group in plan.get("groups", []):
             for chunk in group.get("chunks", []):
@@ -195,7 +246,7 @@ def validate_plan_data(plan: dict, strict_semantic: bool = False) -> dict:
                     if "contentWords" not in route: warnings.append(f"semantic content evidence missing: {route.get('url')}")
         if warnings:
             errors.extend(warnings)
-    return {"status": "pass" if not errors else "fail", "errors": errors, "warnings": warnings}
+    return {"status": "pass" if not errors else "fail", "errors": errors, "warnings": warnings, "freshness": freshness}
 
 
 def agent_files(routes: list[dict[str, str]], origin: str, locale: str = "en") -> dict[str, str]:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@topy-ai/maggie",
-  "version": "0.7.13",
+  "version": "0.7.15",
   "description": "Install and manage Maggie Skills for AI coding agents",
   "license": "MIT",
   "type": "module",