@topy-ai/maggie 0.7.11 → 0.7.12

package/README.md

@@ -218,8 +218,8 @@ artifact schemas.
 Recommended upgrade sequence for the current release:
 
 ```bash
-npx @topy-ai/maggie@0.7.11 update --project . --force
-npx @topy-ai/maggie@0.7.11 cleanup --project .
+npx @topy-ai/maggie@0.7.12 update --project . --force
+npx @topy-ai/maggie@0.7.12 cleanup --project .
 ```
 
 Maintainers should pass npm credentials through the repository helper, never
@@ -229,7 +229,7 @@ as a command-line argument:
 node scripts/publish-npm.mjs --maggie-env-file ../.env
 ```
 
-The 0.7.11 workflow adds served-content equivalence checks, query-route
+The 0.7.12 workflow adds served-content equivalence checks, query-route
 baseline exclusions, changed-surface render evidence gates, generated skill
 catalogs, component-binding audits, and icon-family noise filtering. It also
 includes the 0.7.9 nested section-field contracts, renderer-backed examples,
@@ -255,7 +255,9 @@ also adds image-aware content equivalence with explicit legacy-baseline
 limitations, locale checks for stored image alt text, a validated label/value
 `pairs` section, registry fan-out checks, idempotent reconciliation, per-step
 and section-scoped design evidence, catch-all source confidence, shell-safe VPS
-SSH guidance, and compatible feedback CLI examples. The release
+SSH guidance, compatible feedback CLI examples, conventional sitemap
+formatting, response-header guidance, and access-log diagnostics for absent
+crawler requests. The release
 retains the existing service matching,
 localization, seed-manifest, lockfile/analytics, sitemap, deployment and
 rollback workflows.

package/bin/maggie.js

@@ -112,7 +112,7 @@ Usage:
   maggie localization <extract|plan|generate|preview|validate|review|publish|stale|glossary> [options]
   maggie seo performance|images|sitemap [options] (sitemap supports strict validate and agent-files)
   maggie feedback <collect|preview|submit|list> [options]
-  maggie site-audit URL [--crawl] [--languages en-GB,es-MX,ja-JP] [--check-hreflang]
+  maggie site-audit URL [--crawl] [--access-log FILE] [--require-sitemap-request] [--languages en-GB,es-MX,ja-JP] [--check-hreflang]
   maggie site-audit URL --crawl --save-baseline FILE --reviewer NAME
   maggie site-audit URL --crawl --baseline FILE
   maggie browser-audit URL --browse PATH --output DIR --required SELECTOR [--sticky SELECTOR]

package/bundled-skills/maggie-seo-geo/SKILL.md

@@ -59,6 +59,26 @@ If the change date is unknown, omit `lastmod`. An empty content-type does not
 need a sitemap chunk in the sitemap index; serving an empty endpoint and
 advertising it are separate decisions.
 
+Generated sitemap XML uses the conventional readable shape by default: one
+`<url>`/`<sitemap>` entry per block, UTF-8 XML, and date-only `lastmod` evidence
+rendered as a full UTC W3C datetime. The plan also exposes the response
+contract (`text/xml; charset=utf-8` and `X-Robots-Tag: all`) for the framework
+route or reverse proxy to apply; writing a file cannot set HTTP headers itself.
+When Search Console reports a valid sitemap as unfetched, probe success alone
+is not a diagnosis. Supply a sanitized local access log to distinguish
+`googlebot-request-observed`, another-client-only traffic, and
+`no-matching-request`:
+
+```bash
+maggie site-audit https://example.com --access-log /var/log/nginx/access.log
+maggie site-audit https://example.com --access-log /var/log/nginx/access.log \
+  --require-sitemap-request
+```
+
+The strict flag is for a review window where a crawler request is expected; it
+fails only when the supplied log contains no sitemap request. Never upload raw
+logs or include IPs, credentials, or query data in feedback or reports.
+
 ## Freeze and compare a reviewed site
 
 ```bash

package/bundled-tools/clis/site_audit.py

@@ -257,6 +257,15 @@ def sitemap_urls(base: str) -> tuple[list[str], list[dict[str, str]]]:
     return list(dict.fromkeys(pages)), violations
 
 
+def access_log_sitemap_check(path: Path, sitemap_path: str = "/sitemap.xml") -> dict[str, object]:
+    """Distinguish a sitemap fetch failure from a crawler that never asked."""
+    text = path.read_text(encoding="utf-8", errors="replace")
+    escaped = re.escape(sitemap_path)
+    requests = [line for line in text.splitlines() if re.search(rf"(?:GET|HEAD)\s+{escaped}(?:[?\s\"]|$)", line, re.I)]
+    googlebot = [line for line in requests if "googlebot" in line.lower()]
+    return {"provided": True, "path": str(path), "requestCount": len(requests), "googlebotRequestCount": len(googlebot), "status": "requested" if requests else "not-requested", "diagnosis": "googlebot-request-observed" if googlebot else ("other-client-request-only" if requests else "no-matching-request")}
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("url")
@@ -269,6 +278,8 @@ def main() -> int:
     parser.add_argument("--markets", help="comma-separated markets: global,uk,us")
     parser.add_argument("--check-hreflang", action="store_true")
     parser.add_argument("--check-translation-completeness", action="store_true")
+    parser.add_argument("--access-log", type=Path, help="optional local access log for crawler-request diagnosis")
+    parser.add_argument("--require-sitemap-request", action="store_true", help="fail unless the supplied access log contains a sitemap request")
     baseline_args = parser.add_mutually_exclusive_group()
     baseline_args.add_argument("--save-baseline", type=Path, help="create a new reviewed contract from a passing complete crawl")
     baseline_args.add_argument("--baseline", type=Path, help="fail on differences from a reviewed contract")
@@ -326,6 +337,12 @@ def main() -> int:
                 "mentions_sitemap": "sitemap" in body.lower() if name == "robots" else None,
                 "url_count": len(re.findall(r"<loc>.*?</loc>", body, re.I | re.S)) if name == "sitemap" else None,
             }
+            if name == "sitemap" and args.access_log:
+                access = access_log_sitemap_check(args.access_log)
+                checks["sitemap_access_log"] = {"ok": not args.require_sitemap_request or access["requestCount"] > 0, **access}
+                checks[name]["access_log"] = access
+                if args.require_sitemap_request and not access["requestCount"]:
+                    checks[name]["ok"] = False
         except Exception as exc:
             checks[name] = {"ok": False, "error": type(exc).__name__}
 

package/bundled-tools/runtime/maggie_sitemap.py

@@ -74,13 +74,29 @@ def filter_routes(routes: list[dict[str, str]], origin: str, content_types: set[
     return groups, excluded
 
 
+def conventional_lastmod(value: str) -> str:
+    """Render date-only source evidence as a W3C/ISO UTC datetime."""
+    if re.fullmatch(r"\d{4}-\d{2}-\d{2}", value):
+        return value + "T00:00:00Z"
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return value
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
+
+
 def xml_file(urls: list[dict[str, str]]) -> str:
     rows = []
     for item in urls:
-        lastmod = f"<lastmod>{escape(item['lastmod'])}</lastmod>" if item.get("lastmod") else ""
-        alternate_xml = "".join(f'<xhtml:link rel="alternate" hreflang="{escape(str(lang))}" href="{escape(str(url))}" />' for lang, url in sorted((item.get("alternates") or {}).items()))
-        rows.append(f"<url><loc>{escape(item['url'])}</loc>{lastmod}{alternate_xml}</url>")
-    return '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">' + "".join(rows) + "</urlset>\n"
+        fields = [f"  <loc>{escape(item['url'])}</loc>"]
+        if item.get("lastmod"):
+            fields.append(f"  <lastmod>{escape(conventional_lastmod(item['lastmod']))}</lastmod>")
+        fields.extend(f'  <xhtml:link rel="alternate" hreflang="{escape(str(lang))}" href="{escape(str(url))}" />' for lang, url in sorted((item.get("alternates") or {}).items()))
+        rows.append("  <url>\n" + "\n".join(fields) + "\n  </url>")
+    body = "\n".join(rows)
+    return '<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml">\n' + body + ('\n' if body else '') + '</urlset>\n'
 
 
 def build_plan(routes: list[dict[str, str]], origin: str, content_types: set[str], chunk_target: int = DEFAULT_CHUNK_TARGET, previous: dict | None = None) -> dict:
@@ -100,11 +116,12 @@ def build_plan(routes: list[dict[str, str]], origin: str, content_types: set[str
             chunks.append({"filename": filename, "url": url, "entries": len(chunk_entries), "routes": chunk_entries, "bytes": len(xml.encode()), "sha256": hashlib.sha256(xml.encode()).hexdigest(), "xml": xml})
             sitemap_urls.append(url)
         group_plans.append({"contentType": content_type, "chunks": chunks})
-    index_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">' + "".join(f"<sitemap><loc>{escape(url)}</loc></sitemap>" for url in sitemap_urls) + "</sitemapindex>\n"
+    index_rows = "\n".join(f"  <sitemap>\n    <loc>{escape(url)}</loc>\n  </sitemap>" for url in sitemap_urls)
+    index_xml = '<?xml version="1.0" encoding="UTF-8"?>\n<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n' + index_rows + ('\n' if index_rows else '') + '</sitemapindex>\n'
     current = set(sitemap_urls)
     old = set((previous or {}).get("sitemapUrls", []))
     redirects = [{"from": url, "to": origin.rstrip("/") + "/sitemap.xml", "reason": "previously advertised sitemap removed"} for url in sorted(old - current)]
-    plan = {"schemaVersion": "maggie-seo-sitemap-plan.v1", "planId": "sitemap-" + datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"), "origin": origin.rstrip("/"), "groups": group_plans, "index": {"url": origin.rstrip("/") + "/sitemap.xml", "bytes": len(index_xml.encode()), "xml": index_xml, "sitemapUrls": sitemap_urls}, "redirects": redirects, "excluded": excluded}
+    plan = {"schemaVersion": "maggie-seo-sitemap-plan.v1", "planId": "sitemap-" + datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S"), "origin": origin.rstrip("/"), "groups": group_plans, "index": {"url": origin.rstrip("/") + "/sitemap.xml", "bytes": len(index_xml.encode()), "xml": index_xml, "sitemapUrls": sitemap_urls, "responseHeaders": {"content-type": "text/xml; charset=utf-8", "x-robots-tag": "all"}}, "redirects": redirects, "excluded": excluded}
     plan["validation"] = validate_plan_data(plan)
     return plan
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@topy-ai/maggie",
-  "version": "0.7.11",
+  "version": "0.7.12",
   "description": "Install and manage Maggie Skills for AI coding agents",
   "license": "MIT",
   "type": "module",