@topy-ai/maggie 0.7.54 → 0.7.55

package/bundled-tools/clis/maggie.py

@@ -14,7 +14,7 @@ import time
 from datetime import datetime, timezone
 from pathlib import Path
 from urllib.error import HTTPError, URLError
-from urllib.parse import quote
+from urllib.parse import quote, urlsplit
 from urllib.request import Request, urlopen
 
 
@@ -478,13 +478,73 @@ def command_api_lifecycle(args: argparse.Namespace) -> int:
     def record(step: str, value: object):
         state["steps"][step] = {"status":"passed","completed_at":utc_now(),"summary":response_summary(value)}
         if step in {"posts","updates"}:
-            items = value.get("items", []) if isinstance(value, dict) else value if isinstance(value, list) else []
+            items = delivered_items(value, step)
+            if step == "updates":
+                state["steps"][step]["summary"]["item_count"] = len(items)
             for item in items:
                 if not isinstance(item, dict): continue
                 remote_id = item.get("content_id") or item.get("contentId") or item.get("id") or item.get("post_id") or item.get("postId")
                 if not remote_id: continue
                 body = item.get("content") or item.get("body") or item.get("html") or item.get("content_markdown") or item.get("content_html") or ""
-                state["content"][str(remote_id)] = {"remote_content_id":str(remote_id),"version":item.get("version") or item.get("updated_at") or item.get("updatedAt"),"slug":item.get("slug"),"canonical_url":item.get("canonical_url") or item.get("canonicalUrl") or item.get("url"),"title":item.get("title"),"content":body,"content_markdown":item.get("content_markdown") or item.get("contentMarkdown"),"content_html":item.get("content_html") or item.get("contentHtml"),"content_portable_text":item.get("content_portable_text") or item.get("portable_text") or item.get("contentPortableText"),"status":item.get("status") or "pulled","sha256":hashlib.sha256(json.dumps(item,sort_keys=True,ensure_ascii=False).encode()).hexdigest(),"source_step":step,"pulled_at":utc_now()}
+                fields = {"version":item.get("version") or item.get("version_id") or item.get("updated_at") or item.get("updatedAt"),"content":body,"content_markdown":item.get("content_markdown") or item.get("contentMarkdown"),"content_html":item.get("content_html") or item.get("contentHtml"),"content_portable_text":item.get("content_portable_text") or item.get("portable_text") or item.get("contentPortableText"),"sha256":hashlib.sha256(json.dumps(item,sort_keys=True,ensure_ascii=False).encode()).hexdigest(),"source_step":step,"pulled_at":utc_now()}
+                if step == "updates":
+                    apply_rewrite(item, str(remote_id), fields)
+                    continue
+                previous = state["content"].get(str(remote_id), {})
+                record_value = {"remote_content_id":str(remote_id),"slug":item.get("slug"),"canonical_url":item.get("canonical_url") or item.get("canonicalUrl") or item.get("url"),"title":item.get("title"),**fields,"status":item.get("status") or "pulled"}
+                # A re-delivered post keeps where it went live and its rewrite history.
+                for key in ("live_url","live_url_source","live_url_matched_at","rewrites","previous_slugs"):
+                    if key in previous: record_value[key] = previous[key]
+                state["content"][str(remote_id)] = record_value
+
+    def find_rewrite_target(item: dict, remote_id: str) -> str | None:
+        """The delivered post a rewrite is for: by the id /posts delivered it
+        under, else by its slug or address (previous slugs too)."""
+        source = item.get("source_content_id") or item.get("sourceContentId") or item.get("original_content_id")
+        for key in (source, remote_id):
+            if key is not None and str(key) in state["content"]:
+                return str(key)
+        slugs = {str(item.get("slug") or "").strip("/")}
+        address = str(item.get("canonical_url") or item.get("canonicalUrl") or item.get("url") or "")
+        if address:
+            slugs.add(urlsplit(address).path.rstrip("/").rsplit("/", 1)[-1])
+        slugs.discard("")
+        for key, existing in state["content"].items():
+            if not isinstance(existing, dict): continue
+            known = {str(existing.get("slug") or "").strip("/"), *map(str, existing.get("previous_slugs") or [])}
+            if existing.get("live_url"):
+                known.add(urlsplit(str(existing["live_url"])).path.rstrip("/").rsplit("/", 1)[-1])
+            if slugs & known:
+                return key
+        return None
+
+    def apply_rewrite(item: dict, remote_id: str, fields: dict) -> None:
+        target = find_rewrite_target(item, remote_id)
+        held = str(item.get("publication_status") or "").lower() == "draft"
+        entry = {"rewrite_content_id": remote_id, "version": fields["version"], "change_reason": str(item.get("change_reason") or "")[:500], "sha256": fields["sha256"], "pulled_at": fields["pulled_at"]}
+        if target is None or not str(fields["content"]).strip():
+            # Never create a new post from a rewrite: keep it for review instead.
+            state.setdefault("rewrites_unmatched", []).append({**entry, "slug": item.get("slug"), "status": "unmatched"})
+            return
+        record_value = state["content"][target]
+        entry["status"] = "held" if held else "pending-apply"
+        if not held:
+            # Only the body changes; title, slug, status and the live URL stay.
+            entry["previous_sha256"] = record_value.get("sha256")
+            record_value.update({key: fields[key] for key in ("version","content","content_markdown","content_html","content_portable_text","sha256","pulled_at")})
+            record_value["rewrite_pending"] = True
+        record_value.setdefault("rewrites", []).append(entry)
+
+    def delivered_items(value: object, step: str) -> list:
+        """/posts and /updates answer {items: [...]}, {posts: [...]} or {updates: [...]}."""
+        if isinstance(value, list):
+            return value
+        if not isinstance(value, dict):
+            return []
+        for key in ("items", step, "posts", "updates"):
+            if isinstance(value.get(key), list):
+                return value[key]
+        return []
 
     def record_read(step: str, value: object):
         state["steps"][step] = {"status":"passed","completed_at":utc_now(),"summary":response_summary(value)}

package/bundled-tools/clis/maggie_assistant_tools.py

@@ -6,7 +6,8 @@ server and ChatGPT/Claude apps. The product exports its registry as JSON
 (``maggie-assistant-tool-registry.v1``); these commands never import product
 code or call a model.
 
-- ``registry-validate``: names, risk levels, gates, scopes, channels, and the
+- ``registry-validate``: names, risk levels, gates, scopes, channels, clipped
+  lists, the report inbox, unattended-caller caps and sign-in defaults, and the
   user-facing copy (label, detail, confirm sentence) kept separate from the
   model-facing description. User copy may not contain snake_case tool names,
   "the user", or "(from some_tool)" references.
@@ -17,6 +18,10 @@ code or call a model.
 - ``audit-check``: every write in a recorded session has one audit entry, with
   a readable summary (never a bare UUID) and the right scope (account-level
   writes are not filed under the open site).
+- ``list-check``: a recorded list result marks every item whose declared
+  field was shortened (``clipped``) and carries the id its get tool needs.
+- ``report-check``: a report an agent is about to save to the report inbox
+  (title, one-line summary, Markdown body) fits the limits and holds no secret.
 - ``approval-selftest``: exercises the reference signed one-time approval
   token and standing workflow approvals (``tools/runtime/assistant_approval.py``).
 """
@@ -187,10 +192,222 @@ def validate_registry(registry: dict) -> dict:
             errors.append("visitorPolicy.factsOnly must be true: answers come from the business's own facts, never general knowledge about prices or health")
         if policy.get("cardDetails") != "never":
             errors.append("visitorPolicy.cardDetails must be never: the chat never asks for or accepts card numbers")
+    by_name = {str(t.get("name")): t for t in tools if isinstance(t, dict)}
+    check_clips(by_name, errors, warnings)
+    check_unattended(registry, by_name, servers, errors, warnings)
+    check_report_inbox(registry, by_name, errors)
     public_tools = [t.get("name") for t in tools if isinstance(t, dict) and any(servers.get(s, {}).get("auth") == "none" for s in t.get("mcpServers", []) or [])]
     return {"schemaVersion": "maggie-assistant-registry-validation.v1", "passed": not errors, "errors": errors, "warnings": warnings, "counts": {"tools": len(tools), **counts, "cards": sum(1 for t in tools if isinstance(t, dict) and t.get("card")), "publicMcpTools": len(public_tools)}}
 
 
+CLIP_WORDS_RE = re.compile(r"\b(?:clipped|shortened|truncated|cut to|first \d+ characters|~?\d+-character)\b", re.I)
+WRITE_RISKS = ("write-reversible", "write-spends", "write-public")
+
+
+def check_clips(by_name: dict[str, dict], errors: list[str], warnings: list[str]) -> None:
+    """A list tool that shortens long fields says so per item and names a get tool with the full text.
+
+    An agent checking what will be published must read the full text; a list that
+    silently clips makes it report "cut off mid-sentence" for text that is whole.
+    """
+    for name, tool in by_name.items():
+        clips = tool.get("clips")
+        if clips is None:
+            if risk_of(tool) == "read" and CLIP_WORDS_RE.search(str(tool.get("description", ""))):
+                warnings.append(f"{name}: the description says results are shortened; declare clips (fields, maxChars, getTool) so each item is marked and the full text has a get tool")
+            continue
+        if not isinstance(clips, dict):
+            errors.append(f"{name}: clips must be an object with fields, maxChars and getTool"); continue
+        if risk_of(tool) != "read":
+            errors.append(f"{name}: clips belongs to a read (list) tool")
+        fields = clips.get("fields")
+        if not isinstance(fields, list) or not fields or not all(isinstance(f, str) and f for f in fields):
+            errors.append(f"{name}: clips.fields must name the shortened fields")
+        if not isinstance(clips.get("maxChars"), int) or clips["maxChars"] < 1:
+            errors.append(f"{name}: clips.maxChars must be a positive integer")
+        get_name = str(clips.get("getTool", ""))
+        get_tool = by_name.get(get_name)
+        if get_tool is None:
+            errors.append(f"{name}: clips.getTool {get_name or '<missing>'!r} is not in the registry; a clipped list needs a get tool that returns the full text"); continue
+        if risk_of(get_tool) != "read":
+            errors.append(f"{name}: clips.getTool {get_name} must be a read tool")
+        if get_tool.get("scope") != tool.get("scope"):
+            errors.append(f"{name}: clips.getTool {get_name} must have the same scope as the list")
+        missing = [c for c in tool.get("channels") or [] if c not in (get_tool.get("channels") or [])]
+        if missing:
+            errors.append(f"{name}: clips.getTool {get_name} is not on {', '.join(missing)}; every caller of the list must be able to read the full text")
+
+
+def oauth_full_set(by_name: dict[str, dict], server: str | None) -> set[str]:
+    """The tools a signed-in caller's role can reach on MCP (the server filters by role at call time)."""
+    names = set()
+    for name, tool in by_name.items():
+        if risk_of(tool) == "never" or "mcp" not in (tool.get("channels") or []):
+            continue
+        if server and tool.get("mcpServers") and server not in tool["mcpServers"]:
+            continue
+        names.add(name)
+    return names
+
+
+def check_unattended(registry: dict, by_name: dict[str, dict], servers: dict, errors: list[str], warnings: list[str]) -> None:
+    """Limits for callers no person watches: agent keys and sign-in (OAuth) grants used by routines.
+
+    A prompt is not a limit. Every write such a caller can reach needs a daily cap
+    that the server counts; the OAuth default tool set is the role's whole set (a
+    narrower default needs a reason); an agent key cannot confirm previews.
+    """
+    policy = registry.get("unattended")
+    oauth_servers = [name for name, server in servers.items() if server.get("auth") == "oauth"]
+    if policy is None:
+        if oauth_servers and any(risk_of(by_name[n]) in WRITE_RISKS for n in oauth_full_set(by_name, oauth_servers[0])):
+            warnings.append("a signed-in MCP server exposes writes but registry.unattended is missing; declare server-enforced daily caps for agent keys and OAuth grants")
+        return
+    if not isinstance(policy, dict):
+        errors.append("unattended must be an object"); return
+    if policy.get("enforcedBy") != "server":
+        errors.append("unattended.enforcedBy must be server: caps kept only in a routine's prompt are not limits")
+    keys = policy.get("agentKeys")
+    if keys is not None:
+        if not isinstance(keys, dict):
+            errors.append("unattended.agentKeys must be an object")
+        else:
+            if keys.get("canConfirmPreviews") is not False:
+                errors.append("unattended.agentKeys.canConfirmPreviews must be false: an agent key runs only the writes its standing permission covers")
+            if keys.get("refusalReturnsReason") is not True:
+                errors.append("unattended.agentKeys.refusalReturnsReason must be true: a refused write says why (e.g. today's limit is used) instead of returning a preview nobody can confirm")
+            standing = keys.get("standingTools") if isinstance(keys.get("standingTools"), dict) else {}
+            for name, cap in standing.items():
+                tool = by_name.get(name)
+                if tool is None:
+                    errors.append(f"unattended.agentKeys.standingTools: {name} is not in the registry"); continue
+                if risk_of(tool) not in ("write-reversible", "write-spends"):
+                    errors.append(f"unattended.agentKeys.standingTools: {name} is {risk_of(tool)}; standing permissions cover reversible or spending writes only, never public or irreversible ones")
+                if not isinstance(cap, dict) or not isinstance(cap.get("perDay"), int) or cap["perDay"] < 1:
+                    errors.append(f"unattended.agentKeys.standingTools: {name} needs perDay, the daily cap the server counts")
+    oauth = policy.get("oauth")
+    if oauth is None:
+        if oauth_servers:
+            errors.append("unattended.oauth is required when a signed-in (oauth) MCP server is declared: a sign-in grant acts as the full person")
+        return
+    if not isinstance(oauth, dict):
+        errors.append("unattended.oauth must be an object"); return
+    server = oauth.get("server") or (oauth_servers[0] if oauth_servers else None)
+    if server and server not in servers:
+        errors.append(f"unattended.oauth.server {server!r} is not declared in registry.mcpServers")
+    full = oauth_full_set(by_name, server)
+    default = oauth.get("defaultTools", "role")
+    if default == "role":
+        reachable = full
+    elif isinstance(default, list):
+        unknown = sorted(set(map(str, default)) - full)
+        if unknown:
+            errors.append(f"unattended.oauth.defaultTools names tools the role can't reach on MCP: {', '.join(unknown)}")
+        reachable = set(map(str, default)) & full
+        left_out = sorted(full - reachable)
+        if left_out and not str(oauth.get("narrowedReason", "")).strip():
+            errors.append(f"unattended.oauth.defaultTools exposes {len(reachable)} of the role's {len(full)} tools (missing e.g. {', '.join(left_out[:5])}); a routine connected by sign-in can't reach the rest. Use \"role\" or give narrowedReason")
+    else:
+        errors.append("unattended.oauth.defaultTools must be \"role\" (the caller's role's full set) or a list of tools"); reachable = full
+    per_tool = oauth.get("perToolPerDay") if isinstance(oauth.get("perToolPerDay"), dict) else {}
+    total = oauth.get("dailyWriteLimit")
+    has_total = isinstance(total, int) and total >= 1
+    for name in sorted(reachable):
+        if risk_of(by_name[name]) not in WRITE_RISKS:
+            continue
+        cap = per_tool.get(name)
+        if not (isinstance(cap, int) and cap >= 1) and not has_total:
+            errors.append(f"unattended.oauth: {name} is a write reachable by a sign-in grant with no server cap; set perToolPerDay.{name} or dailyWriteLimit")
+
+
+REPORT_LIMITS = {"titleMaxChars": 120, "summaryMaxChars": 500, "bodyMaxChars": 50000}
+
+
+def check_report_inbox(registry: dict, by_name: dict[str, dict], errors: list[str]) -> None:
+    """One place where scheduled agents leave their results: save, list, get; capped; bodies untrusted."""
+    inbox = registry.get("reportInbox")
+    if inbox is None:
+        return
+    if not isinstance(inbox, dict):
+        errors.append("reportInbox must be an object"); return
+    names = inbox.get("tools") if isinstance(inbox.get("tools"), dict) else {}
+    expected = {"save": ("write-reversible",), "list": ("read",), "get": ("read",)}
+    for role, risks in expected.items():
+        name = str(names.get(role, ""))
+        tool = by_name.get(name)
+        if tool is None:
+            errors.append(f"reportInbox.tools.{role} {name or '<missing>'!r} is not in the registry"); continue
+        if risk_of(tool) not in risks:
+            errors.append(f"reportInbox.tools.{role} {name} must be {risks[0]} (saving a report changes nothing else)")
+        if tool.get("scope") != "workspace":
+            errors.append(f"reportInbox.tools.{role} {name}: reports belong to one website (scope workspace)")
+    if not isinstance(inbox.get("perSitePerDay"), int) or inbox["perSitePerDay"] < 1:
+        errors.append("reportInbox.perSitePerDay is required: the server caps agent reports per website per day")
+    for key, ceiling in REPORT_LIMITS.items():
+        value = inbox.get(key, ceiling)
+        if not isinstance(value, int) or value < 1:
+            errors.append(f"reportInbox.{key} must be a positive integer")
+    if not isinstance(inbox.get("retentionDays"), int) or inbox["retentionDays"] < 1:
+        errors.append("reportInbox.retentionDays is required")
+    untrusted = set(inbox.get("untrusted") or [])
+    missing = sorted({"title", "summary", "body"} - untrusted)
+    if missing:
+        errors.append(f"reportInbox.untrusted must include {', '.join(missing)}: an agent wrote them, so readers treat them as data, never instructions")
+
+
+def list_check(registry: dict, tool_name: str, result: dict) -> dict:
+    """Check a recorded list result: every item whose declared field was shortened says so."""
+    tool = next((t for t in registry.get("tools", []) if isinstance(t, dict) and t.get("name") == tool_name), None)
+    errors: list[str] = []
+    if tool is None or not isinstance(tool.get("clips"), dict):
+        return {"schemaVersion": "maggie-assistant-list-check.v1", "passed": False, "errors": [f"{tool_name}: not in the registry or declares no clips"], "items": 0, "clipped": 0}
+    clips = tool["clips"]
+    id_field = str(clips.get("idField", "id"))
+    max_chars = int(clips.get("maxChars", 0) or 0)
+    items = result.get("items") if isinstance(result.get("items"), list) else result.get("result", {}).get("items", []) if isinstance(result.get("result"), dict) else []
+    clipped = 0
+    for index, item in enumerate(items):
+        if not isinstance(item, dict):
+            continue
+        where = f"item {index} ({item.get(id_field, 'no ' + id_field)})"
+        if item.get(id_field) in (None, ""):
+            errors.append(f"{where}: has no {id_field}, so the full text can't be fetched with {clips.get('getTool')}")
+        marker = item.get("clipped")
+        for field in clips.get("fields", []):
+            text = item.get(field)
+            if not isinstance(text, str):
+                continue
+            shortened = (max_chars and len(text) >= max_chars) or text.rstrip().endswith(("…", "..."))
+            flagged = marker is True or (isinstance(marker, list) and field in marker)
+            if shortened and not flagged:
+                errors.append(f"{where}: {field} looks shortened ({len(text)} characters) but the item does not say clipped")
+            if flagged:
+                clipped += 1
+    return {"schemaVersion": "maggie-assistant-list-check.v1", "passed": not errors, "errors": errors, "items": len(items), "clipped": clipped, "getTool": clips.get("getTool")}
+
+
+def report_check(report: dict, inbox: dict | None = None) -> dict:
+    """Check a report an agent is about to save (maggie-assistant-report.v1)."""
+    limits = {**REPORT_LIMITS, **{k: v for k, v in (inbox or {}).items() if k in REPORT_LIMITS and isinstance(v, int)}}
+    errors: list[str] = []
+    title = str(report.get("title", "")).strip()
+    summary = str(report.get("summary", "")).strip()
+    body = str(report.get("body_md", report.get("body", "")))
+    if not title:
+        errors.append("title is required")
+    elif len(title) > limits["titleMaxChars"]:
+        errors.append(f"title is over {limits['titleMaxChars']} characters")
+    if not summary:
+        errors.append("summary is required: one or two sentences shown in the inbox")
+    elif len(summary) > limits["summaryMaxChars"]:
+        errors.append(f"summary is over {limits['summaryMaxChars']} characters")
+    if len(body) > limits["bodyMaxChars"]:
+        errors.append(f"body is over {limits['bodyMaxChars']} characters")
+    if re.search(r"(?i)\b(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9]{8,}|\bBearer\s+[A-Za-z0-9._-]{20,}|-----BEGIN [A-Z ]*PRIVATE KEY", title + summary + body):
+        errors.append("the report contains something that looks like a secret; remove it before saving")
+    return {"schemaVersion": "maggie-assistant-report-check.v1", "passed": not errors, "errors": errors, "chars": {"title": len(title), "summary": len(summary), "body": len(body)}}
+
+
 def annotations_for(tool: dict) -> dict:
     risk = risk_of(tool)
     title = str(tool.get("user", {}).get("label", "")).strip() or None
@@ -348,14 +565,21 @@ def main(argv: list[str] | None = None) -> int:
     page = sub.add_parser("capability-page"); page.add_argument("--registry", type=Path, required=True); page.add_argument("--title", default="What the assistant can and can't do"); page.add_argument("--output", type=Path, help="Markdown file; JSON is written next to it")
     audit = sub.add_parser("audit-check"); audit.add_argument("--registry", type=Path, required=True); audit.add_argument("--session", type=Path, required=True, help="recorded calls + audit entries (maggie-assistant-session.v1)")
     sub.add_parser("approval-selftest")
+    lc = sub.add_parser("list-check"); lc.add_argument("--registry", type=Path, required=True); lc.add_argument("--tool", required=True); lc.add_argument("--result", type=Path, required=True, help="a recorded result of the list tool")
+    rc = sub.add_parser("report-check"); rc.add_argument("--report", type=Path, required=True, help="the report about to be saved (title, summary, body_md)"); rc.add_argument("--registry", type=Path, help="use this registry's reportInbox limits")
     args = parser.parse_args(argv)
     try:
         if args.command == "approval-selftest":
             report = approval_selftest()
+        elif args.command == "report-check":
+            inbox = _read(args.registry).get("reportInbox") if args.registry else None
+            report = report_check(_read(args.report), inbox if isinstance(inbox, dict) else None)
         else:
             registry = _read(args.registry)
             if args.command == "registry-validate":
                 report = validate_registry(registry)
+            elif args.command == "list-check":
+                report = list_check(registry, args.tool, _read(args.result))
             elif args.command == "annotations":
                 report = derive_annotations(registry, _read(args.mcp_tools) if args.mcp_tools else None)
             elif args.command == "capability-page":