@topy-ai/maggie 0.7.53 → 0.7.55

package/bundled-tools/clis/maggie_social_share.py

@@ -5,9 +5,11 @@ from __future__ import annotations
 import argparse
 import json
 import sys
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
+from urllib.error import HTTPError, URLError
 from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+from urllib.request import Request, urlopen
 
 sys.path.insert(0, str(Path(__file__).resolve().parent))
 from maggie_social_queue import check_campaigns, connection_health, parse_instant, plan_queue, reconcile, validate_adapters  # noqa: E402
@@ -43,6 +45,66 @@ def tracked_url(url: str, channel: str) -> str:
     return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment))
 
 
+# Only a definite "gone" answer drops an article. Blocked (403/429), failing
+# (5xx) or unreachable pages are kept: a firewall or an outage must never empty
+# the list of articles to share.
+GONE = {404, 410}
+
+
+def fetch_status(url: str, timeout: float = 10.0) -> int | None:
+    """Final HTTP status after redirects, or None when the page could not be reached."""
+    for method in ("HEAD", "GET"):
+        request = Request(url, method=method, headers={"User-Agent": "MaggieSocialShare/1.0 (+link check)", "Accept": "text/html,*/*"})
+        try:
+            with urlopen(request, timeout=timeout) as response:
+                return int(response.status)
+        except HTTPError as error:
+            # Some servers refuse HEAD; ask again with GET before deciding.
+            if method == "HEAD" and error.code in {400, 403, 405, 501}:
+                continue
+            return int(error.code)
+        except (URLError, TimeoutError, OSError):
+            return None
+    return None
+
+
+def link_verdict(status: int | None) -> str:
+    if status in GONE:
+        return "gone"
+    if status is not None and 200 <= status < 400:
+        return "live"
+    return "unknown"
+
+
+def check_links(urls: list[str], cache_path: Path | None, ttl_hours: float, now: datetime, fetch=fetch_status) -> dict:
+    cache: dict = {}
+    if cache_path and cache_path.is_file():
+        try:
+            cache = json.loads(cache_path.read_text(encoding="utf-8"))
+        except (OSError, json.JSONDecodeError):
+            cache = {}
+    results = []
+    for url in dict.fromkeys(urls):
+        cached = cache.get(url)
+        fresh = False
+        if isinstance(cached, dict) and cached.get("checked_at"):
+            try:
+                fresh = (now - parse_instant(cached["checked_at"], "checked_at")).total_seconds() < ttl_hours * 3600
+            except ValueError:
+                fresh = False
+        if fresh:
+            status, source = cached.get("status"), "cache"
+        else:
+            status, source = fetch(url), "fetched"
+            cache[url] = {"status": status, "checked_at": now.isoformat().replace("+00:00", "Z")}
+        results.append({"url": url, "status": status, "verdict": link_verdict(status), "source": source})
+    if cache_path:
+        cache_path.parent.mkdir(parents=True, exist_ok=True)
+        cache_path.write_text(json.dumps(cache, indent=2) + "\n", encoding="utf-8")
+    gone = [item for item in results if item["verdict"] == "gone"]
+    return {"valid": True, "checked": len(results), "skip": gone, "keep": [item for item in results if item["verdict"] != "gone"], "results": results}
+
+
 def plan(content: dict, channels: list[str], workspace_domain: str | None = None) -> dict:
     errors = [f"missing {key}" for key in REQUIRED if not content.get(key)]
     invalid = [channel for channel in channels if channel not in LIMITS]
@@ -73,6 +135,130 @@ def plan(content: dict, channels: list[str], workspace_domain: str | None = None
     }
 
 
+def shared_picture_errors(posts: list[dict], label: str) -> list[str]:
+    """Posts written together for several channels share one picture.
+
+    A channel may differ only when a person picked its picture
+    (`image_source: "user"`)."""
+    automatic = [post for post in posts if post.get("image_source") != "user"]
+    pictures = {post.get("image_url") or None for post in automatic}
+    if len(pictures) <= 1:
+        return []
+    by_picture: dict[str, list[str]] = {}
+    for post in automatic:
+        by_picture.setdefault(post.get("image_url") or "(none)", []).append(str(post.get("channel") or post.get("name") or post.get("id")))
+    detail = "; ".join(f"{', '.join(names)}: {picture}" for picture, names in by_picture.items())
+    return [f"{label}: channels written together must share one picture ({detail}); set image_source user on a channel whose picture a person chose"]
+
+
+def picture_groups(value: dict) -> dict:
+    """Check that drafts written from one idea share one picture across channels."""
+    drafts = value.get("drafts", []) if isinstance(value, dict) else value
+    groups: dict[str, list[dict]] = {}
+    for draft in drafts:
+        key = draft.get("idea_id") or draft.get("group_id")
+        if key:
+            groups.setdefault(str(key), []).append(draft)
+    errors: list[str] = []
+    report = []
+    for key, items in sorted(groups.items()):
+        problems = shared_picture_errors(items, f"idea {key}")
+        errors.extend(problems)
+        automatic = [item.get("image_url") for item in items if item.get("image_source") != "user"]
+        report.append({"idea_id": key, "channels": [item.get("channel") for item in items], "picture": automatic[0] if automatic and not problems else None, "shared": not problems})
+    return {"valid": not errors, "errors": errors, "ideas": report}
+
+
+REUSE_DAYS = 30
+SHARED_STATUSES = {"approved", "scheduled", "publishing", "published"}
+OPEN_STATUSES = {"draft", "queued", "held", "needs_review", "pending"}
+
+
+def article_key(item: dict) -> str:
+    """The article a post shares: its source id, else its address without tracking parameters."""
+    if item.get("source_content_id"):
+        return "id:" + str(item["source_content_id"])
+    url = str(item.get("canonical_url") or item.get("url") or "")
+    parts = urlsplit(url)
+    query = urlencode([(key, value) for key, value in parse_qsl(parts.query) if not key.startswith("utm_")])
+    return "url:" + urlunsplit((parts.scheme, parts.netloc.lower().removeprefix("www."), parts.path.rstrip("/"), query, ""))
+
+
+def shared_on(history: list[dict], now: datetime, days: int = REUSE_DAYS) -> dict[str, dict[str, str]]:
+    """Article key -> {channel: last shared instant} within the rolling window."""
+    used: dict[str, dict[str, str]] = {}
+    for post in history:
+        if post.get("status", "published") not in SHARED_STATUSES:
+            continue
+        stamp = post.get("posted_at") or post.get("published_at") or post.get("scheduled_at")
+        if not stamp or not post.get("channel"):
+            continue
+        when = parse_instant(stamp, "history posted_at")
+        if (now - when).total_seconds() > days * 86400 or when > now + timedelta(days=days):
+            continue
+        channels = used.setdefault(article_key(post), {})
+        channel = str(post["channel"])
+        channels[channel] = max(channels.get(channel, ""), stamp)
+    return used
+
+
+def sources_used_on(document: dict, now: datetime) -> dict:
+    """Each article with the channels it was shared to in the last 30 days (rolling)."""
+    used = shared_on(document.get("history") or [], now, int(document.get("reuse_days") or REUSE_DAYS))
+    articles = []
+    for article in document.get("articles") or []:
+        channels = used.get(article_key(article), {})
+        articles.append({**article, "used_on": sorted(channels), "last_shared": channels})
+    return {"valid": True, "reuse_days": int(document.get("reuse_days") or REUSE_DAYS), "articles": articles}
+
+
+def queue_check(document: dict, now: datetime) -> dict:
+    """What an automatic run and "Approve all" may do with the open posts.
+
+    * reuse is per channel: an article shared on a channel in the last 30 days
+      (rolling) is not shared there again — other channels are free;
+    * every new post carries a picture;
+    * in auto mode a hold raised only by optional checks does not block a post;
+      a failing required check always does;
+    * "Approve all" covers every open post, held ones included.
+    """
+    mode = str(document.get("approval_mode") or "manual")
+    if mode not in {"auto", "manual"}:
+        raise ValueError("approval_mode must be auto or manual")
+    days = int(document.get("reuse_days") or REUSE_DAYS)
+    used = shared_on(document.get("history") or [], now, days)
+    planned: dict[tuple[str, str], str] = {}
+    posts, approve_all, blocked = [], [], []
+    for post in document.get("posts") or []:
+        if post.get("status", "draft") not in OPEN_STATUSES:
+            continue
+        pid, channel, key = str(post.get("id")), str(post.get("channel") or ""), article_key(post)
+        reasons, notes = [], []
+        last = used.get(key, {}).get(channel)
+        if last:
+            reasons.append(f"this article was shared on {channel} at {last}, within {days} days")
+        elif (channel, key) in planned:
+            reasons.append(f"post {planned[(channel, key)]} already shares this article on {channel}")
+        planned.setdefault((channel, key), pid)
+        if not post.get("image_url"):
+            reasons.append("needs a picture: every new post carries one (use the picture of its idea)")
+        for hold in post.get("holds") or []:
+            name = str(hold.get("check") or hold.get("reason") or "check")
+            if hold.get("required"):
+                reasons.append(f"required check failed: {name}")
+            elif mode == "auto":
+                notes.append(f"optional check {name} does not hold the post in auto mode")
+            else:
+                notes.append(f"held for review by optional check {name}; Approve all includes it")
+        held = bool(post.get("holds")) or post.get("status") in {"held", "needs_review"}
+        entry = {"id": pid, "channel": channel, "held": held, "blocked": bool(reasons), "reasons": reasons, "notes": notes}
+        if mode == "auto":
+            entry["auto_publish"] = not reasons
+        posts.append(entry)
+        (blocked if reasons else approve_all).append(pid)
+    return {"valid": not blocked, "approval_mode": mode, "reuse_days": days, "approve_all": approve_all, "blocked": blocked, "posts": posts}
+
+
 def validate(value: dict) -> list[str]:
     errors = [f"missing {key}" for key in REQUIRED if not value.get(key)]
     if value.get("status") not in {"draft", "approved", "scheduled", "publishing", "published", "failed", "cancelled"}:
@@ -84,7 +270,10 @@ def validate(value: dict) -> list[str]:
         errors.append(f"canonical_url is on {host_of(value['canonical_url'])}, not the workspace domain {domain}")
     elif not domain and value.get("status") in LIVE_STATUSES:
         errors.append("workspace_domain is required before approval: the source must be published on the workspace's own domain")
-    for channel in value.get("channels", []):
+    channels = value.get("channels", [])
+    if value.get("image_url"):
+        errors.extend(shared_picture_errors([{**channel, "image_url": channel.get("image_url", value["image_url"])} for channel in channels], "draft"))
+    for channel in channels:
         name = channel.get("name")
         if name not in LIMITS:
             errors.append(f"unsupported channel: {name}")
@@ -114,6 +303,19 @@ def main() -> int:
     create.add_argument("--channels", default="linkedin,facebook")
     create.add_argument("--workspace-domain", help="the workspace's own site domain; the source must be published there")
     create.add_argument("--output")
+    create.add_argument("--check-link", action="store_true", help="fetch canonical_url first and refuse a definite 404/410")
+    pictures = sub.add_parser("pictures", help="check that drafts written from one idea share one picture on every channel")
+    pictures.add_argument("input", help='JSON: {"drafts": [{"id", "idea_id", "channel", "image_url", "image_source"?}]}')
+    links = sub.add_parser("link-check", help="drop articles whose link is gone (404/410) before planning posts; keep blocked or failing pages")
+    links.add_argument("input", help="JSON: a list of URLs or article objects with canonical_url, or {\"articles\": [...]}")
+    links.add_argument("--cache", default=".maggie/social-link-cache.json", help="verdict cache (default: .maggie/social-link-cache.json); 'none' disables it")
+    links.add_argument("--ttl-hours", type=float, default=6.0)
+    used_on = sub.add_parser("used-on", help="list, per article, the channels it was shared to in the last 30 days (rolling)")
+    used_on.add_argument("input", help='JSON: {"articles": [...], "history": [{"channel", "source_content_id"|"canonical_url", "posted_at", "status"}]}')
+    used_on.add_argument("--now")
+    qcheck = sub.add_parser("queue-check", help="decide what auto mode and Approve all may do: per-channel 30-day reuse, a picture on every post, optional holds")
+    qcheck.add_argument("input", help='JSON: {"approval_mode": "auto"|"manual", "posts": [...], "history": [...]}')
+    qcheck.add_argument("--now")
     check = sub.add_parser("validate", help="validate a draft channel contract")
     check.add_argument("input")
     adapters = sub.add_parser("adapters", help="check canonical OAuth scope sets and shared OAuth clients")
@@ -134,6 +336,15 @@ def main() -> int:
     schema.add_argument("input")
     args = parser.parse_args()
     try:
+        if args.command == "link-check":
+            raw = json.loads(Path(args.input).read_text(encoding="utf-8"))
+            articles = raw.get("articles", []) if isinstance(raw, dict) else raw
+            urls = [item if isinstance(item, str) else item.get("canonical_url") or item.get("url") for item in articles]
+            result = check_links([url for url in urls if url], None if args.cache == "none" else Path(args.cache), args.ttl_hours, datetime.now(timezone.utc))
+            by_url = {item["url"]: item for item in result["results"]}
+            result["articles"] = [item for item in articles if by_url.get(item if isinstance(item, str) else (item.get("canonical_url") or item.get("url")), {}).get("verdict") != "gone"]
+            print(json.dumps(result, indent=2, ensure_ascii=False))
+            return 0
         value = read(Path(args.input))
         now = datetime.now(timezone.utc)
         if args.command == "schema-check":
@@ -143,6 +354,12 @@ def main() -> int:
             result = validate_adapters(value)
         elif args.command == "connections":
             result = connection_health(value, parse_instant(args.now, "--now") if args.now else now)
+        elif args.command == "used-on":
+            result = sources_used_on(value, parse_instant(args.now, "--now") if args.now else now)
+        elif args.command == "queue-check":
+            result = queue_check(value, parse_instant(args.now, "--now") if args.now else now)
+        elif args.command == "pictures":
+            result = picture_groups(value if isinstance(value, dict) else {"drafts": value})
         elif args.command == "reconcile":
             result = reconcile(value)
         elif args.command == "campaigns":
@@ -156,6 +373,10 @@ def main() -> int:
             return 0
         else:
             if args.command == "plan":
+                if args.check_link and value.get("canonical_url"):
+                    status = fetch_status(value["canonical_url"])
+                    if link_verdict(status) == "gone":
+                        raise ValueError(f"canonical_url answers HTTP {status}: the article is gone; pick a live article (blocked or failing pages are not treated as gone)")
                 value = plan(value, [item.strip() for item in args.channels.split(",") if item.strip()], args.workspace_domain)
                 if args.output:
                     Path(args.output).parent.mkdir(parents=True, exist_ok=True)

package/bundled-tools/clis/maggie_version_sources.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+"""Check that every version source of a package agrees, and that generated data is current.
+
+A packager that writes its own version constant over a hand-bumped manifest
+(or the reverse) ships one version in one place and another elsewhere; a site
+built from generated data that was not regenerated then fails to deploy. This
+check never writes a version: it reads each source and refuses on a mismatch.
+
+Sources:
+  FILE             a plain VERSION file (first line)
+  FILE:dotted.key  a JSON value, e.g. package.json:version or manifest.json:meta.version
+  FILE~NAME        a code constant NAME = "x.y.z" / NAME: "x.y.z" / export const NAME = 'x.y.z'
+
+Generated data:
+  --generated OUT --command "node scripts/gen.mjs {out}"
+  runs the generator into a temporary file ({out}) and fails when OUT differs.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def read_source(root: Path, spec: str) -> str:
+    if "~" in spec:
+        name, constant = spec.split("~", 1)
+        text = (root / name).read_text(encoding="utf-8")
+        match = re.search(rf"\b{re.escape(constant)}\b\s*[:=]\s*['\"]([^'\"]+)['\"]", text)
+        if not match:
+            raise ValueError(f"{spec}: constant {constant} not found")
+        return match.group(1)
+    if ":" in spec and not re.match(r"^[A-Za-z]:[\\/]", spec):
+        name, dotted = spec.split(":", 1)
+        value: object = json.loads((root / name).read_text(encoding="utf-8"))
+        for key in dotted.split("."):
+            if not isinstance(value, dict) or key not in value:
+                raise ValueError(f"{spec}: key {dotted} not found")
+            value = value[key]
+        return str(value)
+    return (root / spec).read_text(encoding="utf-8").splitlines()[0].strip()
+
+
+def check_generated(root: Path, output: str, command: str) -> dict:
+    with tempfile.TemporaryDirectory() as directory:
+        fresh = Path(directory) / Path(output).name
+        argv = [part.replace("{out}", str(fresh)) for part in shlex.split(command)]
+        completed = subprocess.run(argv, cwd=root, capture_output=True, text=True, check=False)
+        if completed.returncode:
+            return {"file": output, "current": False, "error": f"generator exited {completed.returncode}: {(completed.stderr or completed.stdout).strip()[-300:]}"}
+        if not fresh.is_file():
+            return {"file": output, "current": False, "error": "generator did not write {out}"}
+        current = (root / output).is_file() and (root / output).read_bytes() == fresh.read_bytes()
+        return {"file": output, "current": current, **({} if current else {"error": "out of date: regenerate and commit it before deploying"})}
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0], formatter_class=argparse.RawDescriptionHelpFormatter, epilog="\n".join(__doc__.splitlines()[2:]))
+    parser.add_argument("command", choices=["check"])
+    parser.add_argument("--project", default=".")
+    parser.add_argument("--source", action="append", default=[], required=True, help="a version source (repeat; the first is the one to follow)")
+    parser.add_argument("--generated", action="append", default=[], help="generated file that must be current")
+    parser.add_argument("--generator", action="append", default=[], help="command that writes the matching --generated file to {out}")
+    args = parser.parse_args()
+    root = Path(args.project).resolve()
+    errors: list[str] = []
+    versions: dict[str, str | None] = {}
+    for spec in args.source:
+        try:
+            versions[spec] = read_source(root, spec)
+        except (OSError, ValueError, json.JSONDecodeError, IndexError) as error:
+            versions[spec] = None
+            errors.append(str(error))
+    found = {value for value in versions.values() if value}
+    if len(found) > 1:
+        first = versions[args.source[0]]
+        errors.extend(f"{spec} is {value}; {args.source[0]} is {first}" for spec, value in versions.items() if value and value != first)
+    if len(args.generated) != len(args.generator):
+        errors.append("give one --generator per --generated file")
+    generated = [check_generated(root, output, command) for output, command in zip(args.generated, args.generator)]
+    errors.extend(f"{item['file']}: {item['error']}" for item in generated if not item["current"])
+    result = {"valid": not errors, "version": versions.get(args.source[0]) if not errors else None, "sources": versions, "generated": generated, "errors": errors}
+    print(json.dumps(result, indent=2))
+    return 0 if result["valid"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/bundled-tools/integrations/maggie-api-pull.md

@@ -74,6 +74,55 @@ The import is draft-only and never publishes. It rejects missing IDs, invalid
 slugs, and empty content; changed source checksums stop for manual conflict
 review. Approval and publication are separate MaggieDash transitions.
 
+## Where a delivered post went live
+
+A pulled post is useful to other channels (social posts that link it,
+internal links) only once its live URL is known; without it a planner falls
+back to an older stored URL, which after a platform move answers 404. So:
+
+1. When the site publishes a delivered post, it reports the URL it went live
+   at (`POST /content-tracking/report-state` with the content ID, the URL and
+   `published`) and records it locally:
+   `maggie blog live-urls record --content-id ID --url URL`. The site's report
+   is authoritative.
+2. A scheduled sitemap match fills the gaps:
+   `maggie blog live-urls match --sitemap https://example.com/sitemap.xml --write --report-body .maggie/live-urls-report.json`,
+   then send that body with `maggie api report-state --body-file … --execute`.
+
+Every post without a live URL is a candidate, including one already tracked
+with an empty value; when the record has no slug, the last path segment of its
+delivered address is used. Matching is **additive**: a post that already has a live URL keeps it, and a
+post this run did not find is left as it was — never marked unmatched — so the
+match can run daily without retiring earlier matches. When translated copies
+share the slug (`/blog/x`, `/zh-hans/blog/x`), the URL **without a locale
+segment** wins. The same two rules apply to the platform's own sitemap
+matching for API Pull destinations.
+
+## Rewrites on pull sites
+
+A pull site receives rewrites the same way it receives posts: `GET /updates`
+on the same schedule as `/posts` (one rewrite per call, charged like a post;
+the answer is `{delivered, updates: [...]}`). Without this path a rewrite
+requested for a pulled post never reaches the site. Each rewrite is applied to
+the post it is for, never stored as a new post:
+
+1. Find the post by `source_content_id` (the id `/posts` delivered it under),
+   else by its slug or address, previous slugs included.
+2. Replace only the body. Title, slug, status, topics and the live URL stay as
+   the site has them.
+3. Keep the replaced body (and its checksum) so the rewrite can be undone.
+4. A rewrite with `publication_status: "draft"` is held for review, and one
+   with no matching post here is kept as `unmatched`; neither is applied.
+5. Translations of a rewritten post are stale: queue them again.
+6. Report the outcome with `POST /content-tracking/report-state`.
+
+`maggie api lifecycle --execute --allow-quota --pull-updates` does steps 1, 2,
+4 locally: the rewrite lands on the original record in
+`.maggie/ai-cmo-lifecycle.json` (`rewrite_pending`, a `rewrites` history with
+the previous checksum), held ones are listed without changing the body, and
+unmatched ones go to `rewrites_unmatched`. A later `/posts` re-delivery keeps
+the record's live URL and rewrite history.
+
 ## Project context synchronization
 
 `GET /project-context` returns sanitized, read-only context scoped by the API

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@topy-ai/maggie",
-  "version": "0.7.53",
+  "version": "0.7.55",
   "description": "Install and manage Maggie Skills for AI coding agents",
   "license": "MIT",
   "type": "module",