@pentatonic-ai/ai-agent-sdk 0.10.21 → 0.10.23

package/packages/memory-engine-v2/scripts/backfill_person_email_attr.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""Backfill person `attributes.email` + `attributes.domain` from existing alias
+emails (SEE-192). DRY-RUN by default.
+
+WHY: upsert_entities now stamps a person node's own email/domain as a queryable
+attribute at extract time (worker.py / server.py, SDK >=0.10.22), but the
+~38k EXISTING person nodes minted before that carry the email only inside the
+`aliases` array (0 had `attributes.email`). This one-shot populates them so
+retrieval + context-aware mention resolution (SEE-346 Layer B Phase 2) can match
+on a discrete email/domain without waiting for natural re-distill.
+
+ADDITIVE + NON-DESTRUCTIVE: only ever does `attributes = attributes || jsonb`
+(merge, never clobbers an existing key), only on rows that LACK `attributes.email`
+(idempotent), never deletes/merges/tombstones a node. So unlike fusion_defrag it
+needs no snapshot gate — but it's still dry-run-first and arena-scoped.
+
+THE §A GUARD (don't stamp a bystander/fabricated email onto a person):
+  1. `_email_plausibly_belongs` (verbatim from worker.py #128) — the email's
+     local-part must contain a name token or match the person's initials. Drops
+     the over-merge pollution (a hotel/newsletter/stranger's gmail folded onto
+     the node).
+  2. Placeholder/fabricated domains (example.com, test, localhost, …) are
+     dropped even if the local-part matches the name — `katrin.bauer@example.com`
+     (0 source events, extractor-invented) passes #128 but must NOT be stamped.
+  3. AMBIGUITY: if a node has >=2 plausible, non-placeholder emails spanning
+     DIFFERENT domains, SKIP it (leave to the resolution pass) — never guess
+     which is the person's real one.
+
+Usage (run on the engine box / anywhere with org_model reach):
+    python backfill_person_email_attr.py --arena 'pentatonic-team:%'            # dry-run report
+    python backfill_person_email_attr.py --arena 'pentatonic-team:%' --json out.json
+    python backfill_person_email_attr.py --arena 'pentatonic-team:%' --apply     # write
+  PG_DSN env or --pg-dsn. --arena is REQUIRED (never defaults to all arenas;
+  pip-agents is frozen legacy — scope deliberately).
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+
+
+def _connect(dsn: str):
+    import psycopg
+    import psycopg.rows
+
+    return psycopg.connect(dsn, row_factory=psycopg.rows.dict_row)
+
+
+# --- helpers REPLICATED from extractor-async/worker.py (per the build-context
+# split, scripts can't import the extractor cleanly — keep these BYTE-FAITHFUL
+# to worker.py `_email_plausibly_belongs` / `_domain_of`; a drift here re-opens
+# the bystander-pollution the #128 guard closes). ---------------------------
+_ALIAS_NONALPHA = re.compile(r"[^a-z]")
+_ALIAS_SPLIT = re.compile(r"[^a-z]+")
+_EMAIL_DOMAIN_RE = re.compile(r"@([A-Za-z0-9.-]+\.[A-Za-z]{2,})")
+
+# Placeholder / non-routable domains an extractor may have invented by templating
+# `firstname.lastname@<org-in-context>` — these have ~0 real provenance and must
+# never be stamped as a person's email (SEE-341 §A fabrication mode).
+_PLACEHOLDER_DOMAINS = {
+    "example.com", "example.org", "example.net", "example.edu",
+    "test.com", "test", "localhost", "invalid", "none.com", "email.com",
+    "domain.com", "company.com", "yourcompany.com", "acme.com",
+}
+
+
+def _email_plausibly_belongs(person_name: str, email: str) -> bool:
+    """VERBATIM from worker.py #128. True ⇒ the email plausibly belongs to
+    person_name (name token in local-part, or local-part is the initials)."""
+    local = email.split("@", 1)[0].lower()
+    local_letters = _ALIAS_NONALPHA.sub("", local)
+    name_tokens = {t for t in _ALIAS_SPLIT.split(person_name.lower()) if len(t) >= 2}
+    if not name_tokens or not local_letters:
+        return True  # nothing to check against — don't over-filter
+    if any(nt in local_letters for nt in name_tokens):
+        return True
+    initials = "".join(t[0] for t in person_name.lower().split() if t[:1].isalpha())
+    if len(initials) >= 2 and local_letters in (initials, initials[::-1]):
+        return True
+    return False
+
+
+def _domain_of(value: str) -> str | None:
+    """Lowercased domain of an email in `value`, else None. (worker.py)."""
+    if not value or "@" not in value:
+        return None
+    m = _EMAIL_DOMAIN_RE.search(value)
+    return m.group(1).lower() if m else None
+
+
+def _is_bare_email(a: str) -> bool:
+    return bool(a) and "@" in a and " " not in a
+
+
+def pick_person_email(
+    canonical_name: str, aliases: list[str]
+) -> tuple[str | None, str | None, str]:
+    """PURE. Choose the email/domain to stamp on a person from its aliases.
+
+    Returns `(email_lower, domain, reason)`. `email` is None when nothing safe to
+    stamp; `reason` is one of: stamped | none (no email alias) | no_plausible
+    (emails exist but none pass #128) | placeholder (only placeholder/fabricated)
+    | ambiguous (>=2 distinct real domains — leave to resolution).
+    """
+    emails = [a.strip().lower() for a in (aliases or []) if _is_bare_email(a)]
+    if not emails:
+        return None, None, "none"
+    # de-dupe, keep order
+    seen: set[str] = set()
+    emails = [e for e in emails if not (e in seen or seen.add(e))]
+    plausible = [e for e in emails if _email_plausibly_belongs(canonical_name, e)]
+    if not plausible:
+        return None, None, "no_plausible"
+    real = [e for e in plausible if (_domain_of(e) or "") not in _PLACEHOLDER_DOMAINS]
+    if not real:
+        return None, None, "placeholder"
+    domains = {_domain_of(e) for e in real}
+    if len(domains) >= 2:
+        return None, None, "ambiguous"
+    # one real domain (1+ addresses on it) → stamp the shortest local-part as the
+    # canonical address (e.g. johann@ over johann.boedecker@), its domain.
+    email = sorted(real, key=lambda e: (len(e), e))[0]
+    return email, _domain_of(email), "stamped"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument("--arena", required=True, help="arena LIKE filter (REQUIRED)")
+    ap.add_argument("--pg-dsn", default="", help="Postgres DSN (or PG_DSN env)")
+    ap.add_argument("--json", help="write the per-row plan + summary as JSON")
+    ap.add_argument("--limit", type=int, default=0, help="cap rows scanned (0 = all)")
+    ap.add_argument(
+        "--apply",
+        action="store_true",
+        help="WRITE the attributes (default: dry-run report only)",
+    )
+    args = ap.parse_args()
+
+    dsn = args.pg_dsn or os.environ.get("PG_DSN", "")
+    if not dsn:
+        print("FATAL: no --pg-dsn / PG_DSN", file=sys.stderr)
+        return 2
+
+    counts = {"stamped": 0, "none": 0, "no_plausible": 0, "placeholder": 0, "ambiguous": 0}
+    samples: list[dict] = []
+    plan: list[tuple[str, str, str]] = []  # (id, email, domain) to write
+
+    with _connect(dsn) as conn:
+        with conn.cursor() as cur:
+            if not args.apply:
+                cur.execute("SET default_transaction_read_only = on")  # dry-run safety
+            sql = (
+                "SELECT id, canonical_name, aliases FROM entities "
+                "WHERE arena LIKE %s AND entity_type = 'person' "
+                "AND NOT (attributes ? 'email')"
+            )
+            if args.limit:
+                sql += f" LIMIT {int(args.limit)}"
+            cur.execute(sql, (args.arena,))
+            rows = cur.fetchall()
+            print(f"scanning {len(rows)} person nodes lacking attributes.email "
+                  f"(arena LIKE {args.arena!r})")
+
+            for r in rows:
+                email, domain, reason = pick_person_email(
+                    r["canonical_name"] or "", r["aliases"] or []
+                )
+                counts[reason] += 1
+                if reason == "stamped" and email:
+                    plan.append((r["id"], email, domain or ""))
+                    if len(samples) < 15:
+                        samples.append({
+                            "name": r["canonical_name"], "email": email, "domain": domain,
+                        })
+
+            if args.apply and plan:
+                for eid, email, domain in plan:
+                    patch = {"email": email}
+                    if domain:
+                        patch["domain"] = domain
+                    cur.execute(
+                        "UPDATE entities SET attributes = attributes || %s::jsonb "
+                        "WHERE id = %s AND NOT (attributes ? 'email')",
+                        (json.dumps(patch), eid),
+                    )
+                conn.commit()
+            else:
+                conn.rollback()
+
+    total = sum(counts.values())
+    print("\n=== backfill summary ===")
+    print(f"  total scanned : {total}")
+    for k in ("stamped", "ambiguous", "placeholder", "no_plausible", "none"):
+        print(f"  {k:13}: {counts[k]}")
+    print("\n  sample stamps:")
+    for s in samples:
+        print(f"    {s['name']!r:40} -> {s['email']}  ({s['domain']})")
+    print(f"\n{'APPLIED' if args.apply else 'DRY-RUN (no writes; rerun with --apply)'}: "
+          f"{counts['stamped']} nodes "
+          f"{'updated' if args.apply else 'would be stamped'}.")
+
+    if args.json:
+        with open(args.json, "w") as f:
+            json.dump({"counts": counts, "samples": samples,
+                       "applied": bool(args.apply)}, f, indent=2)
+        print(f"  wrote {args.json}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/packages/memory-engine-v2/scripts/test_backfill_person_email_attr.py

@@ -0,0 +1,84 @@
+"""Guards `pick_person_email` (SEE-192 backfill) against stamping the WRONG
+email on a person. Existing nodes carry polluted aliases (the over-merge), so the
+backfill must reuse the #128 plausibility guard AND additionally reject
+placeholder/fabricated domains and genuine ambiguity — pure-logic tests, no DB."""
+
+from __future__ import annotations
+
+import importlib
+import os
+import sys
+
+HERE = os.path.dirname(__file__)
+
+
+def _load():
+    sys.path.insert(0, HERE)
+    sys.modules.pop("backfill_person_email_attr", None)
+    return importlib.import_module("backfill_person_email_attr")
+
+
+bk = _load()
+pick = bk.pick_person_email
+
+
+def test_stamps_the_persons_own_email_drops_bystanders() -> None:
+    # the live Johann case: own email + a hotel + a stranger gmail co-folded.
+    assert pick(
+        "Johann Boedecker",
+        ["Johann Boedecker", "johann@pentatonic.com", "reservations.nyc@acehotel.com"],
+    ) == ("johann@pentatonic.com", "pentatonic.com", "stamped")
+
+
+def test_bystander_only_node_is_not_stamped() -> None:
+    # over-merge pollution with NO plausible own-email → stamp nothing.
+    assert pick(
+        "Johann Boedecker",
+        ["reservations.nyc@acehotel.com", "martinvasquez87@gmail.com"],
+    ) == (None, None, "no_plausible")
+
+
+def test_fabricated_placeholder_domain_rejected() -> None:
+    # katrin.bauer@example.com passes #128 (name in local-part) but is the
+    # extractor-invented placeholder (0 source events) — must NOT be stamped.
+    assert pick("Katrin Bauer", ["Katrin Bauer", "katrin.bauer@example.com"]) == (
+        None,
+        None,
+        "placeholder",
+    )
+
+
+def test_two_distinct_real_domains_is_ambiguous_skip() -> None:
+    assert pick(
+        "Katie Cooper",
+        ["katie.cooper@sosv.com", "katie.cooper@booking.com"],
+    ) == (None, None, "ambiguous")
+
+
+def test_no_email_alias() -> None:
+    assert pick("Will Vickers", ["Will Vickers", "William F. Vickers"]) == (
+        None,
+        None,
+        "none",
+    )
+
+
+def test_prefers_shortest_local_on_one_domain() -> None:
+    assert pick(
+        "Johann Boedecker",
+        ["johann.boedecker@pentatonic.com", "johann@pentatonic.com"],
+    ) == ("johann@pentatonic.com", "pentatonic.com", "stamped")
+
+
+def test_initials_local_part() -> None:
+    assert pick("Johann Boedecker", ["jb@pentatonic.com"]) == (
+        "jb@pentatonic.com",
+        "pentatonic.com",
+        "stamped",
+    )
+
+
+def test_display_name_form_is_not_a_bare_address() -> None:
+    # graph aliases carry the bare email; a "X <a@b>" form (has a space) is not
+    # treated as the address — matches person_id_key selection semantics.
+    assert pick("Bob Smith", ["Bob Smith <bob@lego.com>"]) == (None, None, "none")