@pentatonic-ai/ai-agent-sdk 0.10.22 → 0.10.24

package/packages/memory-engine-v2/extractor-async/worker.py

@@ -1302,6 +1302,396 @@ def _email_plausibly_belongs(person_name: str, email: str) -> bool:
     return False
 
 
+# ----------------------------------------------------------------------
+# Deterministic structured edges (SEE-490)
+#
+# The participant envelope every event already carries — gmail From/To/Cc, drive
+# author/owner — encodes connections that are TRUE BY CONSTRUCTION: who emailed
+# whom, who authored what. Today those edges reach the graph ONLY if the LLM
+# re-derives them from prose (capped at MAX_RELATIONSHIPS_PER_EVENT, lossy, can
+# hallucinate), while the structured bag is consumed only for entity *resolution*
+# (`event_org_domains`, `event_record_subject`). Here we build the edges directly,
+# with no model call, keyed on the SAME hard keys the resolver uses (person → its
+# email via `person_id_key`; document → its drive file-id) so endpoints converge
+# onto the existing nodes instead of forking new ones. The async LLM pass still
+# runs and still emits the inferred/semantic edges only it can find — this just
+# adds the deterministic backbone it shouldn't have to guess at.
+#
+# `structured_graph_elements` returns (entities, relationships) in the EXACT dict
+# shape `upsert_entities` / `upsert_relationships` already consume, so the caller
+# just concatenates them onto the LLM output before the existing upserts. Pure +
+# total: never raises on bad input (it runs in the hot distill path) — it returns
+# empty lists for anything it can't parse, and a misbehaving builder is caught and
+# logged, never propagated.
+#
+# Closed predicate vocabulary — ONE canonical form per concept (not the LLM's
+# "works at"/"works for"/"works on" spread), so retrieval is a typed traversal
+# (`WHERE relationship_type = 'authored'`) rather than an OR over prose variants.
+# ----------------------------------------------------------------------
+
+EDGE_CORRESPONDED = "corresponded_with"  # email sender → each recipient (To+Cc)
+EDGE_AUTHORED = "authored"               # drive author/owner → document
+EDGE_MEMBER_OF = "member_of"             # slack author → channel
+EDGE_MENTIONED = "mentioned"             # slack author → @-mentioned person
+EDGE_ATTENDED = "attended"               # meeting attendee → meeting
+EDGE_EMPLOYED_BY = "employed_by"         # person → org (by their email domain)
+
+# Deterministic edges are flag-revertible: if a builder ever misfires on live
+# data, flip this off and the graph falls back to LLM-only edges. No data loss —
+# rows already written stay; only new structured edges stop being emitted.
+STRUCTURED_EDGES_ENABLED = _envflag("STRUCTURED_EDGES_ENABLED", "true")
+
+# `employed_by` (person → org by email domain) is gated SEPARATELY because it has
+# a known tradeoff the other builders don't: it mints an org node keyed on the
+# domain (the spine's deterministic org hard key), but when the LLM already
+# extracted that org under a name whose token doesn't contain the domain label
+# (accents/abbreviations: `loreal.com` vs "L'Oréal"), the two don't auto-merge via
+# alias-overlap and Fusion has to reconcile them. Correct-but-fragmenting, so it's
+# independently flippable pending a Fusion-reconcile decision. Freemail + the
+# tenant's own domain are always excluded (reusing `event_org_domains`' rules).
+STRUCTURED_EMPLOYED_BY_ENABLED = _envflag("STRUCTURED_EMPLOYED_BY_ENABLED", "true")
+
+# Defensive fan-out bound: a pathological event (a huge distribution list) must
+# not write thousands of edge rows. Real person-to-person mail sits far below
+# this; above it we log and truncate rather than let one event dominate the table.
+STRUCTURED_EDGE_FANOUT_CAP = int(os.environ.get("STRUCTURED_EDGE_FANOUT_CAP", "200"))
+
+# Stable resolution-key prefixes for "container" nodes (document / slack channel /
+# meeting) — mirrors the org domain / person email hard keys (keyed in
+# `upsert_entities` §3b). Each carries its source system's PERMANENT id as a
+# prefixed alias, so re-ingesting the same file/channel/meeting (even renamed)
+# converges to one node instead of fragmenting by title. Only the deterministic
+# edge pass mints these aliases, so there's no collision with LLM-extracted nodes.
+DRIVE_DOC_KEY_PREFIX = "drive:"
+SLACK_CHANNEL_KEY_PREFIX = "slack:channel:"
+MEETING_KEY_PREFIX = "meeting:"
+_EXTERNAL_KEY_PREFIXES = (DRIVE_DOC_KEY_PREFIX, SLACK_CHANNEL_KEY_PREFIX, MEETING_KEY_PREFIX)
+# Entity types whose identity is one of those external ids (when the alias is
+# present). `document`/`doc` = drive files; `channel` = slack channels; `event` =
+# meetings. LLM-minted nodes of these types carry no prefixed alias, so they keep
+# name-keying — this only changes the deterministically-minted ones.
+_EXTERNAL_KEY_TYPES = frozenset({"document", "doc", "channel", "event"})
+
+# Automated / no-reply local-parts — a "person" node for one is pure noise (a
+# mailer, not a human correspondent). Conservative: only the unambiguous machine
+# senders; a real human is never on this list.
+_NONHUMAN_LOCALPARTS = frozenset({
+    "noreply", "no-reply", "donotreply", "do-not-reply", "no_reply",
+    "mailer-daemon", "mailerdaemon", "postmaster", "bounce", "bounces",
+    "notifications", "notification", "notify", "mailer", "automated",
+})
+
+_ADDR_RE = re.compile(
+    r"[\w.+-]+@[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9-]+)+", re.I
+)
+
+
+def _emails_in(value: Any) -> list[str]:
+    """Lowercased bare email addresses found in a To/Cc/From field — accepts a
+    JSON list (the common case), a delimited string, or a `Name <addr>` form,
+    uniformly via regex. Order-preserving, no dedup (callers dedup). Never raises;
+    `[]` on anything non-stringy."""
+    if isinstance(value, list):
+        items = [v for v in value if isinstance(v, str)]
+    elif isinstance(value, str):
+        items = [value]
+    else:
+        return []
+    out: list[str] = []
+    for it in items:
+        out.extend(m.lower() for m in _ADDR_RE.findall(it))
+    return out
+
+
+def _is_human_address(email: str) -> bool:
+    """False for the unambiguous machine senders (`_NONHUMAN_LOCALPARTS`) — we
+    don't mint person nodes (or edges) for mailers/no-reply addresses."""
+    local = email.split("@", 1)[0].lower()
+    # strip the +tag so "noreply+123@" still matches "noreply"
+    local = local.split("+", 1)[0]
+    return local not in _NONHUMAN_LOCALPARTS
+
+
+def _person_node(email: str, *, name: str | None = None) -> tuple[str | None, dict | None]:
+    """A `(canonical_name, entity_dict)` for the person behind `email`, in the
+    shape `upsert_entities` consumes (`type`/`name`/`aliases`). The email rides in
+    `aliases` so `person_id_key` keys the node id on it — converging with the LLM
+    and sync passes for the same person. `canonical_name` is the display name when
+    we have one (gmail's `author`), else the email itself (same as the sync pass).
+    Returns `(None, None)` for a non-address or a machine sender."""
+    email = (email or "").strip().lower()
+    if "@" not in email or not _is_human_address(email):
+        return None, None
+    canonical = (name or "").strip() or email
+    return canonical, {"type": "person", "name": canonical, "aliases": [email]}
+
+
+def _stamp(source: str) -> dict[str, str]:
+    """Edge `attributes` marking a deterministic edge — makes "structured vs
+    LLM-inferred" a queryable distinction (the empty `{}` LLM edges carry today)."""
+    return {"derivation": "structured", "source": source}
+
+
+def _gmail_structured_edges(arena: str, attrs: dict[str, Any]) -> tuple[list[dict], list[dict]]:
+    """`corresponded_with`: the message sender (`contact_email`/`from_email`,
+    named by `author`) → each To+Cc recipient. Directed sender→recipient; the
+    sender's own address is dropped from the recipient set (no self-edge)."""
+    senders = _emails_in(attrs.get("contact_email")) or _emails_in(attrs.get("from_email"))
+    if not senders:
+        return [], []
+    sender = senders[0]
+    s_name, s_ent = _person_node(sender, name=attrs.get("author"))
+    if s_ent is None:
+        return [], []
+    recipients: list[str] = []
+    seen = {sender}
+    for key in ("to_emails", "cc_emails"):
+        for r in _emails_in(attrs.get(key)):
+            if r in seen:
+                continue
+            seen.add(r)
+            recipients.append(r)
+    if len(recipients) > STRUCTURED_EDGE_FANOUT_CAP:
+        log.info(
+            f"structured_edges: gmail event over fan-out cap "
+            f"({len(recipients)} > {STRUCTURED_EDGE_FANOUT_CAP}); truncating"
+        )
+        recipients = recipients[:STRUCTURED_EDGE_FANOUT_CAP]
+    ents: list[dict] = [s_ent]
+    edges: list[dict] = []
+    stamp = _stamp("gmail")
+    for r in recipients:
+        r_name, r_ent = _person_node(r)
+        if r_ent is None:
+            continue
+        ents.append(r_ent)
+        edges.append({
+            "from": s_name, "to": r_name, "type": EDGE_CORRESPONDED,
+            "confidence": 1.0, "attributes": stamp,
+        })
+    # No edges ⇒ no point minting the lone sender node here (the LLM/sync passes
+    # cover senders); only contribute nodes that anchor an edge.
+    return (ents, edges) if edges else ([], [])
+
+
+def _drive_structured_edges(arena: str, attrs: dict[str, Any]) -> tuple[list[dict], list[dict]]:
+    """`authored`: the drive file's owner/author (`contact_email`) → the document
+    node, keyed on the permanent drive file-id so the same file converges across
+    re-ingests and renames."""
+    authors = _emails_in(attrs.get("contact_email")) or _emails_in(attrs.get("author"))
+    file_id = (attrs.get("drive_file_id") or "").strip()
+    if not authors or not file_id:
+        return [], []
+    a_name, a_ent = _person_node(authors[0])
+    if a_ent is None:
+        return [], []
+    doc_name = (attrs.get("drive_file_name") or "").strip() or file_id
+    doc_key = DRIVE_DOC_KEY_PREFIX + file_id
+    doc_ent = {"type": "document", "name": doc_name, "aliases": [doc_key]}
+    edge = {
+        "from": a_name, "to": doc_name, "type": EDGE_AUTHORED,
+        "confidence": 1.0, "attributes": _stamp("drive"),
+    }
+    return [a_ent, doc_ent], [edge]
+
+
+def _container_node(etype: str, key_prefix: str, ext_id: str, title: str) -> tuple[str, dict]:
+    """A `(canonical_name, entity_dict)` for a container node (slack channel /
+    meeting) keyed on its source system's permanent id (via the prefixed alias →
+    `upsert_entities` §3b). `canonical_name` is the human title; the edge
+    references it (the name_to_id key), the alias makes it converge across
+    renames."""
+    canonical = (title or "").strip() or ext_id
+    return canonical, {
+        "type": etype, "name": canonical, "aliases": [key_prefix + ext_id],
+    }
+
+
+def _slack_structured_edges(arena: str, attrs: dict[str, Any]) -> tuple[list[dict], list[dict]]:
+    """`member_of`: the message author (`contact_email`, named by `author`) → the
+    channel. `mentioned`: the author → each @-mentioned person (`mentions` is a
+    list of emails). Author + mentioned people key on their email; the channel
+    keys on its permanent channel-id."""
+    authors = _emails_in(attrs.get("contact_email"))
+    if not authors:
+        return [], []
+    a_name, a_ent = _person_node(authors[0], name=attrs.get("author"))
+    if a_ent is None:
+        return [], []
+    ents: list[dict] = [a_ent]
+    edges: list[dict] = []
+    stamp = _stamp("slack")
+    # member_of → channel
+    channel_id = (attrs.get("channel_id") or "").strip()
+    if channel_id:
+        c_name, c_ent = _container_node(
+            "channel", SLACK_CHANNEL_KEY_PREFIX, channel_id, attrs.get("channel_name"),
+        )
+        ents.append(c_ent)
+        edges.append({
+            "from": a_name, "to": c_name, "type": EDGE_MEMBER_OF,
+            "confidence": 1.0, "attributes": stamp,
+        })
+    # mentioned → each distinct @-mentioned person (skip self-mentions)
+    seen = {authors[0]}
+    for m in _emails_in(attrs.get("mentions"))[:STRUCTURED_EDGE_FANOUT_CAP]:
+        if m in seen:
+            continue
+        seen.add(m)
+        m_name, m_ent = _person_node(m)
+        if m_ent is None:
+            continue
+        ents.append(m_ent)
+        edges.append({
+            "from": a_name, "to": m_name, "type": EDGE_MENTIONED,
+            "confidence": 1.0, "attributes": stamp,
+        })
+    return (ents, edges) if edges else ([], [])
+
+
+def _attended_edges(
+    meeting_id: str, title: Any, attendees_value: Any, source: str,
+) -> tuple[list[dict], list[dict]]:
+    """`attended`: each attendee email → the meeting node. Shared by the granola
+    and calendar builders (they differ only in which field carries the meeting's
+    permanent id). Attendees key on their email; the meeting on its source id, so
+    recurring same-title meetings stay distinct nodes (unlike the LLM's title
+    collapse). `attendees_value` is a list of emails (or a delimited string)."""
+    meeting_id = (meeting_id or "").strip()
+    attendees = _emails_in(attendees_value)
+    if not meeting_id or not attendees:
+        return [], []
+    m_name, m_ent = _container_node("event", MEETING_KEY_PREFIX, meeting_id, title)
+    ents: list[dict] = [m_ent]
+    edges: list[dict] = []
+    stamp = _stamp(source)
+    seen: set[str] = set()
+    for a in attendees[:STRUCTURED_EDGE_FANOUT_CAP]:
+        if a in seen:
+            continue
+        seen.add(a)
+        a_name, a_ent = _person_node(a)
+        if a_ent is None:
+            continue
+        ents.append(a_ent)
+        edges.append({
+            "from": a_name, "to": m_name, "type": EDGE_ATTENDED,
+            "confidence": 1.0, "attributes": stamp,
+        })
+    return (ents, edges) if edges else ([], [])
+
+
+def _granola_structured_edges(arena: str, attrs: dict[str, Any]) -> tuple[list[dict], list[dict]]:
+    """Granola meeting: attendees `attended` the meeting, keyed on the granola
+    doc id (`thread_id`)."""
+    return _attended_edges(
+        attrs.get("thread_id"), attrs.get("title"), attrs.get("attendees"), "granola",
+    )
+
+
+def _calendar_structured_edges(arena: str, attrs: dict[str, Any]) -> tuple[list[dict], list[dict]]:
+    """Calendar event: attendees `attended` the meeting, keyed on the gcal event
+    id (`source_id`; calendar events aren't threaded). Reads the structured
+    `attendees[]` array (SEE-506) — empty/absent on un-upgraded events, so this
+    is a no-op until the emitter stamps it."""
+    return _attended_edges(
+        attrs.get("source_id"), attrs.get("title"), attrs.get("attendees"), "calendar",
+    )
+
+
+# Dispatch on the fine producer label (`attributes.source`). Adding a source =
+# adding one entry + its builder; the wire-in and upsert path are untouched.
+# `pip-calendar-ingest` reads the structured `attendees[]` the emitter stamps
+# (SEE-506) — a no-op on events ingested before that ships.
+# Follow-ups (SEE-490 P3): hubspot associated_contact/at_company (needs the CRM
+# contact-id→email map — assoc_contact_ids are raw numeric ids, the contact nodes
+# are email-keyed); teams (emitter carries no participant envelope — only Entra
+# ids + display names, no emails; needs connector-side member resolution).
+_STRUCTURED_EDGE_BUILDERS = {
+    "pip-gmail-ingest": _gmail_structured_edges,
+    "pip-drive-ingest": _drive_structured_edges,
+    "pip-slack-ingest": _slack_structured_edges,
+    "pip-granola-ingest": _granola_structured_edges,
+    "pip-calendar-ingest": _calendar_structured_edges,
+}
+
+
+def _employed_by_elements(
+    person_ents: list[dict], source: str,
+) -> tuple[list[dict], list[dict]]:
+    """`employed_by`: for each PERSON node a builder already minted, link them to
+    the org of their email domain (the org's deterministic hard key). Scans the
+    builder's own person entities — so it works uniformly across every source
+    without threading anything through the builders. Drops freemail + the tenant's
+    own domain (an internal person isn't 'employed_by' an external org node, and
+    freemail isn't an org). The org node is keyed on the domain (canonical = its
+    registrable label); see `STRUCTURED_EMPLOYED_BY_ENABLED` for the merge caveat.
+    Deduped per (person, org) and per org node."""
+    org_by_domain: dict[str, str] = {}
+    org_ents: list[dict] = []
+    edges: list[dict] = []
+    seen_edge: set[tuple[str, str]] = set()
+    stamp = _stamp(source)
+    for e in person_ents:
+        if e.get("type") != "person":
+            continue
+        email = next((a for a in (e.get("aliases") or []) if "@" in a), None)
+        if not email:
+            continue
+        dom = _domain_of(email)
+        if not dom or dom in FREEMAIL_DOMAINS or dom in TENANT_EMAIL_DOMAINS:
+            continue
+        label = domain_label(dom)
+        if not label or len(label) < 2:
+            continue
+        if dom not in org_by_domain:
+            org_by_domain[dom] = label
+            org_ents.append({"type": "org", "name": label, "aliases": [dom]})
+        org_name = org_by_domain[dom]
+        person_name = e.get("name")
+        key = (person_name, org_name)
+        if not person_name or key in seen_edge:
+            continue
+        seen_edge.add(key)
+        edges.append({
+            "from": person_name, "to": org_name, "type": EDGE_EMPLOYED_BY,
+            "confidence": 1.0, "attributes": stamp,
+        })
+    return org_ents, edges
+
+
+def structured_graph_elements(
+    arena: str, event_attrs: dict[str, Any] | None
+) -> tuple[list[dict], list[dict]]:
+    """Deterministic `(entities, relationships)` from one event's structured
+    envelope — concatenated onto the LLM extraction before the upserts. Empty
+    when the flag is off, attributes are missing, the source has no builder, or
+    the builder finds nothing. Never raises (hot path): a builder exception is
+    logged and swallowed."""
+    if not STRUCTURED_EDGES_ENABLED or not event_attrs:
+        return [], []
+    source = (event_attrs.get("source") or "").strip()
+    builder = _STRUCTURED_EDGE_BUILDERS.get(source)
+    if builder is None:
+        return [], []
+    try:
+        ents, edges = builder(arena, event_attrs)
+        # employed_by is derived from the person nodes the builder produced —
+        # independently gated (it can fragment orgs pending Fusion) but on by
+        # default. Skipped entirely when a builder returned nothing.
+        if STRUCTURED_EMPLOYED_BY_ENABLED and ents:
+            org_ents, eb_edges = _employed_by_elements(ents, source)
+            ents = ents + org_ents
+            edges = edges + eb_edges
+        return ents, edges
+    except Exception as exc:  # noqa: BLE001 — must never break distillation
+        log.warning(
+            f"structured_edges: builder for {source!r} failed "
+            f"({type(exc).__name__}: {exc}); skipping"
+        )
+        return [], []
+
+
 def upsert_entities(
     conn: psycopg.Connection,
     arena: str,
@@ -1519,6 +1909,19 @@ def upsert_entities(
                 elif etype == "org":
                     id_key = org_node_id_key(etype, name, stamped_domain)
                     has_hard_key = stamped_domain is not None
+                elif etype in _EXTERNAL_KEY_TYPES:
+                    # Symmetric to org/person: a container node (drive document /
+                    # slack channel / meeting) carrying a prefixed external-id
+                    # alias (only the deterministic edge pass mints these) keys on
+                    # that permanent id, so the same file/channel/meeting converges
+                    # to one node across re-ingests/renames instead of fragmenting
+                    # by title. Title-keyed otherwise (LLM-extracted nodes).
+                    ext_key = next(
+                        (a for a in aliases if a.startswith(_EXTERNAL_KEY_PREFIXES)),
+                        None,
+                    )
+                    id_key = ext_key or name
+                    has_hard_key = ext_key is not None
                 else:
                     id_key = name
                     has_hard_key = False
@@ -1762,21 +2165,27 @@ def upsert_relationships(
             if not from_id or not to_id or not rtype:
                 continue
             rid = _content_id(arena, from_id, to_id, rtype)
+            # Edge `attributes` — `{derivation,source}` for deterministic edges
+            # (SEE-490), `{}` for LLM edges (unchanged from before). Makes
+            # "structured vs inferred" queryable; merge-only on conflict so a
+            # later corroboration never clears the provenance stamp.
+            edge_attrs = r.get("attributes") if isinstance(r.get("attributes"), dict) else {}
             cur.execute(
                 """
                 INSERT INTO relationships (
                   id, arena, from_entity_id, to_entity_id, relationship_type,
                   weight, provenance_event_ids, participant_set, disclosure_class,
-                  first_seen, last_seen
+                  attributes, first_seen, last_seen
                 ) VALUES (
                   %s, %s, %s, %s, %s, %s, %s, %s, %s::disclosure_class,
-                  COALESCE(%s, NOW()), COALESCE(%s, NOW())
+                  %s::jsonb, COALESCE(%s, NOW()), COALESCE(%s, NOW())
                 )
                 ON CONFLICT (id) DO UPDATE SET
                   weight = relationships.weight + EXCLUDED.weight,
                   provenance_event_ids = (
                     SELECT ARRAY(SELECT DISTINCT UNNEST(relationships.provenance_event_ids || EXCLUDED.provenance_event_ids))
                   ),
+                  attributes = relationships.attributes || EXCLUDED.attributes,
                   last_seen = GREATEST(relationships.last_seen, EXCLUDED.last_seen),
                   first_seen = LEAST(relationships.first_seen, EXCLUDED.first_seen)
                 """,
@@ -1784,6 +2193,7 @@ def upsert_relationships(
                     rid, arena, from_id, to_id, rtype,
                     float(r.get("confidence") or 0.5),
                     [event_id], participant_set, disclosure_class,
+                    json.dumps(edge_attrs),
                     event_time, event_time,
                 ),
             )
@@ -2364,9 +2774,18 @@ def _apply_extraction(
     # `attributes.source` else coarse `source_kind`; None ⇒ column stays NULL.
     src = fact_source(event)
 
+    # Deterministic structured edges (SEE-490): build person↔person / person↔doc /
+    # person↔channel / person↔meeting edges (+ their endpoint nodes) directly from
+    # the event's structured envelope and concatenate them onto the LLM output.
+    # Endpoints key on the same hard keys (`person_id_key` email / drive file-id /
+    # slack channel-id / meeting-id) as the resolver, so they merge onto existing
+    # nodes; the extra entities land in `name_to_id` for the edges to resolve
+    # against. Empty unless the source has a builder + the flag is on.
+    det_ents, det_rels = structured_graph_elements(arena, event.get("attributes"))
+
     try:
         name_to_id = upsert_entities(
-            conn, arena, event_id, participant_set, disclosure, ents,
+            conn, arena, event_id, participant_set, disclosure, ents + det_ents,
             event_time, event.get("attributes"),
         )
         n_facts = upsert_facts(
@@ -2374,13 +2793,14 @@ def _apply_extraction(
             event_time, due_at, src,
         )
         n_rels = upsert_relationships(
-            conn, arena, event_id, participant_set, disclosure, rels, name_to_id,
+            conn, arena, event_id, participant_set, disclosure, rels + det_rels, name_to_id,
             event_time,
         )
         mark_done(conn, queue_id)
         log.info(
             f"completed queue_id={queue_id} event_id={event_id} producer={producer} "
             f"entities={len(name_to_id)} facts={n_facts} relationships={n_rels}"
+            f" det_edges={len(det_rels)}"
             + (f" llm_ms={llm_ms:.0f}" if not stub_mode and llm_ms else "")
         )
         # Trace logging — best-effort. ONLY the teacher produces training gold;