@pentatonic-ai/ai-agent-sdk 0.10.21 → 0.10.23

package/packages/memory-engine-v2/extractor-async/test_structured_edges.py

@@ -0,0 +1,471 @@
+"""Deterministic structured edges (SEE-490) — the builders that turn an event's
+structured envelope (gmail From/To/Cc, drive author/file) into person↔person and
+person↔doc edges WITHOUT the LLM.
+
+Pure-logic tests only (no DB): they exercise the envelope parsing, the canonical
+predicates, the precision guards (self-edge / machine-sender / fan-out cap), the
+hard-key convergence (email/file-id), and the flag/dispatch behaviour. The upsert
+SQL that consumes the dicts is covered by the engine image build+import check in
+CI (and is byte-compiled here by importing worker.py)."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+
+_THIS = Path(__file__).resolve().parent
+_SPEC = importlib.util.spec_from_file_location("extractor_async_worker",
+                                               _THIS / "worker.py")
+assert _SPEC and _SPEC.loader
+worker = importlib.util.module_from_spec(_SPEC)
+try:
+    _SPEC.loader.exec_module(worker)
+except ImportError as e:
+    pytest.skip(f"extractor-async deps unavailable: {e}", allow_module_level=True)
+
+
+ARENA = "pip-agents:abc"
+
+
+def _by_type(ents, etype):
+    return [e for e in ents if e["type"] == etype]
+
+
+def _edge_types(edges):
+    return sorted(e["type"] for e in edges)
+
+
+# ----------------------------------------------------------------------
+# _emails_in — envelope field normalisation
+# ----------------------------------------------------------------------
+
+def test_emails_in_json_list():
+    assert worker._emails_in(["A@X.com", "b@y.com"]) == ["a@x.com", "b@y.com"]
+
+
+def test_emails_in_delimited_string():
+    assert worker._emails_in("a@x.com, b@y.com; c@z.com") == [
+        "a@x.com", "b@y.com", "c@z.com",
+    ]
+
+
+def test_emails_in_name_angle_bracket_form():
+    assert worker._emails_in(["Jane Doe <jane@x.com>"]) == ["jane@x.com"]
+
+
+def test_emails_in_garbage_is_empty():
+    assert worker._emails_in(None) == []
+    assert worker._emails_in(42) == []
+    assert worker._emails_in(["not an email", ""]) == []
+
+
+# ----------------------------------------------------------------------
+# _is_human_address / _person_node — machine-sender guard + node shape
+# ----------------------------------------------------------------------
+
+@pytest.mark.parametrize("addr", [
+    "noreply@example.com", "no-reply@example.com", "mailer-daemon@x.com",
+    "notifications@github.com", "bounce@sendgrid.net", "noreply+tag@x.com",
+])
+def test_machine_senders_rejected(addr):
+    assert worker._is_human_address(addr) is False
+    name, ent = worker._person_node(addr)
+    assert (name, ent) == (None, None)
+
+
+@pytest.mark.parametrize("addr", [
+    "philip.mossop@pentatonic.com", "johann@pentatonic.com", "jane@acme.io",
+])
+def test_real_people_accepted(addr):
+    assert worker._is_human_address(addr) is True
+    name, ent = worker._person_node(addr)
+    assert ent["type"] == "person"
+    assert ent["aliases"] == [addr]
+
+
+def test_person_node_uses_display_name_but_keys_on_email():
+    name, ent = worker._person_node("Philip.Mossop@Pentatonic.com", name="Philip Mossop")
+    # canonical is the nice display name…
+    assert name == "Philip Mossop"
+    assert ent["name"] == "Philip Mossop"
+    # …but the email (lowercased) is the alias, so person_id_key keys on it.
+    assert ent["aliases"] == ["philip.mossop@pentatonic.com"]
+
+
+def test_person_node_falls_back_to_email_as_canonical():
+    name, ent = worker._person_node("recipient@acme.io")
+    assert name == "recipient@acme.io"
+    assert ent["name"] == "recipient@acme.io"
+
+
+# ----------------------------------------------------------------------
+# gmail builder — corresponded_with
+# ----------------------------------------------------------------------
+
+def _gmail_attrs(**over):
+    base = {
+        "source": "pip-gmail-ingest",
+        "contact_email": "philip.mossop@pentatonic.com",
+        "author": "Philip Mossop",
+        "to_emails": ["johann@pentatonic.com", "jane@acme.io"],
+        "cc_emails": ["cc@partner.com"],
+    }
+    base.update(over)
+    return base
+
+
+def test_gmail_sender_to_each_recipient():
+    ents, edges = worker._gmail_structured_edges(ARENA, _gmail_attrs())
+    # 3 recipients ⇒ 3 corresponded_with edges, all from the sender.
+    assert _edge_types(edges) == [worker.EDGE_CORRESPONDED] * 3
+    assert {e["from"] for e in edges} == {"Philip Mossop"}
+    assert {e["to"] for e in edges} == {
+        "johann@pentatonic.com", "jane@acme.io", "cc@partner.com",
+    }
+    # sender + 3 recipient person nodes.
+    assert len(_by_type(ents, "person")) == 4
+
+
+def test_gmail_edges_carry_structured_provenance():
+    _ents, edges = worker._gmail_structured_edges(ARENA, _gmail_attrs())
+    for e in edges:
+        assert e["attributes"] == {"derivation": "structured", "source": "gmail"}
+        assert e["confidence"] == 1.0
+
+
+def test_gmail_no_self_edge_when_sender_also_in_recipients():
+    attrs = _gmail_attrs(to_emails=["philip.mossop@pentatonic.com", "johann@pentatonic.com"])
+    _ents, edges = worker._gmail_structured_edges(ARENA, attrs)
+    assert all(e["to"] != "Philip Mossop" for e in edges)
+    assert {e["to"] for e in edges} == {"johann@pentatonic.com", "cc@partner.com"}
+
+
+def test_gmail_dedupes_repeated_recipient():
+    attrs = _gmail_attrs(to_emails=["johann@pentatonic.com", "johann@pentatonic.com"],
+                         cc_emails=[])
+    _ents, edges = worker._gmail_structured_edges(ARENA, attrs)
+    assert len(edges) == 1
+
+
+def test_gmail_skips_machine_recipient():
+    attrs = _gmail_attrs(to_emails=["noreply@x.com", "johann@pentatonic.com"], cc_emails=[])
+    _ents, edges = worker._gmail_structured_edges(ARENA, attrs)
+    assert {e["to"] for e in edges} == {"johann@pentatonic.com"}
+
+
+def test_gmail_no_sender_yields_nothing():
+    attrs = _gmail_attrs(contact_email="", from_email="")
+    assert worker._gmail_structured_edges(ARENA, attrs) == ([], [])
+
+
+def test_gmail_no_recipients_yields_nothing():
+    # A sender but no resolvable recipients ⇒ don't mint a lone dangling node.
+    attrs = _gmail_attrs(to_emails=[], cc_emails=[])
+    assert worker._gmail_structured_edges(ARENA, attrs) == ([], [])
+
+
+def test_gmail_fanout_cap(monkeypatch):
+    monkeypatch.setattr(worker, "STRUCTURED_EDGE_FANOUT_CAP", 5)
+    attrs = _gmail_attrs(to_emails=[f"user{i}@acme.io" for i in range(20)], cc_emails=[])
+    _ents, edges = worker._gmail_structured_edges(ARENA, attrs)
+    assert len(edges) == 5
+
+
+def test_gmail_endpoint_strings_match_minted_node_names():
+    # The edge from/to MUST equal a minted node's `name` — that's the key
+    # name_to_id resolves on at upsert. Guard against drift.
+    ents, edges = worker._gmail_structured_edges(ARENA, _gmail_attrs())
+    names = {e["name"] for e in ents}
+    for e in edges:
+        assert e["from"] in names
+        assert e["to"] in names
+
+
+# ----------------------------------------------------------------------
+# drive builder — authored
+# ----------------------------------------------------------------------
+
+def _drive_attrs(**over):
+    base = {
+        "source": "pip-drive-ingest",
+        "contact_email": "scott.royalty@thewholegroup.com",
+        "drive_file_id": "1AbCxyz",
+        "drive_file_name": "Q3 Plan.pdf",
+    }
+    base.update(over)
+    return base
+
+
+def test_drive_author_to_document():
+    ents, edges = worker._drive_structured_edges(ARENA, _drive_attrs())
+    assert _edge_types(edges) == [worker.EDGE_AUTHORED]
+    (edge,) = edges
+    assert edge["from"] == "scott.royalty@thewholegroup.com"
+    assert edge["to"] == "Q3 Plan.pdf"
+    assert edge["attributes"] == {"derivation": "structured", "source": "drive"}
+    docs = _by_type(ents, "document")
+    assert len(docs) == 1
+    # document keyed on the permanent drive file-id (hard key, not the title).
+    assert docs[0]["aliases"] == ["drive:1AbCxyz"]
+
+
+def test_drive_doc_name_falls_back_to_file_id():
+    ents, edges = worker._drive_structured_edges(ARENA, _drive_attrs(drive_file_name=""))
+    assert edges[0]["to"] == "1AbCxyz"
+    assert _by_type(ents, "document")[0]["name"] == "1AbCxyz"
+
+
+def test_drive_requires_author_and_file():
+    assert worker._drive_structured_edges(ARENA, _drive_attrs(contact_email="")) == ([], [])
+    assert worker._drive_structured_edges(ARENA, _drive_attrs(drive_file_id="")) == ([], [])
+
+
+# ----------------------------------------------------------------------
+# structured_graph_elements — dispatch, flag, robustness
+# ----------------------------------------------------------------------
+
+def test_dispatch_routes_by_source():
+    g_ents, g_edges = worker.structured_graph_elements(ARENA, _gmail_attrs())
+    assert g_edges and g_edges[0]["type"] == worker.EDGE_CORRESPONDED
+    d_ents, d_edges = worker.structured_graph_elements(ARENA, _drive_attrs())
+    assert d_edges and d_edges[0]["type"] == worker.EDGE_AUTHORED
+
+
+def test_unknown_source_is_noop():
+    assert worker.structured_graph_elements(ARENA, {"source": "pip-slack-ingest"}) == ([], [])
+    assert worker.structured_graph_elements(ARENA, {}) == ([], [])
+    assert worker.structured_graph_elements(ARENA, None) == ([], [])
+
+
+def test_flag_off_disables(monkeypatch):
+    monkeypatch.setattr(worker, "STRUCTURED_EDGES_ENABLED", False)
+    assert worker.structured_graph_elements(ARENA, _gmail_attrs()) == ([], [])
+
+
+def test_builder_exception_is_swallowed(monkeypatch):
+    def boom(arena, attrs):
+        raise ValueError("kaboom")
+    monkeypatch.setitem(worker._STRUCTURED_EDGE_BUILDERS, "pip-gmail-ingest", boom)
+    # must not propagate — the distill path can't be broken by a bad builder.
+    assert worker.structured_graph_elements(ARENA, _gmail_attrs()) == ([], [])
+
+
+# ----------------------------------------------------------------------
+# slack builder — member_of + mentioned
+# ----------------------------------------------------------------------
+
+def _slack_attrs(**over):
+    base = {
+        "source": "pip-slack-ingest",
+        "contact_email": "philip.mossop@pentatonic.com",
+        "author": "Philip | Pentatonic",
+        "channel_id": "C0BA2PCEQGN",
+        "channel_name": "seesa-engineering",
+        "mentions": ["alex.tong@pentatonic.com", "ben.gordon@pentatonic.com"],
+    }
+    base.update(over)
+    return base
+
+
+def test_slack_member_of_channel():
+    ents, edges = worker._slack_structured_edges(ARENA, _slack_attrs())
+    member = [e for e in edges if e["type"] == worker.EDGE_MEMBER_OF]
+    assert len(member) == 1
+    assert member[0]["from"] == "Philip | Pentatonic"
+    assert member[0]["to"] == "seesa-engineering"
+    # channel node keyed on the permanent channel-id.
+    chans = _by_type(ents, "channel")
+    assert chans[0]["aliases"] == ["slack:channel:C0BA2PCEQGN"]
+
+
+def test_slack_mentions_each_person():
+    _ents, edges = worker._slack_structured_edges(ARENA, _slack_attrs())
+    mentioned = sorted(e["to"] for e in edges if e["type"] == worker.EDGE_MENTIONED)
+    assert mentioned == ["alex.tong@pentatonic.com", "ben.gordon@pentatonic.com"]
+    assert all(e["from"] == "Philip | Pentatonic" for e in edges if e["type"] == worker.EDGE_MENTIONED)
+
+
+def test_slack_channel_name_falls_back_to_id():
+    _ents, edges = worker._slack_structured_edges(ARENA, _slack_attrs(channel_name=""))
+    member = [e for e in edges if e["type"] == worker.EDGE_MEMBER_OF]
+    assert member[0]["to"] == "C0BA2PCEQGN"
+
+
+def test_slack_no_author_email_yields_nothing():
+    assert worker._slack_structured_edges(ARENA, _slack_attrs(contact_email="")) == ([], [])
+
+
+def test_slack_channel_only_no_mentions():
+    ents, edges = worker._slack_structured_edges(ARENA, _slack_attrs(mentions=[]))
+    assert _edge_types(edges) == [worker.EDGE_MEMBER_OF]
+
+
+def test_slack_self_mention_dropped():
+    attrs = _slack_attrs(mentions=["philip.mossop@pentatonic.com"], channel_id="")
+    assert worker._slack_structured_edges(ARENA, attrs) == ([], [])
+
+
+def test_slack_provenance_stamp():
+    _ents, edges = worker._slack_structured_edges(ARENA, _slack_attrs())
+    for e in edges:
+        assert e["attributes"] == {"derivation": "structured", "source": "slack"}
+
+
+# ----------------------------------------------------------------------
+# granola builder — attended
+# ----------------------------------------------------------------------
+
+def _granola_attrs(**over):
+    base = {
+        "source": "pip-granola-ingest",
+        "thread_id": "bacf4ffc-cad9-4223-9513-8d02a6c01c2d",
+        "title": "TES Daily standup",
+        "attendees": ["philip.mossop@pentatonic.com", "phil.hauser@pentatonic.com"],
+    }
+    base.update(over)
+    return base
+
+
+def test_granola_attendees_to_meeting():
+    ents, edges = worker._granola_structured_edges(ARENA, _granola_attrs())
+    assert _edge_types(edges) == [worker.EDGE_ATTENDED, worker.EDGE_ATTENDED]
+    assert {e["to"] for e in edges} == {"TES Daily standup"}
+    assert {e["from"] for e in edges} == {
+        "philip.mossop@pentatonic.com", "phil.hauser@pentatonic.com",
+    }
+    meeting = _by_type(ents, "event")
+    assert len(meeting) == 1
+    # keyed on the granola doc id, NOT the (recurring) title.
+    assert meeting[0]["aliases"] == ["meeting:bacf4ffc-cad9-4223-9513-8d02a6c01c2d"]
+
+
+def test_granola_requires_meeting_id_and_attendees():
+    assert worker._granola_structured_edges(ARENA, _granola_attrs(thread_id="")) == ([], [])
+    assert worker._granola_structured_edges(ARENA, _granola_attrs(attendees=[])) == ([], [])
+
+
+def test_granola_dedupes_attendee():
+    attrs = _granola_attrs(attendees=["a@x.com", "a@x.com", "b@y.com"])
+    _ents, edges = worker._granola_structured_edges(ARENA, attrs)
+    assert len(edges) == 2
+
+
+# ----------------------------------------------------------------------
+# calendar builder — attended (keyed on source_id; reads attendees[])
+# ----------------------------------------------------------------------
+
+def _calendar_attrs(**over):
+    base = {
+        "source": "pip-calendar-ingest",
+        "source_id": "gcal-545npq4q",
+        "title": "Mastercard visits Pentatonic",
+        "attendees": ["philip@pentatonic.com", "yan@mastercard.com"],
+    }
+    base.update(over)
+    return base
+
+
+def test_calendar_attendees_to_meeting_keyed_on_source_id():
+    ents, edges = worker._calendar_structured_edges(ARENA, _calendar_attrs())
+    assert _edge_types(edges) == [worker.EDGE_ATTENDED, worker.EDGE_ATTENDED]
+    assert {e["to"] for e in edges} == {"Mastercard visits Pentatonic"}
+    meeting = _by_type(ents, "event")
+    assert meeting[0]["aliases"] == ["meeting:gcal-545npq4q"]
+    for e in edges:
+        assert e["attributes"] == {"derivation": "structured", "source": "calendar"}
+
+
+def test_calendar_noop_without_structured_attendees():
+    # un-upgraded events (pre-SEE-506) have no attendees[] → no edges.
+    assert worker._calendar_structured_edges(ARENA, _calendar_attrs(attendees=None)) == ([], [])
+    assert worker.structured_graph_elements(
+        ARENA, {"source": "pip-calendar-ingest", "source_id": "x"}
+    ) == ([], [])
+
+
+def test_calendar_dispatches():
+    _e, edges = worker.structured_graph_elements(ARENA, _calendar_attrs())
+    # attended for both attendees; plus employed_by for the corporate one
+    # (yan@mastercard.com — philip@pentatonic.com is the tenant, excluded).
+    assert worker.EDGE_ATTENDED in {e["type"] for e in edges}
+    eb = [e for e in edges if e["type"] == worker.EDGE_EMPLOYED_BY]
+    assert {e["to"] for e in eb} == {"mastercard"}
+
+
+# ----------------------------------------------------------------------
+# employed_by — person → org by email domain (derived post-pass)
+# ----------------------------------------------------------------------
+
+def _person(email, name=None):
+    return {"type": "person", "name": name or email, "aliases": [email]}
+
+
+def test_employed_by_corporate_only():
+    ents = [
+        _person("philip@pentatonic.com", "Philip"),  # tenant → excluded
+        _person("jane@acme.io"),                       # corporate → acme
+        _person("bob@gmail.com"),                      # freemail → excluded
+    ]
+    org_ents, edges = worker._employed_by_elements(ents, "gmail")
+    assert [o["name"] for o in org_ents] == ["acme"]
+    assert org_ents[0]["aliases"] == ["acme.io"]
+    assert len(edges) == 1
+    assert edges[0]["from"] == "jane@acme.io"
+    assert edges[0]["to"] == "acme"
+    assert edges[0]["type"] == worker.EDGE_EMPLOYED_BY
+    assert edges[0]["attributes"] == {"derivation": "structured", "source": "gmail"}
+
+
+def test_employed_by_dedupes_org_and_edge():
+    ents = [_person("a@acme.io"), _person("b@acme.io"), _person("a@acme.io")]
+    org_ents, edges = worker._employed_by_elements(ents, "gmail")
+    assert len(org_ents) == 1                 # one acme org node
+    assert sorted(e["from"] for e in edges) == ["a@acme.io", "b@acme.io"]
+
+
+def test_employed_by_ignores_non_person():
+    ents = [{"type": "document", "name": "x", "aliases": ["drive:1"]}]
+    assert worker._employed_by_elements(ents, "drive") == ([], [])
+
+
+def test_gmail_dispatch_adds_employed_by():
+    # Through structured_graph_elements: gmail recipients on corporate domains
+    # get employed_by edges alongside corresponded_with.
+    _ents, edges = worker.structured_graph_elements(ARENA, _gmail_attrs())
+    types = {e["type"] for e in edges}
+    assert worker.EDGE_CORRESPONDED in types
+    assert worker.EDGE_EMPLOYED_BY in types
+    # jane@acme.io and cc@partner.com are corporate; pentatonic is tenant.
+    eb = {e["to"] for e in edges if e["type"] == worker.EDGE_EMPLOYED_BY}
+    assert eb == {"acme", "partner"}
+
+
+def test_employed_by_flag_off(monkeypatch):
+    monkeypatch.setattr(worker, "STRUCTURED_EMPLOYED_BY_ENABLED", False)
+    _ents, edges = worker.structured_graph_elements(ARENA, _gmail_attrs())
+    assert all(e["type"] != worker.EDGE_EMPLOYED_BY for e in edges)
+    # corresponded_with still present.
+    assert any(e["type"] == worker.EDGE_CORRESPONDED for e in edges)
+
+
+# ----------------------------------------------------------------------
+# dispatch covers the P2 sources too
+# ----------------------------------------------------------------------
+
+def test_dispatch_routes_slack_and_granola():
+    _e, slack_edges = worker.structured_graph_elements(ARENA, _slack_attrs())
+    assert worker.EDGE_MEMBER_OF in {e["type"] for e in slack_edges}
+    _e, gran_edges = worker.structured_graph_elements(ARENA, _granola_attrs())
+    assert {e["type"] for e in gran_edges} == {worker.EDGE_ATTENDED}
+
+
+def test_container_endpoint_strings_match_minted_nodes():
+    for attrs in (_slack_attrs(), _granola_attrs()):
+        ents, edges = worker.structured_graph_elements(ARENA, attrs)
+        names = {e["name"] for e in ents}
+        for e in edges:
+            assert e["from"] in names
+            assert e["to"] in names