@pentatonic-ai/ai-agent-sdk 0.9.4 → 0.9.6

package/packages/memory/package-lock.json

@@ -1,20 +1,19 @@
 {
-  "name": "@pentatonic/memory",
-  "version": "0.1.0",
+  "name": "memory",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
-      "name": "@pentatonic/memory",
+      "name": "memory",
       "dependencies": {
         "@modelcontextprotocol/sdk": "^1.0.0",
         "pg": "^8.13.0"
       }
     },
     "node_modules/@hono/node-server": {
-      "version": "1.19.13",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz",
-      "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==",
+      "version": "1.19.14",
+      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.14.tgz",
+      "integrity": "sha512-GwtvgtXxnWsucXvbQXkRgqksiH2Qed37H9xHZocE5sA3N8O8O8/8FA3uclQXxXVzc9XBZuEOMK7+r02FmSpHtw==",
       "license": "MIT",
       "engines": {
         "node": ">=18.14.1"
@@ -77,9 +76,9 @@
       }
     },
     "node_modules/ajv": {
-      "version": "8.18.0",
-      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz",
-      "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==",
+      "version": "8.20.0",
+      "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
+      "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",
       "license": "MIT",
       "dependencies": {
         "fast-deep-equal": "^3.1.3",
@@ -355,9 +354,9 @@
       }
     },
     "node_modules/eventsource-parser": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz",
-      "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==",
+      "version": "3.0.8",
+      "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.8.tgz",
+      "integrity": "sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==",
       "license": "MIT",
       "engines": {
         "node": ">=18.0.0"
@@ -407,12 +406,12 @@
       }
     },
     "node_modules/express-rate-limit": {
-      "version": "8.3.2",
-      "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.3.2.tgz",
-      "integrity": "sha512-77VmFeJkO0/rvimEDuUC5H30oqUC4EyOhyGccfqoLebB0oiEYfM7nwPrsDsBL1gsTpwfzX8SFy2MT3TDyRq+bg==",
+      "version": "8.5.1",
+      "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-8.5.1.tgz",
+      "integrity": "sha512-5O6KYmyJEpuPJV5hNTXKbAHWRqrzyu+OI3vUnSd2kXFubIVpG7ezpgxQy76Zo5GQZtrQBg86hF+CM/NX+cioiQ==",
       "license": "MIT",
       "dependencies": {
-        "ip-address": "10.1.0"
+        "ip-address": "^10.2.0"
       },
       "engines": {
         "node": ">= 16"
@@ -556,9 +555,9 @@
       }
     },
     "node_modules/hasown": {
-      "version": "2.0.2",
-      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz",
-      "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==",
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz",
+      "integrity": "sha512-ej4AhfhfL2Q2zpMmLo7U1Uv9+PyhIZpgQLGT1F9miIGmiCJIoCgSmczFdrc97mWT4kVY72KA+WnnhJ5pghSvSg==",
       "license": "MIT",
       "dependencies": {
         "function-bind": "^1.1.2"
@@ -619,9 +618,9 @@
       "license": "ISC"
     },
     "node_modules/ip-address": {
-      "version": "10.1.0",
-      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.1.0.tgz",
-      "integrity": "sha512-XXADHxXmvT9+CRxhXg56LJovE+bmWnEWB78LB83VZTprKTmaC5QfruXocxzTZ2Kl0DNwKuBdlIhjL8LeY8Sf8Q==",
+      "version": "10.2.0",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.2.0.tgz",
+      "integrity": "sha512-/+S6j4E9AHvW9SWMSEY9Xfy66O5PWvVEJ08O0y5JGyEKQpojb0K0GKpz/v5HJ/G0vi3D2sjGK78119oXZeE0qA==",
       "license": "MIT",
       "engines": {
         "node": ">= 12"
@@ -649,9 +648,9 @@
       "license": "ISC"
     },
     "node_modules/jose": {
-      "version": "6.2.2",
-      "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.2.tgz",
-      "integrity": "sha512-d7kPDd34KO/YnzaDOlikGpOurfF0ByC2sEV4cANCtdqLlTfBlw2p14O/5d/zv40gJPbIQxfES3nSx1/oYNyuZQ==",
+      "version": "6.2.3",
+      "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz",
+      "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==",
       "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/panva"
@@ -1201,17 +1200,34 @@
       }
     },
     "node_modules/type-is": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz",
-      "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==",
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz",
+      "integrity": "sha512-faYHw0anBbc/kWF3zFTEnxSFOAGUX9GFbOBthvDdLsIlEoWOFOtS0zgCiQYwIskL9iGXZL3kAXD8OoZ4GmMATA==",
       "license": "MIT",
       "dependencies": {
-        "content-type": "^1.0.5",
+        "content-type": "^2.0.0",
         "media-typer": "^1.1.0",
         "mime-types": "^3.0.0"
       },
       "engines": {
-        "node": ">= 0.6"
+        "node": ">= 18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
+      }
+    },
+    "node_modules/type-is/node_modules/content-type": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz",
+      "integrity": "sha512-j/O/d7GcZCyNl7/hwZAb606rzqkyvaDctLmckbxLzHvFBzTJHuGEdodATcP3yIRoDrLHkIATJuvzbFlp/ki2cQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/express"
       }
     },
     "node_modules/unpipe": {
@@ -1263,9 +1279,9 @@
       }
     },
     "node_modules/zod": {
-      "version": "4.3.6",
-      "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz",
-      "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==",
+      "version": "4.4.3",
+      "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz",
+      "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==",
       "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"

package/packages/memory/package.json

@@ -1,10 +1,13 @@
 {
   "private": true,
   "name": "memory",
-  "description": "Memory subsystem — imported via @pentatonic-ai/ai-agent-sdk/memory",
+  "description": "Memory subsystem \u2014 imported via @pentatonic-ai/ai-agent-sdk/memory",
   "type": "module",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.0.0",
     "pg": "^8.13.0"
+  },
+  "overrides": {
+    "ip-address": "^10.1.1"
   }
 }

package/packages/memory/src/__tests__/engine.test.js

@@ -730,15 +730,22 @@ describe("engine HTTP client", () => {
   });
 
   describe("engineForget", () => {
-    it("forwards id when provided", async () => {
+    it("forwards id when provided (no arena composition for id-based deletes)", async () => {
       mockOk({ deleted: 1 });
       await engineForget("https://e", { clientId: "acme", id: "abc" });
       const body = JSON.parse(calls[0].init.body);
       expect(calls[0].url).toBe("https://e/forget");
-      expect(body).toEqual({ arena: "acme", id: "abc" });
+      // id-only deletes target the global record id; the engine's
+      // id path doesn't read arena scope, so we don't inject it.
+      expect(body).toEqual({ id: "abc" });
     });
 
-    it("forwards metadata_contains when provided", async () => {
+    it("forwards metadata_contains and injects arena INSIDE it (tenant default)", async () => {
+      // The engine reads `metadata_contains.arena` (not top-level
+      // arena) to scope a forget at L2. Pre-2026-05-14 this helper
+      // put arena at the top level, which the engine silently
+      // ignored — only L6 ever got wiped. Pinning the post-fix
+      // contract here so a regression can't sneak back in.
       mockOk({ deleted: 5 });
       await engineForget("https://e", {
         clientId: "acme",
@@ -746,11 +753,39 @@ describe("engine HTTP client", () => {
       });
       const body = JSON.parse(calls[0].init.body);
       expect(body).toEqual({
-        arena: "acme",
-        metadata_contains: { source_repo: "monorepo" },
+        metadata_contains: { arena: "acme", source_repo: "monorepo" },
+      });
+      // Top-level arena must NOT be sent — the engine ignores it and
+      // its presence would mislead anyone reading wire dumps.
+      expect(body.arena).toBeUndefined();
+    });
+
+    it("composes user-scoped arena when userId is supplied", async () => {
+      mockOk({ deleted: 12 });
+      await engineForget("https://e", {
+        clientId: "acme",
+        userId: "u-1",
+        metadataContains: { actor_user_id: "u-1" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body).toEqual({
+        metadata_contains: { arena: "acme:u-1", actor_user_id: "u-1" },
       });
     });
 
+    it("respects caller-supplied arena inside metadataContains (super-admin override)", async () => {
+      // Super-admin tooling that wipes "some other tenant's user arena"
+      // — pass the explicit arena and the SDK leaves it alone instead
+      // of recomposing from (clientId, userId).
+      mockOk({ deleted: 99 });
+      await engineForget("https://e", {
+        clientId: "tes-admin",
+        metadataContains: { arena: "victim-tenant:u-7", source: "x" },
+      });
+      const body = JSON.parse(calls[0].init.body);
+      expect(body.metadata_contains.arena).toBe("victim-tenant:u-7");
+    });
+
     it("requires id or metadataContains", async () => {
       await expect(
         engineForget("https://e", { clientId: "acme" })

package/packages/memory/src/engine.js

@@ -328,9 +328,31 @@ export async function engineSearch(engineUrl, opts) {
  *
  * Caller must supply exactly one of `id` or `metadataContains`.
  *
+ * Arena scope: the engine extracts the arena from `metadata_contains.arena`
+ * (see memory-engine `compat/server.py:1048-1052`). Top-level `arena` is
+ * NOT read by the engine — previous versions of this helper put it there
+ * and the resulting calls only ever wiped L6, leaving L0/L2/L3/L4 records
+ * untouched. The 2026-05-14 Pip dedup cutover surfaced the bug: an
+ * actor_user_id wipe returned 0 against an arena that personFacets
+ * confirmed held thousands of records. This helper now injects `arena`
+ * into `metadata_contains` so the engine forwards to L2 /forget-internal
+ * and actually wipes the cross-layer arena.
+ *
+ * By default the row is **user-scoped** (`arena = clientId:userId`) when
+ * `userId` is supplied, otherwise **tenant-wide** (`arena = clientId`).
+ * Pass `scope: "tenant"` explicitly to bypass the user-arena scope from a
+ * user-context. Matches `engineStore`'s arena semantics for symmetry.
+ *
+ * If the caller passes `arena` inside `metadataContains` themselves, the
+ * SDK respects it as-is and skips composition — useful for super-admin
+ * tools that need to wipe an arena other than the one derived from
+ * (clientId, userId).
+ *
  * @param {string} engineUrl
  * @param {object} opts
  * @param {string} opts.clientId
+ * @param {string} [opts.userId]            user id within the tenant; controls default scope
+ * @param {"tenant"|"user"} [opts.scope]    override the default scope. "user" requires userId.
  * @param {string} [opts.id]                forget a single record by engine id
  * @param {object} [opts.metadataContains]  forget all records matching every key=value pair
  * @param {Record<string,string>} [opts.headers]  forwarded HTTP headers
@@ -338,15 +360,28 @@ export async function engineSearch(engineUrl, opts) {
  * @returns {Promise<{deleted: number}>}
  */
 export async function engineForget(engineUrl, opts) {
-  const { clientId, id, metadataContains, headers } = opts || {};
+  const { clientId, userId, scope, id, metadataContains, headers } = opts || {};
   if (!clientId) throw new Error("engineForget: clientId required");
   if (!id && !metadataContains) {
     throw new Error("engineForget: provide id or metadataContains");
   }
+
+  // Compose arena from (clientId, userId, scope) using the same shape
+  // engineStore uses. Caller-supplied `metadataContains.arena` wins —
+  // the SDK shouldn't second-guess a super-admin explicitly targeting
+  // a specific arena.
+  let mergedMetadata;
+  if (metadataContains) {
+    const hasExplicitArena =
+      typeof metadataContains.arena === "string" && metadataContains.arena;
+    mergedMetadata = hasExplicitArena
+      ? metadataContains
+      : { ...metadataContains, arena: composeArena(clientId, userId, scope) };
+  }
+
   const body = {
-    arena: clientId,
     ...(id ? { id } : {}),
-    ...(metadataContains ? { metadata_contains: metadataContains } : {}),
+    ...(mergedMetadata ? { metadata_contains: mergedMetadata } : {}),
   };
   return fetchEngine(engineUrl, "/forget", body, { headers });
 }

package/packages/memory-engine/docker-compose.yml

@@ -72,7 +72,22 @@ services:
     environment:
       NEO4J_AUTH: ${NEO4J_AUTH:-neo4j/local-dev-pw}
       NEO4J_PLUGINS: '["apoc"]'
-      NEO4J_dbms_memory_heap_max__size: 512m
+      # Heap defaults were 512m hardcoded — fine for an empty dev
+      # graph, catastrophic at production scale. A 2026-05-14 prod
+      # incident on a ~10M-relationship KG saw L3 sit at >600% CPU
+      # locked in parallel GC, blocking the L2 write fan-out and
+      # triggering cascading 5xx through L6 and the embed gateway.
+      # The graph fit in RAM fine; the JVM just had nowhere to put
+      # short-lived allocations.
+      #
+      # Defaults now sized for a small-but-realistic local graph
+      # (~1M relationships): 1g heap + 256m initial + 512m pagecache.
+      # Production deployments override via PME_L3_HEAP_MAX etc.
+      # (the AWS overlay sets 4g/1g/1g — see thing-event-system
+      # modules/pentatonic-memory/deploy/docker-compose.aws.yml).
+      NEO4J_dbms_memory_heap_max__size: ${PME_L3_HEAP_MAX:-1g}
+      NEO4J_dbms_memory_heap_initial__size: ${PME_L3_HEAP_INITIAL:-256m}
+      NEO4J_dbms_memory_pagecache_size: ${PME_L3_PAGECACHE:-512m}
     volumes:
       - pme-l3-data:/data
     healthcheck:
@@ -220,7 +235,14 @@ services:
       interval: 10s
       timeout: 5s
       retries: 30
-      start_period: 60s
+      # 180s gives L2 enough time to finish Neo4j schema + index creation
+      # on a cold start before compat's healthcheck starts counting failures.
+      # Observed concretely on the v0.9.4 deploy (2026-05-14): L2 took
+      # ~90s to warm up; with start_period: 60s, compat went unhealthy
+      # mid-startup, cloudflared's `depends_on: condition: service_healthy`
+      # failed, and `docker compose up` errored out before wait_for_health
+      # could observe the eventual recovery.
+      start_period: 180s
 
 networks:
   engine-net:

package/packages/memory-engine/engine/services/_shared/embed_provider.py

@@ -212,6 +212,9 @@ class EmbedClient:
         timeout: float = 120.0,
         env_prefix: str = "",
         max_batch: int = 5,
+        max_retries: int = 3,
+        retry_base_delay: float = 0.1,
+        retry_max_delay: float = 1.0,
     ) -> None:
         self._configured_provider = provider
         self._provider = provider
@@ -229,6 +232,25 @@ class EmbedClient:
         # cap observed on Pentatonic AI Gateway — above which it 502s and the
         # caller silently loses vector writes (see test_chunking_* tests).
         self._max_batch = max(0, max_batch)
+        # Retry-with-jitter for transient gateway saturation. The
+        # Pentatonic AI Gateway has a K≈10 concurrent-request cap; when
+        # multiple chunks of a single batch (or multiple concurrent
+        # batches from different layers) saturate it, individual POSTs
+        # 502/503. The 2026-05-15 incident showed an L6 fallback path
+        # 502-rate of 96% under Pip backfill load — every shared-embed
+        # failed, every per-layer fallback also failed, the cascade
+        # cleared only when traffic dropped.
+        #
+        # Retries with full jitter let those transient saturations
+        # absorb instead of cascading: when many concurrent chunks all
+        # 502 at once, jittered backoff staggers their retries so the
+        # gateway recovers slot-by-slot rather than thundering-herding.
+        # Tuned via {prefix}EMBED_MAX_RETRIES (default 3); set to 0
+        # to restore pre-fix behaviour. Only 429/502/503/504 are
+        # retried — auth + 4xx errors fail fast.
+        self._max_retries = max(0, max_retries)
+        self._retry_base_delay = max(0.0, retry_base_delay)
+        self._retry_max_delay = max(self._retry_base_delay, retry_max_delay)
 
     # ------------------------------------------------------------------
     # Construction
@@ -268,6 +290,13 @@ class EmbedClient:
         autodetect = os.environ.get(f"{prefix}EMBED_AUTODETECT", "true").lower() == "true"
         timeout = float(os.environ.get(f"{prefix}EMBED_TIMEOUT", "120"))
         max_batch = int(os.environ.get(f"{prefix}EMBED_MAX_BATCH", "5"))
+        max_retries = int(os.environ.get(f"{prefix}EMBED_MAX_RETRIES", "3"))
+        retry_base_delay = float(
+            os.environ.get(f"{prefix}EMBED_RETRY_BASE_DELAY", "0.1")
+        )
+        retry_max_delay = float(
+            os.environ.get(f"{prefix}EMBED_RETRY_MAX_DELAY", "1.0")
+        )
 
         provider = resolve_provider(provider_name, env_prefix=prefix)
         return cls(
@@ -279,6 +308,9 @@ class EmbedClient:
             timeout=timeout,
             env_prefix=prefix,
             max_batch=max_batch,
+            max_retries=max_retries,
+            retry_base_delay=retry_base_delay,
+            retry_max_delay=retry_max_delay,
         )
 
     # ------------------------------------------------------------------
@@ -369,41 +401,103 @@ class EmbedClient:
     # Request paths
     # ------------------------------------------------------------------
 
+    # Status codes that indicate transient gateway capacity issues
+    # (rate-limit, upstream saturation, transient unavailability,
+    # upstream timeout). 401 + other 4xx + non-listed 5xx fail fast —
+    # they typically indicate caller or config problems where retrying
+    # won't help.
+    _RETRYABLE_STATUS = frozenset({429, 502, 503, 504})
+
+    def _backoff_delay(self, attempt: int) -> float:
+        """Exponential backoff with full jitter.
+
+        Full jitter (random.uniform(0, cap)) is preferred over equal
+        jitter for the embed gateway case: many concurrent chunks all
+        503 at the same instant, and full jitter maximally spreads
+        their retries so the gateway recovers slot-by-slot instead of
+        seeing periodic thundering herds.
+        """
+        import random
+        cap = min(self._retry_base_delay * (2 ** attempt), self._retry_max_delay)
+        return random.uniform(0, cap)
+
     def _post_with_autodetect(self, texts: list[str], *, async_mode: bool) -> list[list[float]]:
         del async_mode  # kept for symmetry; sync path is its own method
-        body = self._provider.body_builder(texts, self._model)
-        headers = self._headers(self._provider)
-        try:
-            r = httpx.post(self._url, json=body, headers=headers, timeout=self._timeout)
-        except httpx.HTTPError as exc:
-            raise EmbedHTTPError(0, str(exc)) from exc
-
-        if r.status_code == 401 and self._autodetect and not self._detected:
-            return self._autodetect_and_retry(texts, last_body=r.text)
-
-        if r.status_code == 401:
-            raise EmbedAuthError(r.text)
-        if not r.is_success:
-            raise EmbedHTTPError(r.status_code, r.text)
-        return self._provider.response_parser(r.json())
+        import time as _time
+        last_exc: EmbedHTTPError | None = None
+        for attempt in range(self._max_retries + 1):
+            body = self._provider.body_builder(texts, self._model)
+            headers = self._headers(self._provider)
+            try:
+                r = httpx.post(
+                    self._url, json=body, headers=headers, timeout=self._timeout
+                )
+            except httpx.HTTPError as exc:
+                # Network-level error (DNS, connect refused, timeout).
+                # Treat as retryable — transient network blips are
+                # exactly what jittered retry is designed to absorb.
+                last_exc = EmbedHTTPError(0, str(exc))
+                if attempt >= self._max_retries:
+                    raise last_exc from exc
+                _time.sleep(self._backoff_delay(attempt))
+                continue
+
+            if r.status_code == 401 and self._autodetect and not self._detected:
+                # Autodetect runs at most once (gated by self._detected)
+                # and tries other providers in sequence; no retry layer
+                # needed on top.
+                return self._autodetect_and_retry(texts, last_body=r.text)
+            if r.status_code == 401:
+                raise EmbedAuthError(r.text)
+            if not r.is_success:
+                if (
+                    r.status_code in self._RETRYABLE_STATUS
+                    and attempt < self._max_retries
+                ):
+                    last_exc = EmbedHTTPError(r.status_code, r.text)
+                    _time.sleep(self._backoff_delay(attempt))
+                    continue
+                raise EmbedHTTPError(r.status_code, r.text)
+            return self._provider.response_parser(r.json())
+
+        # Loop exited without success or raise — shouldn't happen, but
+        # keep the type checker happy.
+        assert last_exc is not None
+        raise last_exc
 
     async def _post_with_autodetect_async(self, texts: list[str]) -> list[list[float]]:
-        body = self._provider.body_builder(texts, self._model)
-        headers = self._headers(self._provider)
-        try:
-            async with httpx.AsyncClient(timeout=self._timeout) as client:
-                r = await client.post(self._url, json=body, headers=headers)
-        except httpx.HTTPError as exc:
-            raise EmbedHTTPError(0, str(exc)) from exc
-
-        if r.status_code == 401 and self._autodetect and not self._detected:
-            return await self._autodetect_and_retry_async(texts, last_body=r.text)
-
-        if r.status_code == 401:
-            raise EmbedAuthError(r.text)
-        if not r.is_success:
-            raise EmbedHTTPError(r.status_code, r.text)
-        return self._provider.response_parser(r.json())
+        import asyncio as _asyncio
+        last_exc: EmbedHTTPError | None = None
+        for attempt in range(self._max_retries + 1):
+            body = self._provider.body_builder(texts, self._model)
+            headers = self._headers(self._provider)
+            try:
+                async with httpx.AsyncClient(timeout=self._timeout) as client:
+                    r = await client.post(self._url, json=body, headers=headers)
+            except httpx.HTTPError as exc:
+                last_exc = EmbedHTTPError(0, str(exc))
+                if attempt >= self._max_retries:
+                    raise last_exc from exc
+                await _asyncio.sleep(self._backoff_delay(attempt))
+                continue
+
+            if r.status_code == 401 and self._autodetect and not self._detected:
+                return await self._autodetect_and_retry_async(texts, last_body=r.text)
+            if r.status_code == 401:
+                raise EmbedAuthError(r.text)
+            if not r.is_success:
+                if (
+                    r.status_code in self._RETRYABLE_STATUS
+                    and attempt < self._max_retries
+                ):
+                    last_exc = EmbedHTTPError(r.status_code, r.text)
+                    await _asyncio.sleep(self._backoff_delay(attempt))
+                    continue
+                raise EmbedHTTPError(r.status_code, r.text)
+            return self._provider.response_parser(r.json())
+
+        assert last_exc is not None
+        raise last_exc
 
     # ------------------------------------------------------------------
     # Auto-detect

package/packages/memory-engine/engine/services/l2/Dockerfile

@@ -9,9 +9,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 # Reranker = sentence-transformers MiniLM cross-encoder.
 # Torch CPU wheels are fine — reranker is small enough to be CPU-bound.
+#
+# sqlite-vec 0.1.9: native KNN over packed-f32 vectors stored in a vec0
+# virtual table. Replaces the legacy hand-rolled Python cosine loop over
+# JSON-serialised embeddings in search_qmd_informed (~15s timeout at 450k
+# rows → ~50ms native MATCH). Pin to 0.1.9 — that's the version probed
+# against L4 QMD's wire format (struct.pack f32 + cosine distance_metric).
 RUN pip install --no-cache-dir \
         fastapi "uvicorn[standard]" httpx requests pydantic \
         neo4j \
+        sqlite-vec==0.1.9 \
         "sentence-transformers" \
         "torch" --extra-index-url https://download.pytorch.org/whl/cpu