@pentatonic-ai/ai-agent-sdk 0.5.10 → 0.5.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.5.10",
+  "version": "0.5.11",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory/src/__tests__/api-contract.test.js

@@ -587,23 +587,50 @@ describe("ingest options contract", () => {
 describe("ingest dedup option", () => {
   function makeMockDb(state = {}) {
     const calls = [];
-    const existing = state.existing || []; // [{ id, client_id, content }, ...]
+    // Each existing row optionally carries created_at (Date). Defaults to
+    // "now" so legacy-window tests that don't care still match.
+    const existing = state.existing || [];
     const inserted = [];
+    const now = state.now || new Date();
     const db = async (sql, params) => {
       calls.push({ sql, params });
       if (sql.includes("SELECT id FROM memory_layers")) {
         return { rows: [{ id: "layer-1" }] };
       }
-      // Dedup pre-check (raw + LIKE legacy form)
-      if (sql.includes("SELECT id FROM memory_nodes")) {
-        const [clientId, content] = params;
-        const match = existing.find(
-          (r) =>
-            r.client_id === clientId &&
-            (r.content === content ||
-              r.content.endsWith(`] ${content}`)) // legacy timestamp-prefixed
+      // Dedup pre-check. Two SQL shapes — one with the legacy LIKE branch
+      // (3 params: clientId, dedupKey, legacyDays as string), one without
+      // (2 params: clientId, dedupKey).
+      if (sql.includes("FROM memory_nodes")) {
+        const [clientId, content, legacyDaysStr] = params;
+        const legacyDays = legacyDaysStr ? Number(legacyDaysStr) : 0;
+        const cutoff = new Date(
+          now.getTime() - legacyDays * 24 * 60 * 60 * 1000
         );
-        return { rows: match ? [{ id: match.id }] : [] };
+
+        const exactMatch = existing.find(
+          (r) => r.client_id === clientId && r.content === content
+        );
+        if (exactMatch) {
+          return {
+            rows: [{ id: exactMatch.id, match_kind: "exact" }],
+          };
+        }
+
+        if (legacyDays > 0) {
+          const legacyMatch = existing.find(
+            (r) =>
+              r.client_id === clientId &&
+              r.content.endsWith(`] ${content}`) &&
+              (!r.created_at || r.created_at > cutoff)
+          );
+          if (legacyMatch) {
+            return {
+              rows: [{ id: legacyMatch.id, match_kind: "legacy" }],
+            };
+          }
+        }
+
+        return { rows: [] };
       }
       // Insert path
       if (sql.startsWith("INSERT INTO memory_nodes")) {
@@ -636,7 +663,7 @@ describe("ingest dedup option", () => {
     expect(inserted[0].content).toBe("fresh content");
   });
 
-  it("returns the existing row's id when raw content matches", async () => {
+  it("returns the existing row's id when raw content matches (exact)", async () => {
     const { db, inserted } = makeMockDb({
       existing: [
         { id: "mem_existing", client_id: "c", content: "duplicate content" },
@@ -649,18 +676,21 @@ describe("ingest dedup option", () => {
     });
 
     expect(out.deduped).toBe(true);
+    expect(out.dedupMatchKind).toBe("exact");
     expect(out.id).toBe("mem_existing");
     expect(out.content).toBe("duplicate content");
     expect(inserted).toHaveLength(0); // no insert happened
   });
 
-  it("matches legacy timestamp-prefixed rows (`[<iso>] <content>`)", async () => {
+  it("matches legacy timestamp-prefixed rows (`[<iso>] <content>`) within the default window", async () => {
     const { db, inserted } = makeMockDb({
       existing: [
         {
           id: "mem_legacy",
           client_id: "c",
           content: "[2026-04-26T10:00:00Z] duplicate content",
+          // Default window is 7 days; created_at omitted → mocked as
+          // "now" so the row is in-window.
         },
       ],
     });
@@ -671,10 +701,89 @@ describe("ingest dedup option", () => {
     });
 
     expect(out.deduped).toBe(true);
+    expect(out.dedupMatchKind).toBe("legacy");
     expect(out.id).toBe("mem_legacy");
     expect(inserted).toHaveLength(0);
   });
 
+  it("legacy LIKE branch ignores rows older than dedupLegacyWindowDays", async () => {
+    const now = new Date("2026-04-27T00:00:00Z");
+    const tenDaysAgo = new Date(now.getTime() - 10 * 24 * 60 * 60 * 1000);
+    const { db, inserted } = makeMockDb({
+      now,
+      existing: [
+        {
+          id: "mem_old",
+          client_id: "c",
+          content: "[2026-04-17T00:00:00Z] duplicate content",
+          created_at: tenDaysAgo,
+        },
+      ],
+    });
+
+    const out = await ingest(db, mockAi, mockLlm, "duplicate content", {
+      clientId: "c",
+      dedup: true,
+      // Default window 7 days — old row should NOT match
+    });
+
+    expect(out.deduped).toBeUndefined();
+    expect(inserted).toHaveLength(1);
+  });
+
+  it("dedupLegacyWindowDays: 0 disables the legacy LIKE branch", async () => {
+    const { db, inserted, calls } = makeMockDb({
+      existing: [
+        {
+          id: "mem_legacy",
+          client_id: "c",
+          content: "[2026-04-26T10:00:00Z] duplicate content",
+        },
+      ],
+    });
+
+    const out = await ingest(db, mockAi, mockLlm, "duplicate content", {
+      clientId: "c",
+      dedup: true,
+      dedupLegacyWindowDays: 0,
+    });
+
+    // Legacy match wouldn't fire — row inserts as fresh.
+    expect(out.deduped).toBeUndefined();
+    expect(inserted).toHaveLength(1);
+
+    // SQL shouldn't include the LIKE branch at all.
+    const dedupQuery = calls.find((c) => c.sql.includes("FROM memory_nodes"));
+    expect(dedupQuery.sql).not.toContain("LIKE");
+    expect(dedupQuery.params).toHaveLength(2); // [clientId, dedupKey] only
+  });
+
+  it("dedupLegacyWindowDays: 30 widens the window — old rows match", async () => {
+    const now = new Date("2026-04-27T00:00:00Z");
+    const tenDaysAgo = new Date(now.getTime() - 10 * 24 * 60 * 60 * 1000);
+    const { db, inserted } = makeMockDb({
+      now,
+      existing: [
+        {
+          id: "mem_old",
+          client_id: "c",
+          content: "[2026-04-17T00:00:00Z] duplicate content",
+          created_at: tenDaysAgo,
+        },
+      ],
+    });
+
+    const out = await ingest(db, mockAi, mockLlm, "duplicate content", {
+      clientId: "c",
+      dedup: true,
+      dedupLegacyWindowDays: 30,
+    });
+
+    expect(out.deduped).toBe(true);
+    expect(out.dedupMatchKind).toBe("legacy");
+    expect(inserted).toHaveLength(0);
+  });
+
   it("dedup off (default) still inserts on duplicate content", async () => {
     const { db, inserted } = makeMockDb({
       existing: [
@@ -746,7 +855,7 @@ describe("ingest dedup option", () => {
       if (sql.includes("SELECT id FROM memory_layers")) {
         return { rows: [{ id: "layer-1" }] };
       }
-      if (sql.includes("SELECT id FROM memory_nodes")) {
+      if (sql.includes("FROM memory_nodes")) {
         dupCheckSql = sql;
         throw new Error("DB unreachable");
       }

package/packages/memory/src/ingest.js

@@ -25,19 +25,29 @@ import { distill } from "./distill.js";
  *   byte-equal content already exists for this `client_id`. Use for
  *   retry-safe pipelines where the same logical event may be processed
  *   twice (queue retries, consumer fan-out). Returns the existing row's
- *   id with `{deduped: true}` instead of inserting. Strict equality —
- *   not a semantic similarity match. Best-effort: if the SELECT itself
- *   fails, ingest proceeds (worst case: duplicate row, identical to
- *   `dedup:false` behaviour). The eventual structural fix is a
- *   `UNIQUE(client_id, content_hash)` constraint at the schema level;
- *   this option is the bridge.
+ *   id with `{deduped: true, dedupMatchKind: "exact"|"legacy"}` instead
+ *   of inserting. Strict equality — not a semantic similarity match.
+ *   Best-effort: if the SELECT itself fails, ingest proceeds (worst case:
+ *   duplicate row, identical to `dedup:false` behaviour). The eventual
+ *   structural fix is a `UNIQUE(client_id, content_hash)` constraint at
+ *   the schema level; this option is the bridge.
  * @param {string} [opts.dedupContent] - Optional: the string to dedup
  *   against, when it differs from what gets stored. Use when callers
  *   wrap the stored content in a non-stable prefix (timestamps, run
  *   ids) — pass the raw form here so retries of the same logical event
  *   match across runs whose prefixes differ by a few ms. Defaults to
  *   `content`.
- * @returns {Promise<{id: string, content: string, layerId: string, deduped?: boolean}>}
+ * @param {number} [opts.dedupLegacyWindowDays=7] - How far back the
+ *   `[<iso>] <content>` legacy-form `LIKE` match scans. Default 7 days.
+ *   The leading-wildcard `LIKE` can't use a btree index, so without a
+ *   bound it would scan the whole tenant partition on every ingest as
+ *   the corpus grows. Real retries land within seconds, so 7 days is
+ *   generous; the window only exists so the dedup check transitions
+ *   cleanly when callers stop writing the legacy form. Set `0` to
+ *   disable the legacy-form match entirely (recommended once the
+ *   backfill script has run and no caller writes the prefix anymore —
+ *   strict equality alone is then enough).
+ * @returns {Promise<{id: string, content: string, layerId: string, deduped?: boolean, dedupMatchKind?: "exact"|"legacy"}>}
  */
 export async function ingest(db, ai, llm, content, opts = {}) {
   const clientId = opts.clientId;
@@ -60,29 +70,62 @@ export async function ingest(db, ai, llm, content, opts = {}) {
 
   // Optional dedup: skip the insert (and all the embedding/HyDE/distill
   // work that would follow) if a row with byte-equal content already
-  // exists for this tenant. The dedup key is `opts.dedupContent` if
-  // provided (use for callers that wrap the stored form in a non-stable
-  // prefix like a timestamp), else `content`. The OR-LIKE branch matches
-  // against legacy `[<iso>] <content>` rows so callers that wrote with a
-  // timestamp prefix dedup correctly until the legacy corpus ages out.
+  // exists for this tenant.
+  //
+  // Two match strategies:
+  //   - exact:  content = $key  (uses the (client_id, content) btree if
+  //             one is present; degrades to a partition scan if not)
+  //   - legacy: content LIKE '%] ' || $key  for `[<iso>] <key>` rows
+  //             that callers wrote with a timestamp prefix. Bounded by
+  //             dedupLegacyWindowDays so the leading-wildcard scan
+  //             stays cheap as the corpus grows.
+  //
+  // Caller can disable the legacy branch by setting
+  // dedupLegacyWindowDays: 0 — once the backfill has run and no caller
+  // writes the prefix anymore, strict equality alone is enough.
   if (opts.dedup) {
     const dedupKey =
       typeof opts.dedupContent === "string" ? opts.dedupContent : content;
+    const legacyWindowDays =
+      opts.dedupLegacyWindowDays === undefined
+        ? 7
+        : Number(opts.dedupLegacyWindowDays);
     try {
-      const dupCheck = await db(
-        `SELECT id FROM memory_nodes
-           WHERE client_id = $1
-             AND (content = $2 OR content LIKE '%] ' || $2)
-           LIMIT 1`,
-        [clientId, dedupKey]
-      );
+      const sql =
+        legacyWindowDays > 0
+          ? `SELECT id,
+                    CASE WHEN content = $2 THEN 'exact' ELSE 'legacy' END AS match_kind
+               FROM memory_nodes
+              WHERE client_id = $1
+                AND (
+                  content = $2
+                  OR (
+                    content LIKE '%] ' || $2
+                    AND created_at > NOW() - ($3 || ' days')::interval
+                  )
+                )
+              LIMIT 1`
+          : `SELECT id, 'exact' AS match_kind
+               FROM memory_nodes
+              WHERE client_id = $1
+                AND content = $2
+              LIMIT 1`;
+      const params =
+        legacyWindowDays > 0
+          ? [clientId, dedupKey, String(legacyWindowDays)]
+          : [clientId, dedupKey];
+      const dupCheck = await db(sql, params);
       if (dupCheck.rows?.length) {
-        log(`dedup: matched existing memory ${dupCheck.rows[0].id}`);
+        const matchKind = dupCheck.rows[0].match_kind || "exact";
+        log(
+          `dedup: matched existing memory ${dupCheck.rows[0].id} (${matchKind})`
+        );
         return {
           id: dupCheck.rows[0].id,
           content,
           layerId,
           deduped: true,
+          dedupMatchKind: matchKind,
         };
       }
     } catch (err) {