@pentatonic-ai/ai-agent-sdk 0.5.2 → 0.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory/src/__tests__/api-contract.test.js

@@ -283,6 +283,89 @@ describe("search options contract", () => {
 
     expect(Array.isArray(results)).toBe(true);
   });
+
+  it("SQL includes atomBoost and verbosityPenalty terms", async () => {
+    const seenSqls = [];
+    const mockDb = async (sql) => {
+      seenSqls.push(sql);
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    await search(mockDb, mockAi, "q", { clientId: "c" });
+
+    const scoringSql = seenSqls.find((s) => s.includes("final_score"));
+    expect(scoringSql).toBeDefined();
+    expect(scoringSql).toMatch(/source_id IS NOT NULL/);
+    expect(scoringSql).toMatch(/length\(mn\.content\)/);
+  });
+
+  it("dedupeBySource drops raw rows whose id is a source of a matched atom", async () => {
+    const rows = [
+      { id: "raw-1", client_id: "c", layer_id: "l", content: "long raw turn",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: null },
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "Phil owns a Subaru",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.8, source_id: "raw-1" },
+    ];
+    let searchCallCount = 0;
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) {
+        searchCallCount++;
+        return { rows };
+      }
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", { clientId: "c", minScore: 0 });
+
+    expect(searchCallCount).toBe(1);
+    expect(out.length).toBe(1);
+    expect(out[0].id).toBe("atom-1");
+    expect(out[0].source_id).toBe("raw-1");
+  });
+
+  it("dedupeBySource: false keeps both atom and its raw source", async () => {
+    const rows = [
+      { id: "raw-1", client_id: "c", layer_id: "l", content: "long",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: null },
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "short",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.8, source_id: "raw-1" },
+    ];
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) return { rows };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", {
+      clientId: "c",
+      minScore: 0,
+      dedupeBySource: false,
+    });
+
+    expect(out.length).toBe(2);
+    expect(out.map((r) => r.id).sort()).toEqual(["atom-1", "raw-1"]);
+  });
+
+  it("search results include source_id (null for raw, set for atoms)", async () => {
+    const rows = [
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "atom",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: "raw-1" },
+    ];
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) return { rows };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", { clientId: "c", minScore: 0 });
+    expect(out[0].source_id).toBe("raw-1");
+  });
 });
 
 // --- Ingest options contract ---

package/packages/memory/src/search.js

@@ -10,6 +10,11 @@ const DEFAULT_WEIGHTS = {
   relevance: 0.6,
   recency: 0.25,
   frequency: 0.15,
+  // Boost distilled atoms — they're high signal per token by design.
+  atomBoost: 0.15,
+  // Penalty on verbose raw turns. Short focused memories rank higher.
+  // Atoms are exempt (penalty skipped when source_id IS NOT NULL).
+  verbosityPenalty: 0.1,
 };
 
 /**
@@ -25,6 +30,10 @@ const DEFAULT_WEIGHTS = {
  * @param {number} [opts.minScore=0.5] - Minimum score threshold
  * @param {string} [opts.userId] - Optional user scope
  * @param {object} [opts.weights] - Override scoring weights
+ *   (relevance, recency, frequency, atomBoost, verbosityPenalty)
+ * @param {boolean} [opts.dedupeBySource=true] - When an atom matches,
+ *   drop its raw source memory from the results (atoms are already
+ *   distillations of the source, so returning both is redundant).
  * @param {Function} [opts.logger] - Optional logger
  * @returns {Promise<Array>} Scored memory results
  */
@@ -107,7 +116,19 @@ export async function search(db, ai, query, opts = {}) {
         ${w.recency} * exp(
           -0.01 * EXTRACT(EPOCH FROM NOW() - COALESCE(mn.last_accessed, mn.created_at)) / 3600
         ) +
-        ${w.frequency} * (ln(mn.access_count + 1) / ln(ma.val + 1))
+        ${w.frequency} * (ln(mn.access_count + 1) / ln(ma.val + 1)) +
+        ${w.atomBoost} * (CASE WHEN mn.source_id IS NOT NULL THEN 1 ELSE 0 END) -
+        ${w.verbosityPenalty} * (
+          CASE WHEN mn.source_id IS NULL THEN
+            LEAST(
+              GREATEST(
+                (ln(length(mn.content) + 1) - ln(200)) / (ln(10000) - ln(200)),
+                0
+              ),
+              1
+            )
+          ELSE 0 END
+        )
       ) AS final_score
     FROM memory_nodes mn
     CROSS JOIN max_ac ma
@@ -123,10 +144,21 @@ export async function search(db, ai, query, opts = {}) {
 
   const result = await db(sql, params);
 
-  const filtered = (result.rows || []).filter(
+  let filtered = (result.rows || []).filter(
     (r) => parseFloat(r.final_score) >= threshold
   );
 
+  // De-dupe: when an atom matches, drop its raw source from the set.
+  // Default on; set opts.dedupeBySource: false to keep both.
+  if (opts.dedupeBySource !== false) {
+    const atomSources = new Set(
+      filtered.filter((r) => r.source_id).map((r) => r.source_id)
+    );
+    if (atomSources.size > 0) {
+      filtered = filtered.filter((r) => !atomSources.has(r.id));
+    }
+  }
+
   // Increment access counts
   const ids = filtered.map((r) => r.id);
   if (ids.length) {
@@ -182,6 +214,7 @@ function mapRow(row) {
     client_id: row.client_id,
     user_id: row.user_id || null,
     layer_id: row.layer_id,
+    source_id: row.source_id || null,
     content: row.content,
     metadata:
       typeof row.metadata === "string"

package/packages/memory/src/server.js

@@ -347,7 +347,7 @@ async function main() {
         const health = {
           status: "ok",
           client: CLIENT_ID,
-          version: "0.5.2",
+          version: "0.5.3",
           search: "text",
           db: false,
           ollama: false,