@pentatonic-ai/ai-agent-sdk 0.5.1 → 0.5.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pentatonic-ai/ai-agent-sdk",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "TES SDK — LLM observability and lifecycle tracking via Pentatonic Thing Event System. Track token usage, tool calls, and conversations. Manage things through event-sourced lifecycle stages with AI enrichment and vector search.",
   "type": "module",
   "main": "./dist/index.cjs",

package/packages/memory/src/__tests__/api-contract.test.js

@@ -168,6 +168,11 @@ describe("named exports", () => {
 // --- AI client ---
 
 describe("createAIClient", () => {
+  const realFetch = globalThis.fetch;
+  afterEach(() => {
+    globalThis.fetch = realFetch;
+  });
+
   it("returns an object with embed() and chat()", () => {
     const client = createAIClient({
       url: "http://localhost:11434/v1",
@@ -185,6 +190,79 @@ describe("createAIClient", () => {
     });
     expect(client).toBeDefined();
   });
+
+  it("hits /embeddings by default (OpenAI spec)", async () => {
+    let hitUrl;
+    globalThis.fetch = async (url) => {
+      hitUrl = url;
+      return { ok: true, json: async () => ({ data: [{ embedding: [0.1, 0.2] }] }) };
+    };
+    const client = createAIClient({
+      url: "http://localhost:11434/v1",
+      model: "test",
+    });
+    await client.embed("hello");
+    expect(hitUrl).toBe("http://localhost:11434/v1/embeddings");
+  });
+
+  it("uses embeddingPath override (e.g. Pentatonic AI Gateway)", async () => {
+    let hitUrl;
+    globalThis.fetch = async (url) => {
+      hitUrl = url;
+      return { ok: true, json: async () => ({ data: [{ embedding: [0.1] }] }) };
+    };
+    const client = createAIClient({
+      url: "https://lambda-gateway.pentatonic.com/v1",
+      model: "NV-Embed-v2",
+      embeddingPath: "embed",
+    });
+    await client.embed("hello");
+    expect(hitUrl).toBe("https://lambda-gateway.pentatonic.com/v1/embed");
+  });
+
+  it("normalises leading slashes and trailing base-url slashes", async () => {
+    let hitUrl;
+    globalThis.fetch = async (url) => {
+      hitUrl = url;
+      return { ok: true, json: async () => ({ data: [{ embedding: [0.1] }] }) };
+    };
+    const client = createAIClient({
+      url: "https://gateway.test/v1/",
+      model: "m",
+      embeddingPath: "/embed",
+    });
+    await client.embed("hi");
+    expect(hitUrl).toBe("https://gateway.test/v1/embed");
+  });
+
+  it("chatPath override applies to chat() too", async () => {
+    let hitUrl;
+    globalThis.fetch = async (url) => {
+      hitUrl = url;
+      return { ok: true, json: async () => ({ choices: [{ message: { content: "hi" } }] }) };
+    };
+    const client = createAIClient({
+      url: "https://gateway.test/v1",
+      model: "m",
+      chatPath: "chat",
+    });
+    await client.chat([{ role: "user", content: "q" }]);
+    expect(hitUrl).toBe("https://gateway.test/v1/chat");
+  });
+
+  it("chat defaults to /chat/completions", async () => {
+    let hitUrl;
+    globalThis.fetch = async (url) => {
+      hitUrl = url;
+      return { ok: true, json: async () => ({ choices: [{ message: { content: "hi" } }] }) };
+    };
+    const client = createAIClient({
+      url: "http://localhost:11434/v1",
+      model: "m",
+    });
+    await client.chat([{ role: "user", content: "q" }]);
+    expect(hitUrl).toBe("http://localhost:11434/v1/chat/completions");
+  });
 });
 
 // --- Search options contract ---
@@ -205,6 +283,89 @@ describe("search options contract", () => {
 
     expect(Array.isArray(results)).toBe(true);
   });
+
+  it("SQL includes atomBoost and verbosityPenalty terms", async () => {
+    const seenSqls = [];
+    const mockDb = async (sql) => {
+      seenSqls.push(sql);
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    await search(mockDb, mockAi, "q", { clientId: "c" });
+
+    const scoringSql = seenSqls.find((s) => s.includes("final_score"));
+    expect(scoringSql).toBeDefined();
+    expect(scoringSql).toMatch(/source_id IS NOT NULL/);
+    expect(scoringSql).toMatch(/length\(mn\.content\)/);
+  });
+
+  it("dedupeBySource drops raw rows whose id is a source of a matched atom", async () => {
+    const rows = [
+      { id: "raw-1", client_id: "c", layer_id: "l", content: "long raw turn",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: null },
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "Phil owns a Subaru",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.8, source_id: "raw-1" },
+    ];
+    let searchCallCount = 0;
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) {
+        searchCallCount++;
+        return { rows };
+      }
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", { clientId: "c", minScore: 0 });
+
+    expect(searchCallCount).toBe(1);
+    expect(out.length).toBe(1);
+    expect(out[0].id).toBe("atom-1");
+    expect(out[0].source_id).toBe("raw-1");
+  });
+
+  it("dedupeBySource: false keeps both atom and its raw source", async () => {
+    const rows = [
+      { id: "raw-1", client_id: "c", layer_id: "l", content: "long",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: null },
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "short",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.8, source_id: "raw-1" },
+    ];
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) return { rows };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", {
+      clientId: "c",
+      minScore: 0,
+      dedupeBySource: false,
+    });
+
+    expect(out.length).toBe(2);
+    expect(out.map((r) => r.id).sort()).toEqual(["atom-1", "raw-1"]);
+  });
+
+  it("search results include source_id (null for raw, set for atoms)", async () => {
+    const rows = [
+      { id: "atom-1", client_id: "c", layer_id: "l", content: "atom",
+        confidence: 1, decay_rate: 0.05, access_count: 0, final_score: 0.9, source_id: "raw-1" },
+    ];
+    const mockDb = async (sql) => {
+      if (sql.includes("information_schema.columns")) return { rows: [{ "?column?": 1 }] };
+      if (sql.includes("final_score")) return { rows };
+      return { rows: [] };
+    };
+    const mockAi = { embed: async () => ({ embedding: [0.1], dimensions: 1, model: "t" }) };
+
+    const out = await search(mockDb, mockAi, "q", { clientId: "c", minScore: 0 });
+    expect(out[0].source_id).toBe("raw-1");
+  });
 });
 
 // --- Ingest options contract ---

package/packages/memory/src/ai.js

@@ -8,10 +8,16 @@
 /**
  * Create an AI client from config.
  *
+ * Defaults to OpenAI-standard paths (`/embeddings`, `/chat/completions`).
+ * Override with `embeddingPath` / `chatPath` for gateways that use
+ * different routes — e.g. Pentatonic AI Gateway exposes `/embed`.
+ *
  * @param {object} config
  * @param {string} config.url - Base URL (e.g. "http://ollama:11434/v1")
  * @param {string} config.model - Model name
  * @param {string} [config.apiKey] - Optional API key
+ * @param {string} [config.embeddingPath="embeddings"] - Path appended to url
+ * @param {string} [config.chatPath="chat/completions"] - Path appended to url
  * @param {number} [config.dimensions] - Expected embedding dimensions
  * @returns {object} Client with embed() and chat() methods
  */
@@ -22,6 +28,23 @@ export function createAIClient(config) {
     headers["X-API-Key"] = config.apiKey;
   }
 
+  // Strip leading slashes so callers can use "embed" or "/embed"
+  // interchangeably. Base url may or may not have a trailing slash.
+  // Plain loops (not regex) to avoid polynomial-regex scanner flags.
+  const stripLeading = (s) => {
+    let i = 0;
+    while (i < s.length && s[i] === "/") i++;
+    return i === 0 ? s : s.slice(i);
+  };
+  const stripTrailing = (s) => {
+    let i = s.length;
+    while (i > 0 && s[i - 1] === "/") i--;
+    return i === s.length ? s : s.slice(0, i);
+  };
+  const embeddingPath = stripLeading(config.embeddingPath || "embeddings");
+  const chatPath = stripLeading(config.chatPath || "chat/completions");
+  const baseUrl = stripTrailing(config.url);
+
   return {
     /**
      * Generate an embedding vector for text.
@@ -32,7 +55,7 @@ export function createAIClient(config) {
      */
     async embed(text, inputType = "passage") {
       try {
-        const res = await fetch(`${config.url}/embeddings`, {
+        const res = await fetch(`${baseUrl}/${embeddingPath}`, {
           method: "POST",
           headers,
           body: JSON.stringify({
@@ -70,7 +93,7 @@ export function createAIClient(config) {
      */
     async chat(messages, opts = {}) {
       try {
-        const res = await fetch(`${config.url}/chat/completions`, {
+        const res = await fetch(`${baseUrl}/${chatPath}`, {
           method: "POST",
           headers,
           body: JSON.stringify({

package/packages/memory/src/search.js

@@ -10,6 +10,11 @@ const DEFAULT_WEIGHTS = {
   relevance: 0.6,
   recency: 0.25,
   frequency: 0.15,
+  // Boost distilled atoms — they're high signal per token by design.
+  atomBoost: 0.15,
+  // Penalty on verbose raw turns. Short focused memories rank higher.
+  // Atoms are exempt (penalty skipped when source_id IS NOT NULL).
+  verbosityPenalty: 0.1,
 };
 
 /**
@@ -25,6 +30,10 @@ const DEFAULT_WEIGHTS = {
  * @param {number} [opts.minScore=0.5] - Minimum score threshold
  * @param {string} [opts.userId] - Optional user scope
  * @param {object} [opts.weights] - Override scoring weights
+ *   (relevance, recency, frequency, atomBoost, verbosityPenalty)
+ * @param {boolean} [opts.dedupeBySource=true] - When an atom matches,
+ *   drop its raw source memory from the results (atoms are already
+ *   distillations of the source, so returning both is redundant).
  * @param {Function} [opts.logger] - Optional logger
  * @returns {Promise<Array>} Scored memory results
  */
@@ -107,7 +116,19 @@ export async function search(db, ai, query, opts = {}) {
         ${w.recency} * exp(
           -0.01 * EXTRACT(EPOCH FROM NOW() - COALESCE(mn.last_accessed, mn.created_at)) / 3600
         ) +
-        ${w.frequency} * (ln(mn.access_count + 1) / ln(ma.val + 1))
+        ${w.frequency} * (ln(mn.access_count + 1) / ln(ma.val + 1)) +
+        ${w.atomBoost} * (CASE WHEN mn.source_id IS NOT NULL THEN 1 ELSE 0 END) -
+        ${w.verbosityPenalty} * (
+          CASE WHEN mn.source_id IS NULL THEN
+            LEAST(
+              GREATEST(
+                (ln(length(mn.content) + 1) - ln(200)) / (ln(10000) - ln(200)),
+                0
+              ),
+              1
+            )
+          ELSE 0 END
+        )
       ) AS final_score
     FROM memory_nodes mn
     CROSS JOIN max_ac ma
@@ -123,10 +144,21 @@ export async function search(db, ai, query, opts = {}) {
 
   const result = await db(sql, params);
 
-  const filtered = (result.rows || []).filter(
+  let filtered = (result.rows || []).filter(
     (r) => parseFloat(r.final_score) >= threshold
   );
 
+  // De-dupe: when an atom matches, drop its raw source from the set.
+  // Default on; set opts.dedupeBySource: false to keep both.
+  if (opts.dedupeBySource !== false) {
+    const atomSources = new Set(
+      filtered.filter((r) => r.source_id).map((r) => r.source_id)
+    );
+    if (atomSources.size > 0) {
+      filtered = filtered.filter((r) => !atomSources.has(r.id));
+    }
+  }
+
   // Increment access counts
   const ids = filtered.map((r) => r.id);
   if (ids.length) {
@@ -182,6 +214,7 @@ function mapRow(row) {
     client_id: row.client_id,
     user_id: row.user_id || null,
     layer_id: row.layer_id,
+    source_id: row.source_id || null,
     content: row.content,
     metadata:
       typeof row.metadata === "string"

package/packages/memory/src/server.js

@@ -13,6 +13,9 @@
  *   LLM_URL          — OpenAI-compatible chat endpoint (required)
  *   LLM_MODEL        — Chat model name for HyDE (required)
  *   API_KEY          — API key for embedding/LLM endpoints (optional)
+ *   EMBEDDING_PATH   — Path appended to EMBEDDING_URL (default: "embeddings").
+ *                      Set to "embed" for the Pentatonic AI Gateway.
+ *   CHAT_PATH        — Path appended to LLM_URL (default: "chat/completions")
  *   CLIENT_ID        — Client ID for memory scoping (default: "default")
  *   PORT             — HTTP port for SSE transport (default: 3333)
  */
@@ -46,11 +49,13 @@ function createMemory() {
       url: process.env.EMBEDDING_URL,
       model: process.env.EMBEDDING_MODEL,
       apiKey: process.env.API_KEY,
+      embeddingPath: process.env.EMBEDDING_PATH,
     },
     llm: {
       url: process.env.LLM_URL,
       model: process.env.LLM_MODEL,
       apiKey: process.env.API_KEY,
+      chatPath: process.env.CHAT_PATH,
     },
     logger: (msg) => process.stderr.write(`[memory] ${msg}\n`),
   });
@@ -342,7 +347,7 @@ async function main() {
         const health = {
           status: "ok",
           client: CLIENT_ID,
-          version: "0.5.1",
+          version: "0.5.3",
           search: "text",
           db: false,
           ollama: false,