@pentatonic-ai/ai-agent-sdk 0.5.11 → 0.7.0

package/packages/memory/src/__tests__/corpus-signatures.test.js

@@ -0,0 +1,175 @@
+import { extractReferences } from "../corpus/signatures.js";
+
+function file(relPath, content, ext) {
+  return {
+    relPath,
+    content,
+    ext: ext ?? "." + relPath.split(".").pop(),
+  };
+}
+
+describe("corpus/signatures.js — reference extraction", () => {
+  describe("metadata shape (universal)", () => {
+    it("every reference carries kind=code_reference + path + lines + language", () => {
+      const refs = extractReferences(
+        file("src/x.ts", "export function foo() { return 1; }")
+      );
+      expect(refs.length).toBeGreaterThan(0);
+      for (const r of refs) {
+        expect(r.metadata.kind).toBe("code_reference");
+        expect(r.metadata.path).toBe("src/x.ts");
+        expect(typeof r.metadata.start_line).toBe("number");
+        expect(typeof r.metadata.end_line).toBe("number");
+        expect(r.metadata.language).toBe("typescript");
+        expect(r.metadata.lines).toBe(
+          `${r.metadata.start_line}-${r.metadata.end_line}`
+        );
+      }
+    });
+  });
+
+  describe("markdown", () => {
+    it("emits one reference per H1/H2 section", () => {
+      const md = [
+        "# Top",
+        "intro paragraph one.",
+        "",
+        "## Auth",
+        "JWT verification middleware lives here.",
+        "",
+        "## Storage",
+        "Postgres + R2.",
+      ].join("\n");
+      const refs = extractReferences(file("docs/arch.md", md));
+      expect(refs.length).toBe(3);
+      expect(refs[0].metadata.symbol).toBe("Top");
+      expect(refs[1].metadata.symbol).toBe("Auth");
+      expect(refs[2].metadata.symbol).toBe("Storage");
+      expect(refs[1].content).toMatch(/JWT verification/);
+    });
+
+    it("falls back to a file-level reference when there are no headings", () => {
+      const refs = extractReferences(
+        file("README.txt", "no headings here, just text", ".md")
+      );
+      expect(refs.length).toBe(1);
+      expect(refs[0].metadata.start_line).toBe(1);
+      expect(refs[0].content).toMatch(/README\.txt/);
+    });
+  });
+
+  describe("javascript / typescript", () => {
+    it("extracts top-level function declarations", () => {
+      const code = [
+        "import x from 'y';",
+        "",
+        "export function authenticate(req) {",
+        "  return verify(req);",
+        "}",
+        "",
+        "function helper() { return 1; }",
+      ].join("\n");
+      const refs = extractReferences(file("src/auth.js", code));
+      const symbols = refs.map((r) => r.metadata.symbol);
+      expect(symbols).toContain("authenticate");
+      expect(symbols).toContain("helper");
+    });
+
+    it("extracts class declarations", () => {
+      const code = "export class UserService {\n  ping() {}\n}";
+      const refs = extractReferences(file("src/svc.ts", code));
+      expect(refs.find((r) => r.metadata.symbol === "UserService")).toBeDefined();
+    });
+
+    it("extracts top-level const/let/var bindings", () => {
+      const code = "export const handler = async (req) => req;";
+      const refs = extractReferences(file("src/h.js", code));
+      expect(refs[0].metadata.symbol).toBe("handler");
+    });
+
+    it("ignores indented (non-top-level) declarations", () => {
+      const code = [
+        "function outer() {",
+        "  function inner() { return 1; }",
+        "}",
+      ].join("\n");
+      const refs = extractReferences(file("src/x.js", code));
+      const symbols = refs.map((r) => r.metadata.symbol);
+      expect(symbols).toContain("outer");
+      expect(symbols).not.toContain("inner");
+    });
+
+    it("falls back to file-level reference when nothing matches", () => {
+      const code = "// just a comment\nconsole.log('side effect');";
+      const refs = extractReferences(file("src/y.js", code));
+      expect(refs.length).toBe(1);
+      expect(refs[0].metadata.start_line).toBe(1);
+      expect(refs[0].content).toMatch(/src\/y\.js/);
+    });
+
+    it("captures path:line + symbol in the embedded content", () => {
+      const code = "\n\nexport function foo() {}\n";
+      const refs = extractReferences(file("a/b.ts", code));
+      expect(refs[0].content).toMatch(/a\/b\.ts:3 — foo/);
+    });
+  });
+
+  describe("python", () => {
+    it("extracts top-level def and class declarations", () => {
+      const py = [
+        "import os",
+        "",
+        "def authenticate(req):",
+        "    return True",
+        "",
+        "class UserService:",
+        "    def ping(self): pass",
+      ].join("\n");
+      const refs = extractReferences(file("svc/auth.py", py));
+      const symbols = refs.map((r) => r.metadata.symbol);
+      expect(symbols).toContain("authenticate");
+      expect(symbols).toContain("UserService");
+      // Methods of the class should NOT be separately extracted
+      expect(symbols).not.toContain("ping");
+    });
+  });
+
+  describe("json / yaml configs", () => {
+    it("collapses to a single reference of top-level keys (json)", () => {
+      const json = JSON.stringify(
+        { name: "x", version: "1.0", dependencies: {} },
+        null,
+        2
+      );
+      const refs = extractReferences(file("package.json", json));
+      expect(refs.length).toBe(1);
+      expect(refs[0].content).toMatch(/top-level keys/);
+      expect(refs[0].content).toMatch(/name/);
+      expect(refs[0].content).toMatch(/version/);
+      expect(refs[0].content).toMatch(/dependencies/);
+    });
+
+    it("collapses to a single reference of top-level keys (yaml)", () => {
+      const yaml = "name: x\nversion: 1.0\nbuild:\n  steps: []\n";
+      const refs = extractReferences(file("ci.yml", yaml));
+      expect(refs.length).toBe(1);
+      expect(refs[0].content).toMatch(/name/);
+      expect(refs[0].content).toMatch(/version/);
+      expect(refs[0].content).toMatch(/build/);
+      // Indented `steps:` should not appear as a top-level key
+      expect(refs[0].content).not.toMatch(/\bsteps\b/);
+    });
+  });
+
+  describe("unknown languages", () => {
+    it("emits a single file-level reference with the head of the content", () => {
+      const refs = extractReferences(
+        file("data.txt", "some random text\nover multiple lines", ".txt")
+      );
+      expect(refs.length).toBe(1);
+      expect(refs[0].metadata.language).toBe("text");
+      expect(refs[0].content).toMatch(/data\.txt/);
+      expect(refs[0].content).toMatch(/some random text/);
+    });
+  });
+});

package/packages/memory/src/__tests__/corpus-state.test.js

@@ -0,0 +1,161 @@
+/**
+ * Tests for corpus state persistence — atomic writes, version handling,
+ * and the upsert/remove API used by ingest.js.
+ */
+
+import { promises as fsp, existsSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdtemp } from "node:fs/promises";
+
+import {
+  loadState,
+  saveState,
+  upsertSource,
+  removeSource,
+  getSource,
+  recordFile,
+  forgetFile,
+  recomputeStats,
+  emptyState,
+  STATE_VERSION,
+} from "../corpus/state.js";
+
+let tmp;
+let statePath;
+
+beforeEach(async () => {
+  tmp = await mkdtemp(join(tmpdir(), "tes-state-"));
+  statePath = join(tmp, "corpus.json");
+});
+
+afterEach(async () => {
+  await fsp.rm(tmp, { recursive: true, force: true });
+});
+
+describe("loadState", () => {
+  it("returns empty state when file does not exist", async () => {
+    const state = await loadState(statePath);
+    expect(state.version).toBe(STATE_VERSION);
+    expect(state.sources).toEqual({});
+  });
+
+  it("loads previously-saved state", async () => {
+    const initial = emptyState();
+    upsertSource(initial, "/repo/x", { sourceType: "git" });
+    await saveState(initial, statePath);
+
+    const loaded = await loadState(statePath);
+    expect(loaded.sources["/repo/x"]).toBeDefined();
+    expect(loaded.sources["/repo/x"].sourceType).toBe("git");
+  });
+
+  it("rejects state with newer version than supported", async () => {
+    await fsp.writeFile(
+      statePath,
+      JSON.stringify({ version: STATE_VERSION + 99, sources: {} })
+    );
+    await expect(loadState(statePath)).rejects.toThrow(/unsupported version/);
+  });
+
+  it("rejects corrupt JSON with a clear error", async () => {
+    await fsp.writeFile(statePath, "{ not valid json");
+    await expect(loadState(statePath)).rejects.toThrow(/corrupt JSON/);
+  });
+});
+
+describe("saveState", () => {
+  it("creates parent directory if missing", async () => {
+    const nested = join(tmp, "a", "b", "c", "corpus.json");
+    await saveState(emptyState(), nested);
+    expect(existsSync(nested)).toBe(true);
+  });
+
+  it("writes with mode 0600 (owner-only)", async () => {
+    await saveState(emptyState(), statePath);
+    const stat = await fsp.stat(statePath);
+    // mode bits: only check owner read+write set, group/other empty
+    const mode = stat.mode & 0o777;
+    expect(mode).toBe(0o600);
+  });
+
+  it("uses atomic rename (no half-written file on crash)", async () => {
+    // Save twice; the second save shouldn't leave a .tmp behind
+    await saveState(emptyState(), statePath);
+    await saveState(emptyState(), statePath);
+    const dir = await fsp.readdir(tmp);
+    expect(dir.filter((f) => f.endsWith(".tmp"))).toHaveLength(0);
+  });
+});
+
+describe("upsertSource / removeSource / getSource", () => {
+  it("upsert merges patches and resolves to absolute path", async () => {
+    const state = emptyState();
+    upsertSource(state, "./relative-path", { sourceType: "directory" });
+    const keys = Object.keys(state.sources);
+    expect(keys).toHaveLength(1);
+    expect(keys[0]).toMatch(/^\//); // absolute
+  });
+
+  it("upsert preserves existing fields when patching", async () => {
+    const state = emptyState();
+    upsertSource(state, "/repo", { sourceType: "git" });
+    upsertSource(state, "/repo", { lastSyncedCommit: "abc" });
+    expect(state.sources["/repo"].sourceType).toBe("git");
+    expect(state.sources["/repo"].lastSyncedCommit).toBe("abc");
+  });
+
+  it("removeSource returns false for unknown paths", () => {
+    const state = emptyState();
+    expect(removeSource(state, "/never-added")).toBe(false);
+  });
+
+  it("removeSource returns true and deletes when present", () => {
+    const state = emptyState();
+    upsertSource(state, "/repo", {});
+    expect(removeSource(state, "/repo")).toBe(true);
+    expect(state.sources["/repo"]).toBeUndefined();
+  });
+
+  it("getSource resolves relative paths to absolute lookups", () => {
+    const state = emptyState();
+    upsertSource(state, "/abs/repo", {});
+    // Looking up a different relative form that resolves to same abs
+    // returns the same record (assuming cwd matches when the test runs
+    // — we use an absolute path here to avoid flakiness)
+    expect(getSource(state, "/abs/repo")).toBeDefined();
+    expect(getSource(state, "/abs/other")).toBeNull();
+  });
+});
+
+describe("recordFile / forgetFile / recomputeStats", () => {
+  it("recordFile sets hash, chunks, indexedAt", () => {
+    const state = emptyState();
+    const src = upsertSource(state, "/repo", {});
+    recordFile(src, "src/x.ts", "hash123", 4);
+    expect(src.files["src/x.ts"]).toMatchObject({
+      hash: "hash123",
+      chunks: 4,
+    });
+    expect(src.files["src/x.ts"].indexedAt).toBeTruthy();
+  });
+
+  it("forgetFile removes the entry", () => {
+    const state = emptyState();
+    const src = upsertSource(state, "/repo", {});
+    recordFile(src, "src/x.ts", "h", 1);
+    expect(forgetFile(src, "src/x.ts")).toBe(true);
+    expect(src.files["src/x.ts"]).toBeUndefined();
+    expect(forgetFile(src, "src/x.ts")).toBe(false);
+  });
+
+  it("recomputeStats reflects current files", () => {
+    const state = emptyState();
+    const src = upsertSource(state, "/repo", {});
+    recordFile(src, "a", "h1", 3);
+    recordFile(src, "b", "h2", 7);
+    const stats = recomputeStats(src);
+    expect(stats.fileCount).toBe(2);
+    expect(stats.chunkCount).toBe(10);
+  });
+});

package/packages/memory/src/__tests__/ingest-corpus-opts.test.js

@@ -0,0 +1,129 @@
+/**
+ * Tests for ingest opts.distill / opts.hyde — both must be skippable
+ * for corpus ingest so code references don't trigger conversation-shaped
+ * LLM enrichment (distillation hallucinates "user facts" from code,
+ * HyDE generates expansion that won't match how callers query symbols).
+ */
+
+import { ingest } from "../ingest.js";
+
+function mockDb() {
+  const queries = [];
+  const db = async (sql, params) => {
+    queries.push({ sql, params });
+    if (sql.includes("FROM memory_layers")) {
+      return { rows: [{ id: "layer_xyz" }] };
+    }
+    return { rows: [] };
+  };
+  db.queries = queries;
+  return db;
+}
+
+function mockAi() {
+  let embedCalls = 0;
+  return {
+    embed: async () => {
+      embedCalls++;
+      return {
+        embedding: new Array(8).fill(0.1),
+        dimensions: 8,
+        model: "mock-embed",
+      };
+    },
+    chat: async () => "",
+    get embedCalls() {
+      return embedCalls;
+    },
+  };
+}
+
+function mockLlm() {
+  let chatCalls = 0;
+  const calls = [];
+  return {
+    chat: async (...args) => {
+      chatCalls++;
+      calls.push(args);
+      // Fake hypothetical-question response (3 lines)
+      return "What does this do?\nHow does this work?\nWhen is this called?";
+    },
+    get chatCalls() {
+      return chatCalls;
+    },
+    get calls() {
+      return calls;
+    },
+  };
+}
+
+describe("ingest — corpus opt-outs", () => {
+  it("runs HyDE by default (chat is invoked)", async () => {
+    const db = mockDb();
+    const ai = mockAi();
+    const llm = mockLlm();
+    await ingest(db, ai, llm, "user said hello", {
+      clientId: "acme",
+      distill: false, // isolate HyDE behavior
+    });
+    expect(llm.chatCalls).toBeGreaterThan(0);
+    // The HyDE update should land in metadata
+    const hydeUpdate = db.queries.find((q) =>
+      q.sql.includes("hypothetical_queries")
+    );
+    expect(hydeUpdate).toBeDefined();
+  });
+
+  it("skips HyDE when opts.hyde === false", async () => {
+    const db = mockDb();
+    const ai = mockAi();
+    const llm = mockLlm();
+    await ingest(db, ai, llm, "function authenticate(req)", {
+      clientId: "acme",
+      distill: false,
+      hyde: false,
+    });
+    expect(llm.chatCalls).toBe(0);
+    const hydeUpdate = db.queries.find((q) =>
+      q.sql.includes("hypothetical_queries")
+    );
+    expect(hydeUpdate).toBeUndefined();
+  });
+
+  it("skips both HyDE and distill for code references", async () => {
+    const db = mockDb();
+    const ai = mockAi();
+    const llm = mockLlm();
+    await ingest(db, ai, llm, "function authenticate(req)", {
+      clientId: "acme",
+      distill: false,
+      hyde: false,
+      metadata: {
+        kind: "code_reference",
+        path: "src/auth.js",
+        symbol: "authenticate",
+      },
+    });
+    // No LLM calls at all (only embedding)
+    expect(llm.chatCalls).toBe(0);
+    expect(ai.embedCalls).toBe(1);
+  });
+
+  it("preserves the metadata.kind field through to the insert", async () => {
+    const db = mockDb();
+    const ai = mockAi();
+    const llm = mockLlm();
+    await ingest(db, ai, llm, "fn x()", {
+      clientId: "acme",
+      hyde: false,
+      distill: false,
+      metadata: { kind: "code_reference", path: "x.js" },
+    });
+    const insert = db.queries.find((q) =>
+      q.sql.includes("INSERT INTO memory_nodes")
+    );
+    expect(insert).toBeDefined();
+    const metadataParam = JSON.parse(insert.params[4]);
+    expect(metadataParam.kind).toBe("code_reference");
+  });
+});

package/packages/memory/src/__tests__/search-kind.test.js

@@ -0,0 +1,108 @@
+/**
+ * Tests for search opts.kind — must add a metadata->>'kind' filter to
+ * the SQL so corpus-shaped queries can scope to code references and
+ * not compete with conversational memory for top-K slots.
+ */
+
+import { search, textSearch } from "../search.js";
+
+function makeDb() {
+  const calls = [];
+  const db = async (sql, params) => {
+    calls.push({ sql, params });
+    // Vector column existence check — pretend it's there
+    if (sql.includes("information_schema.columns")) {
+      return { rows: [{}] };
+    }
+    // Search SQL — return empty rows; we only assert SQL shape
+    return { rows: [] };
+  };
+  db.calls = calls;
+  return db;
+}
+
+function makeAi() {
+  return {
+    embed: async () => ({
+      embedding: new Array(8).fill(0.1),
+      dimensions: 8,
+      model: "mock",
+    }),
+  };
+}
+
+describe("search — kind filter", () => {
+  it("adds metadata->>'kind' filter when opts.kind is provided", async () => {
+    const db = makeDb();
+    await search(db, makeAi(), "auth", {
+      clientId: "acme",
+      kind: "code_reference",
+    });
+    const searchCall = db.calls.find(
+      (c) =>
+        c.sql.includes("FROM memory_nodes mn") &&
+        c.sql.includes("ORDER BY final_score")
+    );
+    expect(searchCall).toBeDefined();
+    expect(searchCall.sql).toMatch(/metadata->>'kind' = \$\d+/);
+    expect(searchCall.params).toContain("code_reference");
+  });
+
+  it("omits the kind filter entirely when not provided", async () => {
+    const db = makeDb();
+    await search(db, makeAi(), "auth", { clientId: "acme" });
+    const searchCall = db.calls.find(
+      (c) =>
+        c.sql.includes("FROM memory_nodes mn") &&
+        c.sql.includes("ORDER BY final_score")
+    );
+    expect(searchCall.sql).not.toMatch(/metadata->>'kind'/);
+  });
+
+  it("supports kind together with userId (param numbering stays valid)", async () => {
+    const db = makeDb();
+    await search(db, makeAi(), "auth", {
+      clientId: "acme",
+      userId: "u1",
+      kind: "code_reference",
+    });
+    const searchCall = db.calls.find(
+      (c) =>
+        c.sql.includes("FROM memory_nodes mn") &&
+        c.sql.includes("ORDER BY final_score")
+    );
+    expect(searchCall.sql).toMatch(/mn.user_id = \$\d+/);
+    expect(searchCall.sql).toMatch(/metadata->>'kind' = \$\d+/);
+    // Check params include both
+    expect(searchCall.params).toContain("u1");
+    expect(searchCall.params).toContain("code_reference");
+  });
+});
+
+describe("textSearch fallback — kind filter", () => {
+  it("propagates kind to the fallback path SQL", async () => {
+    const db = makeDb();
+    await textSearch(db, "auth", {
+      clientId: "acme",
+      kind: "code_reference",
+    });
+    const searchCall = db.calls.find(
+      (c) =>
+        c.sql.includes("FROM memory_nodes mn") &&
+        c.sql.includes("plainto_tsquery")
+    );
+    expect(searchCall.sql).toMatch(/metadata->>'kind' = \$\d+/);
+    expect(searchCall.params).toContain("code_reference");
+  });
+
+  it("omits kind from textSearch when not provided", async () => {
+    const db = makeDb();
+    await textSearch(db, "auth", { clientId: "acme" });
+    const searchCall = db.calls.find(
+      (c) =>
+        c.sql.includes("FROM memory_nodes mn") &&
+        c.sql.includes("plainto_tsquery")
+    );
+    expect(searchCall.sql).not.toMatch(/metadata->>'kind'/);
+  });
+});