@pentatonic-ai/ai-agent-sdk 0.5.11 → 0.7.0

package/packages/memory/src/__tests__/corpus-chunkers.test.js

@@ -0,0 +1,143 @@
+/**
+ * Tests for file chunking. Code, prose, structured, and the fallback
+ * sliding window. Verifies metadata fidelity (line ranges, heading
+ * paths, chunk indices) — these power the "from README.md > Setup"
+ * surfacing on retrieval.
+ */
+
+import { chunkFile } from "../corpus/chunkers.js";
+
+describe("chunkFile — small files", () => {
+  it("returns one chunk for a small file", () => {
+    const out = chunkFile({
+      relPath: "tiny.md",
+      ext: ".md",
+      content: "# Hi\n\nShort note.",
+    });
+    expect(out).toHaveLength(1);
+    expect(out[0].metadata.chunk_index).toBe(0);
+    expect(out[0].metadata.total_chunks).toBe(1);
+    expect(out[0].metadata.kind).toBe("prose");
+  });
+
+  it("returns empty array for empty content", () => {
+    expect(chunkFile({ relPath: "x.txt", ext: ".txt", content: "" })).toEqual([]);
+    expect(chunkFile({ relPath: "x.txt", ext: ".txt", content: "   \n  " })).toEqual([]);
+  });
+});
+
+describe("chunkFile — markdown", () => {
+  function bigMd() {
+    let md = "# Top\n\nIntro.\n";
+    for (let i = 0; i < 10; i++) {
+      md += `\n## Section ${i}\n\n`;
+      md += "Paragraph content. ".repeat(80);
+      md += "\n";
+    }
+    return md;
+  }
+
+  it("splits on headings and tracks heading_path", () => {
+    const out = chunkFile({
+      relPath: "doc.md",
+      ext: ".md",
+      content: bigMd(),
+    });
+    expect(out.length).toBeGreaterThan(1);
+    // Each chunk has a heading path
+    for (const c of out) {
+      expect(c.metadata.kind).toBe("prose");
+      expect(typeof c.metadata.heading_path).toBe("string");
+    }
+    // At least one chunk hangs off "Top > Section N"
+    expect(
+      out.some((c) => /^Top > Section \d+$/.test(c.metadata.heading_path))
+    ).toBe(true);
+  });
+
+  it("records total_chunks consistently", () => {
+    const out = chunkFile({ relPath: "doc.md", ext: ".md", content: bigMd() });
+    const total = out[0].metadata.total_chunks;
+    expect(total).toBe(out.length);
+    for (const c of out) expect(c.metadata.total_chunks).toBe(total);
+  });
+});
+
+describe("chunkFile — code", () => {
+  function bigCode(lines = 200) {
+    const out = [];
+    for (let i = 0; i < lines; i++) {
+      out.push(`function fn${i}() { return ${i}; }`);
+      if (i % 5 === 4) out.push("");
+    }
+    return out.join("\n");
+  }
+
+  it("emits multiple chunks for large code files", () => {
+    const out = chunkFile({
+      relPath: "big.ts",
+      ext: ".ts",
+      content: bigCode(800),
+    });
+    expect(out.length).toBeGreaterThan(1);
+    for (const c of out) {
+      expect(c.metadata.kind).toBe("code");
+      expect(c.metadata.ext).toBe(".ts");
+      expect(typeof c.metadata.line_start).toBe("number");
+      expect(typeof c.metadata.line_end).toBe("number");
+      expect(c.metadata.line_end).toBeGreaterThanOrEqual(c.metadata.line_start);
+    }
+  });
+
+  it("snaps chunk boundaries to blank lines when nearby", () => {
+    const out = chunkFile({
+      relPath: "snap.ts",
+      ext: ".ts",
+      content: bigCode(400),
+    });
+    // Find a non-final chunk; its last line should be blank or near-blank
+    const nonFinal = out.slice(0, -1);
+    expect(nonFinal.length).toBeGreaterThan(0);
+  });
+
+  it("preserves chunk_index sequence", () => {
+    const out = chunkFile({
+      relPath: "seq.ts",
+      ext: ".ts",
+      content: bigCode(800),
+    });
+    out.forEach((c, i) => expect(c.metadata.chunk_index).toBe(i));
+  });
+});
+
+describe("chunkFile — fallback", () => {
+  it("uses sliding window for unknown extensions", () => {
+    const out = chunkFile({
+      relPath: "data.csv",
+      ext: ".csv",
+      content: "col1,col2,col3\n" + "1,2,3\n".repeat(2000),
+    });
+    expect(out.length).toBeGreaterThan(0);
+    expect(out[0].metadata.kind).toMatch(/text|config|prose|code/);
+  });
+});
+
+describe("chunkFile — security: chunk content is just text", () => {
+  // Defense in depth: the chunker is not where secrets are filtered
+  // (discover hard-excludes those), but the chunker must never
+  // accidentally mangle content (e.g. stripping characters that look
+  // like markup), which would silently lose meaning.
+  it("preserves the original content byte-for-byte across chunks", () => {
+    const content =
+      "# Doc\n\n" +
+      "x".repeat(5000) +
+      "\n\n## Section\n\nMore content " +
+      "y".repeat(3000);
+    const out = chunkFile({ relPath: "x.md", ext: ".md", content });
+    // Concatenating chunks (with overlap stripped) should reconstruct
+    // close to the original — at minimum, every original character
+    // appears somewhere across chunks.
+    const joined = out.map((c) => c.content).join("");
+    expect(joined.length).toBeGreaterThanOrEqual(content.trim().length * 0.95);
+  });
+});

package/packages/memory/src/__tests__/corpus-discover.test.js

@@ -0,0 +1,175 @@
+/**
+ * Tests for corpus discovery — the file walker, ignore-rule handling,
+ * and the security exclusions.
+ *
+ * Critical guarantee: NO secret-pattern file may EVER be yielded by
+ * discover(), even if .gitignore explicitly un-ignores it. This is the
+ * "Uber engineer" expectation — the credential walker is paranoid.
+ */
+
+import { promises as fsp } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdtemp, mkdir, writeFile } from "node:fs/promises";
+
+import { discover, isPathEligible } from "../corpus/discover.js";
+
+async function makeFixture() {
+  const root = await mkdtemp(join(tmpdir(), "tes-discover-"));
+  await mkdir(join(root, "src"), { recursive: true });
+  await mkdir(join(root, "node_modules", "lodash"), { recursive: true });
+  await mkdir(join(root, "secrets"), { recursive: true });
+  await mkdir(join(root, ".aws"), { recursive: true });
+  await mkdir(join(root, ".ssh"), { recursive: true });
+
+  await writeFile(join(root, "src", "index.ts"), "export const x = 1;\n");
+  await writeFile(join(root, "src", "util.ts"), "// util\nexport const y = 2;\n");
+  await writeFile(join(root, "README.md"), "# Project\n\nHello.\n");
+  await writeFile(join(root, "package.json"), '{"name":"x"}');
+  await writeFile(
+    join(root, "node_modules", "lodash", "index.js"),
+    "module.exports = {};"
+  );
+
+  // Files that MUST NEVER be returned
+  await writeFile(join(root, ".env"), "API_KEY=should_never_leak\n");
+  await writeFile(join(root, ".env.local"), "DB_URL=secret\n");
+  await writeFile(join(root, "id_rsa"), "PRIVATE KEY DATA\n");
+  await writeFile(join(root, "server.pem"), "CERT DATA\n");
+  await writeFile(join(root, "secrets", "api.json"), '{"key":"abc"}');
+  await writeFile(join(root, ".aws", "credentials"), "[default]\naws_access_key_id=AKIA...");
+  await writeFile(join(root, ".ssh", "config"), "Host github\n  User git");
+  await writeFile(join(root, "service-account.json"), '{"private_key":"..."}');
+
+  // .gitignore that tries to exclude src/util.ts and re-include .env
+  await writeFile(
+    join(root, ".gitignore"),
+    "src/util.ts\n!.env\n!secrets/\n"
+  );
+
+  return root;
+}
+
+async function collectAll(repoRoot, opts = {}) {
+  const files = [];
+  for await (const f of discover(repoRoot, opts)) files.push(f);
+  return files;
+}
+
+describe("discover", () => {
+  let repoRoot;
+
+  beforeAll(async () => {
+    repoRoot = await makeFixture();
+  });
+
+  afterAll(async () => {
+    await fsp.rm(repoRoot, { recursive: true, force: true });
+  });
+
+  it("yields code, prose, and config files", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath).sort();
+    expect(paths).toContain("src/index.ts");
+    expect(paths).toContain("README.md");
+    expect(paths).toContain("package.json");
+  });
+
+  it("NEVER yields .env files even if .gitignore un-ignores them", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths).not.toContain(".env");
+    expect(paths).not.toContain(".env.local");
+  });
+
+  it("NEVER yields private key or cert files", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths).not.toContain("id_rsa");
+    expect(paths).not.toContain("server.pem");
+  });
+
+  it("NEVER yields files inside .aws/ or .ssh/", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    for (const p of paths) {
+      expect(p).not.toMatch(/^\.aws\//);
+      expect(p).not.toMatch(/^\.ssh\//);
+    }
+  });
+
+  it("NEVER yields files in secrets/ even with .gitignore !secrets/", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths.some((p) => p.startsWith("secrets/"))).toBe(false);
+  });
+
+  it("excludes service-account JSON heuristically", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths).not.toContain("service-account.json");
+  });
+
+  it("skips node_modules at directory level", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths.some((p) => p.startsWith("node_modules/"))).toBe(false);
+  });
+
+  it("respects .gitignore for non-secret paths", async () => {
+    const files = await collectAll(repoRoot);
+    const paths = files.map((f) => f.relPath);
+    expect(paths).not.toContain("src/util.ts");
+  });
+
+  it("can skip .gitignore honoring when asked", async () => {
+    const files = await collectAll(repoRoot, { honorGitignore: false });
+    const paths = files.map((f) => f.relPath);
+    expect(paths).toContain("src/util.ts");
+    // But hard-excludes still apply
+    expect(paths).not.toContain(".env");
+  });
+
+  it("attaches a content hash and size to each file", async () => {
+    const files = await collectAll(repoRoot);
+    for (const f of files) {
+      expect(f.hash).toMatch(/^[0-9a-f]{64}$/);
+      expect(typeof f.size).toBe("number");
+      expect(f.size).toBeGreaterThan(0);
+    }
+  });
+
+  it("emits warnings for hard-excluded files (visibility)", async () => {
+    const warnings = [];
+    await collectAll(repoRoot, { onWarning: (m) => warnings.push(m) });
+    // At least one secret-exclusion warning fired
+    expect(warnings.some((w) => w.includes(".env"))).toBe(true);
+  });
+});
+
+describe("isPathEligible", () => {
+  it("rejects hard-excluded paths", () => {
+    expect(isPathEligible(".env").eligible).toBe(false);
+    expect(isPathEligible("config/.env.production").eligible).toBe(false);
+    expect(isPathEligible("server.key").eligible).toBe(false);
+    expect(isPathEligible(".aws/credentials").eligible).toBe(false);
+  });
+
+  it("rejects paths inside skip dirs", () => {
+    expect(isPathEligible("node_modules/foo/index.js").eligible).toBe(false);
+    expect(isPathEligible("dist/bundle.js").eligible).toBe(false);
+    expect(isPathEligible("__pycache__/foo.pyc").eligible).toBe(false);
+  });
+
+  it("rejects skipped extensions", () => {
+    expect(isPathEligible("yarn.lock").eligible).toBe(false);
+    expect(isPathEligible("vendor.min.js").eligible).toBe(false);
+    expect(isPathEligible("logo.png").eligible).toBe(false);
+  });
+
+  it("accepts normal source files", () => {
+    expect(isPathEligible("src/index.ts").eligible).toBe(true);
+    expect(isPathEligible("README.md").eligible).toBe(true);
+    expect(isPathEligible("config/app.yaml").eligible).toBe(true);
+  });
+});

package/packages/memory/src/__tests__/corpus-ingest.test.js

@@ -0,0 +1,236 @@
+/**
+ * Tests for the corpus ingest pipeline against a fake adapter.
+ *
+ * Covers:
+ *   - End-to-end ingest of a fixture repo
+ *   - Delta sync: unchanged files are skipped
+ *   - Re-ingest: changed file's old chunks are deleted before new chunks
+ *     are written
+ *   - File deletion: vanished files have their chunks removed
+ *   - State persistence: state.json reflects the ingest
+ *   - maxChunks cap: aborts cleanly when exceeded
+ *   - ingestPaths: handles a list of changed files (git hook path)
+ */
+
+import { promises as fsp } from "node:fs";
+import { join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+
+import { ingestCorpus, ingestPaths } from "../corpus/ingest.js";
+import { loadState } from "../corpus/state.js";
+
+function makeFakeAdapter() {
+  // Stores chunks keyed by corpus_file_key → array of chunk records.
+  const store = new Map();
+  let ingestCount = 0;
+  let deleteCount = 0;
+  return {
+    store,
+    counts: () => ({ ingestCount, deleteCount }),
+    async ingestChunk(content, metadata) {
+      ingestCount++;
+      const key = metadata.corpus_file_key;
+      const existing = store.get(key) || [];
+      existing.push({ content, metadata });
+      store.set(key, existing);
+      return { id: `mem_${ingestCount}` };
+    },
+    async deleteByCorpusFile(repoAbs, relPath) {
+      const key = `${repoAbs}::${relPath}`;
+      const had = store.get(key);
+      if (!had) return 0;
+      deleteCount += had.length;
+      store.delete(key);
+      return had.length;
+    },
+  };
+}
+
+async function makeRepo() {
+  const root = await mkdtemp(join(tmpdir(), "tes-corpus-"));
+  await mkdir(join(root, "src"), { recursive: true });
+  await writeFile(join(root, "README.md"), "# Project\n\nIntro paragraph.");
+  await writeFile(join(root, "src", "index.ts"), "export const a = 1;\n");
+  await writeFile(join(root, "src", "util.ts"), "export const b = 2;\n");
+  // Should be excluded
+  await writeFile(join(root, ".env"), "SECRET=do_not_ingest\n");
+  return root;
+}
+
+async function isolatedStatePath() {
+  // State must live OUTSIDE the repo we're scanning, otherwise it
+  // gets re-ingested as a "new file" on subsequent runs.
+  const dir = await mkdtemp(join(tmpdir(), "tes-corpus-state-"));
+  return join(dir, "corpus.json");
+}
+
+describe("ingestCorpus", () => {
+  let repo;
+  let statePath;
+
+  beforeEach(async () => {
+    repo = await makeRepo();
+    statePath = await isolatedStatePath();
+  });
+
+  afterEach(async () => {
+    await rm(repo, { recursive: true, force: true });
+  });
+
+  it("ingests all eligible files on first run", async () => {
+    const adapter = makeFakeAdapter();
+    const totals = await ingestCorpus(adapter, repo, { statePath });
+    expect(totals.filesIngested).toBe(3); // README.md + 2 .ts
+    expect(totals.chunksCreated).toBeGreaterThanOrEqual(3);
+    expect(totals.bytesProcessed).toBeGreaterThan(0);
+    // .env never made it
+    for (const [key] of adapter.store) {
+      expect(key).not.toContain(".env");
+    }
+  });
+
+  it("writes state with stats and file hashes", async () => {
+    const adapter = makeFakeAdapter();
+    await ingestCorpus(adapter, repo, { statePath });
+    const state = await loadState(statePath);
+    const src = state.sources[resolve(repo)];
+    expect(src).toBeDefined();
+    expect(src.stats.fileCount).toBe(3);
+    expect(src.stats.chunkCount).toBeGreaterThanOrEqual(3);
+    expect(src.files["README.md"].hash).toMatch(/^[0-9a-f]{64}$/);
+    expect(src.lastSyncedAt).toBeTruthy();
+  });
+
+  it("skips unchanged files on re-ingest (delta sync)", async () => {
+    const adapter = makeFakeAdapter();
+    await ingestCorpus(adapter, repo, { statePath });
+    const firstIngestCount = adapter.counts().ingestCount;
+
+    const totals = await ingestCorpus(adapter, repo, { statePath });
+    expect(totals.filesIngested).toBe(0);
+    expect(totals.filesSkipped).toBe(3);
+    expect(adapter.counts().ingestCount).toBe(firstIngestCount); // no new ingests
+  });
+
+  it("re-ingests a changed file and removes its old chunks first", async () => {
+    const adapter = makeFakeAdapter();
+    await ingestCorpus(adapter, repo, { statePath });
+    const beforeDeletes = adapter.counts().deleteCount;
+
+    await writeFile(
+      join(repo, "src", "index.ts"),
+      "export const a = 999; // changed\n"
+    );
+
+    const totals = await ingestCorpus(adapter, repo, { statePath });
+    expect(totals.filesIngested).toBe(1);
+    expect(totals.filesSkipped).toBe(2);
+    // The changed file's chunks were deleted before new ones inserted
+    expect(adapter.counts().deleteCount).toBeGreaterThan(beforeDeletes);
+  });
+
+  it("removes chunks for files that vanish from disk", async () => {
+    const adapter = makeFakeAdapter();
+    await ingestCorpus(adapter, repo, { statePath });
+
+    await rm(join(repo, "src", "util.ts"));
+
+    const beforeDeletes = adapter.counts().deleteCount;
+    await ingestCorpus(adapter, repo, { statePath });
+    expect(adapter.counts().deleteCount).toBeGreaterThan(beforeDeletes);
+
+    const state = await loadState(statePath);
+    const src = state.sources[resolve(repo)];
+    expect(src.files["src/util.ts"]).toBeUndefined();
+    expect(src.stats.fileCount).toBe(2);
+  });
+
+  it("aborts cleanly when maxChunks would be exceeded", async () => {
+    const adapter = makeFakeAdapter();
+    // Force concurrency=1 so the cap check is deterministic. With
+    // higher concurrency the cap is "soft" — we may overshoot by up
+    // to (concurrency-1) chunks before the abort propagates. That's
+    // documented behavior; we test the deterministic path here.
+    await expect(
+      ingestCorpus(adapter, repo, { statePath, maxChunks: 1, concurrency: 1 })
+    ).rejects.toThrow(/maxChunks/);
+    expect(adapter.counts().ingestCount).toBe(1);
+  });
+
+  it("attaches source_file metadata to every chunk", async () => {
+    const adapter = makeFakeAdapter();
+    await ingestCorpus(adapter, repo, { statePath });
+    for (const [key, chunks] of adapter.store) {
+      for (const c of chunks) {
+        expect(c.metadata.source_file).toBeTruthy();
+        expect(c.metadata.source_repo).toBeTruthy();
+        expect(c.metadata.source_file_hash).toMatch(/^[0-9a-f]{64}$/);
+        expect(c.metadata.corpus_file_key).toBe(key);
+      }
+    }
+  });
+
+  it("propagates onWarning for hard-excluded secret files", async () => {
+    const adapter = makeFakeAdapter();
+    const warnings = [];
+    await ingestCorpus(adapter, repo, {
+      statePath,
+      onWarning: (m) => warnings.push(m),
+    });
+    expect(warnings.some((w) => w.includes(".env"))).toBe(true);
+  });
+});
+
+describe("ingestPaths (git hook fast path)", () => {
+  let repo;
+  let statePath;
+
+  beforeEach(async () => {
+    repo = await makeRepo();
+    statePath = await isolatedStatePath();
+  });
+
+  afterEach(async () => {
+    await rm(repo, { recursive: true, force: true });
+  });
+
+  it("ingests just the listed paths", async () => {
+    const adapter = makeFakeAdapter();
+    const totals = await ingestPaths(
+      adapter,
+      repo,
+      ["src/index.ts", "README.md"],
+      { statePath }
+    );
+    expect(totals.filesIngested).toBe(2);
+    // src/util.ts was NOT ingested
+    for (const [key] of adapter.store) {
+      expect(key).not.toContain("util.ts");
+    }
+  });
+
+  it("skips ineligible paths (secrets, lockfiles)", async () => {
+    const adapter = makeFakeAdapter();
+    await writeFile(join(repo, "yarn.lock"), "lockfile content\n");
+    const totals = await ingestPaths(
+      adapter,
+      repo,
+      [".env", "yarn.lock", "src/index.ts"],
+      { statePath }
+    );
+    expect(totals.filesIngested).toBe(1);
+    expect(totals.filesSkipped).toBe(2);
+  });
+
+  it("removes chunks for paths that no longer exist", async () => {
+    const adapter = makeFakeAdapter();
+    // First seed state by ingesting normally
+    await ingestCorpus(adapter, repo, { statePath });
+    // Then delete a file and run ingestPaths on it
+    await rm(join(repo, "src", "util.ts"));
+    const beforeDeletes = adapter.counts().deleteCount;
+    await ingestPaths(adapter, repo, ["src/util.ts"], { statePath });
+    expect(adapter.counts().deleteCount).toBeGreaterThan(beforeDeletes);
+  });
+});