@pentatonic-ai/ai-agent-sdk 0.5.11 → 0.7.0

package/packages/memory/src/corpus/ingest.js

@@ -0,0 +1,356 @@
+/**
+ * Corpus ingest pipeline.
+ *
+ * Takes a discovered file → chunks it → ingests each chunk via an
+ * adapter. Two adapters ship: `localAdapter` (writes via the existing
+ * memory.ingest path against a Pool) and `hostedAdapter` (emits
+ * STORE_MEMORY events via the existing TES createModuleEvent mutation —
+ * no new server-side schema needed).
+ *
+ * This module knows nothing about HTTP or pg directly — adapters
+ * encapsulate that. Keeps it testable and lets us add e.g. a Cloudflare
+ * Worker adapter later without changing the pipeline.
+ */
+
+import { chunkFile } from "./chunkers.js";
+import { extractReferences } from "./signatures.js";
+import {
+  loadState,
+  saveState,
+  upsertSource,
+  recordFile,
+  forgetFile,
+  recomputeStats,
+} from "./state.js";
+import { discover } from "./discover.js";
+import { resolve, basename } from "node:path";
+
+/**
+ * Adapter contract:
+ *   adapter.ingestChunk(content, metadata) → Promise<{ id, skipped? }>
+ *   adapter.deleteByCorpusFile(repoAbs, relPath) → Promise<number> // chunks removed
+ *
+ * Both methods MUST be safe to call repeatedly with the same args
+ * (idempotent on adapter side or via metadata content_hash).
+ */
+
+/**
+ * Ingest a single repo end-to-end. Walks the tree, chunks each file,
+ * sends chunks through the adapter, and updates corpus state.
+ *
+ * Caller controls concurrency by setting `opts.concurrency` (default 4
+ * — small enough not to swamp local Ollama, large enough to amortize
+ * hosted GraphQL round trips).
+ *
+ * @param {object} adapter - { ingestChunk, deleteByCorpusFile }
+ * @param {string} repoPath - Path to the repo (will be resolved to abs)
+ * @param {object} [opts]
+ * @param {number} [opts.concurrency=4]
+ * @param {number} [opts.maxChunks=100000] - Hard cap per ingest run; abort if exceeded
+ * @param {string} [opts.statePath] - Override state file path (tests)
+ * @param {Function} [opts.onProgress] - ({phase, processed, total, file}) => void
+ * @param {Function} [opts.onWarning] - (msg) => void
+ * @param {string}   [opts.sourceUrl] - Optional git remote URL to record
+ * @param {object}   [opts.discoverOpts] - Forwarded to discover()
+ * @param {"references"|"content"} [opts.mode="references"] - Storage mode.
+ *   "references" (default): store path + signature pointers; the agent
+ *   reads source files at query time. Stale source = `Read` fails →
+ *   loud, self-correcting failure mode.
+ *   "content": store full chunk content (legacy behaviour). Stale chunks
+ *   silently mislead retrieval until re-ingested. Kept for callers who
+ *   explicitly want a self-contained index.
+ * @returns {Promise<{filesProcessed, filesIngested, filesSkipped, chunksCreated, bytesProcessed}>}
+ */
+export async function ingestCorpus(adapter, repoPath, opts = {}) {
+  const repoAbs = resolve(repoPath);
+  const concurrency = opts.concurrency || 4;
+  const maxChunks = opts.maxChunks ?? 100000;
+  const onProgress = opts.onProgress || (() => {});
+  const onWarning = opts.onWarning || (() => {});
+
+  // Default mode is "references" — store pointers, not chunks. See JSDoc
+  // above. Set mode: "content" to opt back into the chunk-content
+  // behaviour for callers who explicitly want a self-contained index
+  // (e.g. air-gapped retrieval where the source isn't readable at
+  // query time).
+  const mode = opts.mode === "content" ? "content" : "references";
+  const extract = mode === "content" ? chunkFile : extractReferences;
+
+  const state = await loadState(opts.statePath);
+  const source = upsertSource(state, repoAbs, {
+    sourceType: "directory",
+    sourceUrl: opts.sourceUrl ?? null,
+  });
+
+  const totals = {
+    filesProcessed: 0,
+    filesIngested: 0,
+    filesSkipped: 0,
+    chunksCreated: 0,
+    bytesProcessed: 0,
+  };
+
+  // Phase 1: discovery (counts files for progress)
+  const queue = [];
+  for await (const file of discover(repoAbs, {
+    ...(opts.discoverOpts || {}),
+    onWarning,
+  })) {
+    queue.push(file);
+  }
+  onProgress({ phase: "discovered", total: queue.length });
+
+  // Phase 2: ingest with bounded concurrency
+  const stillSeen = new Set();
+  let inFlight = 0;
+  let cursor = 0;
+  let aborted = null;
+
+  async function worker() {
+    while (cursor < queue.length && !aborted) {
+      const file = queue[cursor++];
+      stillSeen.add(file.relPath);
+      totals.filesProcessed++;
+
+      const prev = source.files[file.relPath];
+      if (prev && prev.hash === file.hash) {
+        totals.filesSkipped++;
+        onProgress({
+          phase: "file",
+          processed: totals.filesProcessed,
+          total: queue.length,
+          file: file.relPath,
+          status: "unchanged",
+        });
+        continue;
+      }
+
+      // Changed file — drop existing chunks before re-ingesting
+      if (prev) {
+        try {
+          await adapter.deleteByCorpusFile(repoAbs, file.relPath);
+        } catch (err) {
+          onWarning(
+            `corpus: failed to delete stale chunks for ${file.relPath}: ${err.message}`
+          );
+        }
+      }
+
+      const chunks = extract(file);
+      if (totals.chunksCreated + chunks.length > maxChunks) {
+        aborted = new Error(
+          `corpus: maxChunks (${maxChunks}) exceeded — stopped at ${file.relPath}`
+        );
+        break;
+      }
+
+      let chunksCreatedHere = 0;
+      for (const chunk of chunks) {
+        // Per-chunk cap check — concurrency-safe because totals.chunksCreated
+        // is incremented inside the loop atomically (single-threaded JS).
+        if (totals.chunksCreated >= maxChunks) {
+          aborted = new Error(
+            `corpus: maxChunks (${maxChunks}) reached — stopped at ${file.relPath}`
+          );
+          break;
+        }
+        const metadata = {
+          ...chunk.metadata,
+          source_repo: repoAbs,
+          source_repo_name: basename(repoAbs),
+          source_file: file.relPath,
+          source_file_hash: file.hash,
+          corpus_file_key: `${repoAbs}::${file.relPath}`,
+        };
+        try {
+          const result = await adapter.ingestChunk(chunk.content, metadata);
+          if (!result?.skipped) {
+            chunksCreatedHere++;
+            totals.chunksCreated++;
+          }
+        } catch (err) {
+          onWarning(
+            `corpus: ingest failed for ${file.relPath} chunk ${chunk.metadata.chunk_index}: ${err.message}`
+          );
+        }
+      }
+
+      if (chunksCreatedHere > 0) {
+        recordFile(source, file.relPath, file.hash, chunksCreatedHere);
+        totals.filesIngested++;
+        totals.bytesProcessed += file.size;
+      }
+      if (aborted) break;
+
+      onProgress({
+        phase: "file",
+        processed: totals.filesProcessed,
+        total: queue.length,
+        file: file.relPath,
+        status: prev ? "updated" : "ingested",
+        chunks: chunksCreatedHere,
+      });
+    }
+  }
+
+  const workers = Array.from({ length: concurrency }, () => worker());
+  await Promise.all(workers);
+
+  // Phase 3: detect deletions — files in state but no longer on disk
+  const removed = [];
+  for (const relPath of Object.keys(source.files)) {
+    if (!stillSeen.has(relPath)) {
+      try {
+        await adapter.deleteByCorpusFile(repoAbs, relPath);
+        forgetFile(source, relPath);
+        removed.push(relPath);
+      } catch (err) {
+        onWarning(
+          `corpus: failed to delete chunks for vanished ${relPath}: ${err.message}`
+        );
+      }
+    }
+  }
+  if (removed.length) {
+    onProgress({ phase: "removed", count: removed.length });
+  }
+
+  source.lastSyncedAt = new Date().toISOString();
+  recomputeStats(source);
+  await saveState(state, opts.statePath);
+
+  if (aborted) throw aborted;
+
+  return totals;
+}
+
+/**
+ * Delta-sync a known repo. Same as ingestCorpus but useful as a
+ * semantic distinction in the CLI ("resync"). Skips files whose
+ * content hash matches state; deletes chunks for removed files.
+ */
+export async function syncCorpus(adapter, repoPath, opts = {}) {
+  return ingestCorpus(adapter, repoPath, opts);
+}
+
+/**
+ * Ingest a specific list of files (e.g. those changed in a git commit).
+ * Cheaper than walking the whole tree.
+ *
+ * @param {object} adapter
+ * @param {string} repoPath
+ * @param {string[]} relPaths - Paths relative to repoPath
+ * @param {object} [opts]
+ */
+export async function ingestPaths(adapter, repoPath, relPaths, opts = {}) {
+  const repoAbs = resolve(repoPath);
+  const onWarning = opts.onWarning || (() => {});
+  const state = await loadState(opts.statePath);
+  const source = upsertSource(state, repoAbs, {});
+
+  const totals = {
+    filesProcessed: 0,
+    filesIngested: 0,
+    filesSkipped: 0,
+    chunksCreated: 0,
+    bytesProcessed: 0,
+  };
+
+  const { join } = await import("node:path");
+  const { promises: fsp, existsSync } = await import("node:fs");
+  const { createHash } = await import("node:crypto");
+  const { isPathEligible } = await import("./discover.js");
+
+  for (const relPath of relPaths) {
+    totals.filesProcessed++;
+    const eligible = isPathEligible(relPath);
+    const fullPath = join(repoAbs, relPath);
+
+    // File deleted on disk — drop its chunks
+    if (!existsSync(fullPath)) {
+      try {
+        await adapter.deleteByCorpusFile(repoAbs, relPath);
+        forgetFile(source, relPath);
+      } catch (err) {
+        onWarning(`corpus: cleanup failed for ${relPath}: ${err.message}`);
+      }
+      continue;
+    }
+
+    if (!eligible.eligible) {
+      totals.filesSkipped++;
+      continue;
+    }
+
+    let content;
+    try {
+      content = await fsp.readFile(fullPath, "utf-8");
+    } catch (err) {
+      onWarning(`corpus: cannot read ${relPath}: ${err.message}`);
+      continue;
+    }
+    if (content.includes("\0")) {
+      totals.filesSkipped++;
+      continue;
+    }
+
+    const hash = createHash("sha256").update(content).digest("hex");
+    const prev = source.files[relPath];
+    if (prev && prev.hash === hash) {
+      totals.filesSkipped++;
+      continue;
+    }
+
+    if (prev) {
+      try {
+        await adapter.deleteByCorpusFile(repoAbs, relPath);
+      } catch (err) {
+        onWarning(
+          `corpus: failed to delete stale chunks for ${relPath}: ${err.message}`
+        );
+      }
+    }
+
+    const ext = relPath.includes(".")
+      ? "." + relPath.split(".").pop().toLowerCase()
+      : "";
+
+    const chunks = chunkFile({
+      relPath,
+      content,
+      ext,
+      basename: relPath.split("/").pop(),
+    });
+
+    let chunksCreatedHere = 0;
+    for (const chunk of chunks) {
+      const metadata = {
+        ...chunk.metadata,
+        source_repo: repoAbs,
+        source_repo_name: basename(repoAbs),
+        source_file: relPath,
+        source_file_hash: hash,
+        corpus_file_key: `${repoAbs}::${relPath}`,
+      };
+      try {
+        const result = await adapter.ingestChunk(chunk.content, metadata);
+        if (!result?.skipped) chunksCreatedHere++;
+      } catch (err) {
+        onWarning(`corpus: ingest failed for ${relPath}: ${err.message}`);
+      }
+    }
+
+    if (chunksCreatedHere > 0) {
+      recordFile(source, relPath, hash, chunksCreatedHere);
+      totals.filesIngested++;
+      totals.chunksCreated += chunksCreatedHere;
+      totals.bytesProcessed += content.length;
+    }
+  }
+
+  source.lastSyncedAt = new Date().toISOString();
+  recomputeStats(source);
+  await saveState(state, opts.statePath);
+
+  return totals;
+}

package/packages/memory/src/corpus/signatures.js

@@ -0,0 +1,280 @@
+/**
+ * Signature / reference extraction for corpus ingest.
+ *
+ * The default mode for corpus ingest. Stores POINTERS to source content
+ * (path + line range + a short summary) rather than full chunk content.
+ *
+ * Why pointers instead of content?
+ *
+ *   1. Code rots. The repo on disk is the source of truth; an embedded
+ *      chunk goes stale silently the moment a file is edited. Pointers
+ *      "rot loudly" — a `Read` of a moved/deleted/changed file is a
+ *      signal the LLM observes and adjusts to.
+ *
+ *   2. Privacy. Pentatonic-hosted retrieval needs only the signature
+ *      and path, not the full source. Customer code stays on the
+ *      customer's machine; only the index leaves.
+ *
+ *   3. Index size. ~50–200 chars per reference vs ~500–2000 per chunk.
+ *      A 12k-chunk repo becomes a ~3k-reference index, often smaller.
+ *
+ * The extractor is regex-based, not AST. That means it covers ~80% of
+ * common cases and falls back to a single file-level reference for
+ * anything it doesn't recognise. The walker (discover.js) already
+ * filters out generated/binary noise, so the inputs here are
+ * generally well-formed text.
+ *
+ * Per-language strategies:
+ *   - Markdown: per-section reference (`## Heading` + first paragraph)
+ *   - JS/TS:    per top-level `function`/`class`/`export` definition
+ *   - Python:   per `def`/`class` at top level (indent-aware)
+ *   - JSON/YAML: top-level keys as a single reference
+ *   - Other:    single file-level reference
+ *
+ * Each reference shape:
+ *   {
+ *     content,              // text that gets embedded (signature + brief body)
+ *     metadata: {
+ *       kind: 'code_reference',
+ *       path,               // relative path within the repo
+ *       symbol?,            // e.g. function/class name, when extractable
+ *       start_line,         // 1-indexed
+ *       end_line,           // 1-indexed
+ *       language,           // 'markdown' | 'javascript' | 'typescript' | 'python' | 'json' | 'yaml' | 'text'
+ *       lines,              // "<start>-<end>" — convenience for display
+ *     },
+ *   }
+ */
+
+const MD_EXTS = new Set([".md", ".mdx", ".markdown"]);
+const JS_EXTS = new Set([".js", ".jsx", ".mjs", ".cjs"]);
+const TS_EXTS = new Set([".ts", ".tsx"]);
+const PY_EXTS = new Set([".py"]);
+const JSON_EXTS = new Set([".json"]);
+const YAML_EXTS = new Set([".yaml", ".yml"]);
+
+function languageOf(ext) {
+  if (MD_EXTS.has(ext)) return "markdown";
+  if (JS_EXTS.has(ext)) return "javascript";
+  if (TS_EXTS.has(ext)) return "typescript";
+  if (PY_EXTS.has(ext)) return "python";
+  if (JSON_EXTS.has(ext)) return "json";
+  if (YAML_EXTS.has(ext)) return "yaml";
+  return "text";
+}
+
+function ref(content, metadata) {
+  return {
+    content,
+    metadata: {
+      kind: "code_reference",
+      ...metadata,
+      lines: `${metadata.start_line}-${metadata.end_line}`,
+    },
+  };
+}
+
+/**
+ * Markdown: one reference per H1/H2 section. Body of the reference is
+ * the heading + the first paragraph of prose under it (trimmed).
+ */
+function extractMarkdownReferences(file) {
+  const lines = file.content.split(/\r?\n/);
+  const refs = [];
+  let currentHeading = null;
+  let currentStart = 1;
+  let currentBody = [];
+
+  function flush(endLine) {
+    if (!currentHeading) return;
+    const summary = `${currentHeading}\n\n${currentBody.slice(0, 4).join(" ").trim().slice(0, 240)}`;
+    refs.push(
+      ref(summary, {
+        path: file.relPath,
+        symbol: currentHeading.replace(/^#+\s*/, "").trim(),
+        start_line: currentStart,
+        end_line: endLine,
+        language: "markdown",
+      })
+    );
+  }
+
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const headingMatch = line.match(/^(#{1,2})\s+(.+?)\s*$/);
+    if (headingMatch) {
+      flush(i); // close previous section at the line above
+      currentHeading = line.trim();
+      currentStart = i + 1;
+      currentBody = [];
+    } else if (currentHeading) {
+      const t = line.trim();
+      if (t && !t.startsWith("```")) currentBody.push(t);
+    }
+  }
+  flush(lines.length);
+
+  // No headings? Fall back to a single file-level reference.
+  if (refs.length === 0) {
+    refs.push(fileLevelReference(file, "markdown"));
+  }
+  return refs;
+}
+
+/**
+ * JS / TS: per-top-level-symbol references. Regex over `function`,
+ * `class`, `const X =`, and `export ...` declarations at indent 0.
+ * Not perfect (no AST), but catches the vast majority of public-API
+ * surface. Anything we miss falls into a file-level reference.
+ */
+function extractJsLikeReferences(file, language) {
+  const lines = file.content.split(/\r?\n/);
+  const refs = [];
+  // Top-level only — anchored at start of line, optionally with `export`
+  // (and optionally `default` / `async`).
+  const decl =
+    /^(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function\s*\*?\s+([A-Za-z_$][\w$]*)|class\s+([A-Za-z_$][\w$]*)|(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=)/;
+
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i].match(decl);
+    if (!m) continue;
+    const symbol = m[1] || m[2] || m[3];
+    if (!symbol) continue;
+
+    // Body extent — read until next blank line or matching brace count.
+    // Cheap heuristic: capture up to 8 lines or end-of-block.
+    const start = i + 1;
+    const endIdx = Math.min(i + 8, lines.length);
+    const snippet = lines.slice(i, endIdx).join("\n");
+    const summary = `${file.relPath}:${start} — ${symbol}\n${snippet}`.slice(
+      0,
+      400
+    );
+
+    refs.push(
+      ref(summary, {
+        path: file.relPath,
+        symbol,
+        start_line: start,
+        end_line: endIdx,
+        language,
+      })
+    );
+  }
+
+  if (refs.length === 0) {
+    refs.push(fileLevelReference(file, language));
+  }
+  return refs;
+}
+
+/**
+ * Python: per top-level `def`/`class`. Indent 0 only — methods of a
+ * class are not separately yielded; the class entry covers them.
+ */
+function extractPythonReferences(file) {
+  const lines = file.content.split(/\r?\n/);
+  const refs = [];
+  const decl = /^(def|class)\s+([A-Za-z_][\w]*)/;
+
+  for (let i = 0; i < lines.length; i++) {
+    const m = lines[i].match(decl);
+    if (!m) continue;
+    const symbol = m[2];
+    const start = i + 1;
+    const endIdx = Math.min(i + 8, lines.length);
+    const snippet = lines.slice(i, endIdx).join("\n");
+    const summary = `${file.relPath}:${start} — ${symbol}\n${snippet}`.slice(
+      0,
+      400
+    );
+    refs.push(
+      ref(summary, {
+        path: file.relPath,
+        symbol,
+        start_line: start,
+        end_line: endIdx,
+        language: "python",
+      })
+    );
+  }
+  if (refs.length === 0) refs.push(fileLevelReference(file, "python"));
+  return refs;
+}
+
+/**
+ * JSON / YAML: collapse to a single reference whose body is the
+ * top-level keys. Useful as "this config exists and contains X, Y, Z";
+ * the agent reads the file for the actual values.
+ */
+function extractConfigReferences(file, language) {
+  const lines = file.content.split(/\r?\n/);
+  let keys = [];
+
+  if (language === "json") {
+    // Match top-level `"key":` at indent 2 — common json formatting.
+    const seen = new Set();
+    for (const line of lines) {
+      const m = line.match(/^\s{0,4}"([^"]+)"\s*:/);
+      if (m && !seen.has(m[1])) {
+        seen.add(m[1]);
+        keys.push(m[1]);
+      }
+    }
+  } else {
+    // YAML: top-level keys appear at indent 0.
+    for (const line of lines) {
+      const m = line.match(/^([A-Za-z_][\w-]*)\s*:/);
+      if (m) keys.push(m[1]);
+    }
+    keys = [...new Set(keys)];
+  }
+
+  const summary = `${file.relPath} — top-level keys: ${keys.slice(0, 12).join(", ") || "(none extracted)"}`;
+  return [
+    ref(summary, {
+      path: file.relPath,
+      start_line: 1,
+      end_line: lines.length,
+      language,
+    }),
+  ];
+}
+
+function fileLevelReference(file, language) {
+  const lines = file.content.split(/\r?\n/);
+  const head = file.content.slice(0, 240).replace(/\s+/g, " ").trim();
+  const summary = `${file.relPath} — ${head}`;
+  return ref(summary, {
+    path: file.relPath,
+    start_line: 1,
+    end_line: lines.length,
+    language,
+  });
+}
+
+/**
+ * Public entry point. Returns an array of references, each with a
+ * `content` field suitable for embedding and a `metadata` field with
+ * the pointer back to the source.
+ *
+ * @param {{relPath: string, content: string, ext: string}} file
+ * @returns {Array<{content: string, metadata: object}>}
+ */
+export function extractReferences(file) {
+  const language = languageOf(file.ext || "");
+  switch (language) {
+    case "markdown":
+      return extractMarkdownReferences(file);
+    case "javascript":
+    case "typescript":
+      return extractJsLikeReferences(file, language);
+    case "python":
+      return extractPythonReferences(file);
+    case "json":
+    case "yaml":
+      return extractConfigReferences(file, language);
+    default:
+      return [fileLevelReference(file, language)];
+  }
+}