@mastra/core 1.31.0-alpha.3 → 1.31.0-alpha.5

package/dist/{chunk-OQ6ORZLY.cjs → chunk-5ANWVE3P.cjs}

@@ -5,6 +5,7 @@ var chunkKF4FCWWL_cjs = require('./chunk-KF4FCWWL.cjs');
 var chunkFCQNDFEW_cjs = require('./chunk-FCQNDFEW.cjs');
 var chunk7GW2TQXP_cjs = require('./chunk-7GW2TQXP.cjs');
 var nodePath = require('path');
+var pMap = require('p-map');
 var posixPath = require('path/posix');
 var fs = require('fs');
 var fs2 = require('fs/promises');
@@ -41,6 +42,7 @@ function _interopNamespace(e) {
 }
 
 var nodePath__namespace = /*#__PURE__*/_interopNamespace(nodePath);
+var pMap__default = /*#__PURE__*/_interopDefault(pMap);
 var posixPath__default = /*#__PURE__*/_interopDefault(posixPath);
 var fs2__namespace = /*#__PURE__*/_interopNamespace(fs2);
 var os3__namespace = /*#__PURE__*/_interopNamespace(os3);
@@ -1658,7 +1660,7 @@ function isLSPAvailable() {
     return jsonrpcModule !== null;
   }
   try {
-    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-OQ6ORZLY.cjs', document.baseURI).href)));
+    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-5ANWVE3P.cjs', document.baseURI).href)));
     req.resolve("vscode-jsonrpc/node");
     req.resolve("vscode-languageserver-protocol");
     return true;
@@ -1672,7 +1674,7 @@ async function loadLSPDeps() {
     return { ...jsonrpcModule, ...lspProtocolModule };
   }
   try {
-    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-OQ6ORZLY.cjs', document.baseURI).href)));
+    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-5ANWVE3P.cjs', document.baseURI).href)));
     const jsonrpc = req("vscode-jsonrpc/node");
     const protocol = req("vscode-languageserver-protocol");
     jsonrpcModule = {
@@ -4666,8 +4668,10 @@ var BM25Index = class _BM25Index {
     return idf * (numerator / denominator);
   }
 };
-
-// src/workspace/search/search-engine.ts
+function isBatchEmbedder(embedder) {
+  return typeof embedder === "function" && embedder.batch === true;
+}
+var DEFAULT_INDEX_MANY_CONCURRENCY = 8;
 var DEFAULT_MAX_CHUNK_CHARS = 4e3;
 var DEFAULT_OVERLAP_LINES = 3;
 function splitIntoChunks(text, options = {}) {
@@ -4759,12 +4763,15 @@ var SearchEngine = class {
     }
   }
   /**
-   * Index multiple documents
+   * Index multiple documents (up to `concurrency` at a time when async vector work runs).
+   *
+   * @param docs - Documents to index
+   * @param options - `p-map` options; `concurrency` defaults to 8
    */
-  async indexMany(docs) {
-    for (const doc of docs) {
-      await this.index(doc);
-    }
+  async indexMany(docs, options) {
+    const stopOnError = options?.stopOnError;
+    const concurrency = options?.concurrency ?? DEFAULT_INDEX_MANY_CONCURRENCY;
+    await pMap__default.default(docs, (doc) => this.index(doc), { stopOnError, concurrency });
   }
   /**
    * Remove a document from the index
@@ -4915,12 +4922,63 @@ var SearchEngine = class {
     throw new Error("No search configuration available. Provide bm25 or vector config.");
   }
   /**
-   * Index a single document in the vector store
+   * Embed a single text, dispatching to the batch path with a one-element array
+   * when the configured embedder is batch-capable.
+   */
+  async #embedOne(text) {
+    if (!this.#vectorConfig) {
+      throw new Error("Vector configuration is required to embed text.");
+    }
+    const { embedder } = this.#vectorConfig;
+    if (isBatchEmbedder(embedder)) {
+      const [embedding] = await embedder([text]);
+      if (!embedding) {
+        throw new Error("Batch embedder returned no embedding for input text.");
+      }
+      return embedding;
+    }
+    return embedder(text);
+  }
+  /**
+   * Embed many texts. Uses a single batched call (chunked by `maxBatchSize`)
+   * when the embedder is batch-capable; otherwise falls back to parallel
+   * single-text calls.
+   */
+  async #embedAll(texts) {
+    if (!this.#vectorConfig) {
+      throw new Error("Vector configuration is required to embed texts.");
+    }
+    if (texts.length === 0) return [];
+    const { embedder } = this.#vectorConfig;
+    if (isBatchEmbedder(embedder)) {
+      const max = embedder.maxBatchSize;
+      if (max === void 0 || texts.length <= max) {
+        return embedder(texts);
+      }
+      const chunks = [];
+      for (let i = 0; i < texts.length; i += max) {
+        chunks.push(texts.slice(i, i + max));
+      }
+      const results = await pMap__default.default(chunks, (chunk) => embedder(chunk), {
+        concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+      });
+      return results.flat();
+    }
+    return pMap__default.default(texts, (text) => embedder(text), {
+      concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+    });
+  }
+  /**
+   * Index a single document in the vector store.
+   *
+   * Used by the eager (non-lazy) write path. The lazy flush path embeds all
+   * pending docs together via {@link SearchEngine.#flushVectorBatch} for true
+   * batch embedding when supported.
    */
   async #indexVector(doc) {
     if (!this.#vectorConfig) return;
-    const { vectorStore, embedder, indexName } = this.#vectorConfig;
-    const embedding = await embedder(doc.content);
+    const { vectorStore, indexName } = this.#vectorConfig;
+    const embedding = await this.#embedOne(doc.content);
     if (!this.#vectorIndexReady) {
       try {
         await vectorStore.createIndex({ indexName, dimension: embedding.length });
@@ -4942,16 +5000,80 @@ var SearchEngine = class {
     this.#vectorIndexReady = true;
   }
   /**
-   * Ensure vector index is built (for lazy mode)
+   * Embed and upsert a batch of documents in as few provider calls as possible.
+   *
+   * - If the embedder is batch-capable, all texts go through a single embedder
+   *   call (chunked by `maxBatchSize`), then a single `upsert` with all vectors.
+   * - Otherwise falls back to per-doc embedding via {@link SearchEngine.#indexVector}.
+   */
+  async #flushVectorBatch(docs) {
+    if (!this.#vectorConfig || docs.length === 0) return;
+    const { vectorStore, embedder, indexName } = this.#vectorConfig;
+    if (!isBatchEmbedder(embedder)) {
+      await pMap__default.default(docs, (doc) => this.#indexVector(doc), {
+        concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+      });
+      return;
+    }
+    const embeddings = await this.#embedAll(docs.map((d) => d.content));
+    if (embeddings.length !== docs.length) {
+      throw new Error(`Batch embedder returned ${embeddings.length} embeddings for ${docs.length} inputs.`);
+    }
+    if (!this.#vectorIndexReady) {
+      const dim = embeddings[0].length;
+      try {
+        await vectorStore.createIndex({ indexName, dimension: dim });
+      } catch {
+      }
+    }
+    await vectorStore.upsert({
+      indexName,
+      vectors: embeddings,
+      metadata: docs.map((doc) => ({
+        id: doc.id,
+        text: doc.content,
+        ...doc.metadata
+      })),
+      ids: docs.map((d) => d.id)
+    });
+    this.#vectorIndexReady = true;
+  }
+  /**
+   * Collapse duplicate document ids so a single deterministic upsert runs per id (last queue entry wins).
+   */
+  #dedupePendingVectorDocsLastWins(docs) {
+    const byId = /* @__PURE__ */ new Map();
+    for (const doc of docs) {
+      byId.set(doc.id, doc);
+    }
+    return [...byId.values()];
+  }
+  /**
+   * Ensure vector index is built (for lazy mode).
+   *
+   * Drains the pending queue into a local batch before awaiting upserts so concurrent `index()` calls
+   * append to a fresh queue and are not wiped by a blanket clear. Loops until the queue is empty so
+   * documents added mid-flush are indexed before search runs. Re-queues the batch on flush failure.
    */
   async #ensureVectorIndex() {
-    if (!this.#lazyVectorIndex || this.#vectorIndexBuilt || this.#pendingVectorDocs.length === 0) {
+    if (!this.#lazyVectorIndex) {
       return;
     }
-    for (const doc of this.#pendingVectorDocs) {
-      await this.#indexVector(doc);
+    if (this.#pendingVectorDocs.length === 0) {
+      this.#vectorIndexBuilt = true;
+      return;
+    }
+    while (this.#pendingVectorDocs.length > 0) {
+      const batch = this.#pendingVectorDocs;
+      this.#pendingVectorDocs = [];
+      const uniqueDocs = this.#dedupePendingVectorDocsLastWins(batch);
+      try {
+        await this.#flushVectorBatch(uniqueDocs);
+      } catch (error) {
+        this.#pendingVectorDocs = [...uniqueDocs, ...this.#pendingVectorDocs];
+        throw error;
+      }
     }
-    this.#pendingVectorDocs = [];
     this.#vectorIndexBuilt = true;
   }
   /**
@@ -4985,8 +5107,8 @@ var SearchEngine = class {
       throw new Error("Vector search requires vector configuration.");
     }
     await this.#ensureVectorIndex();
-    const { vectorStore, embedder, indexName } = this.#vectorConfig;
-    const queryEmbedding = await embedder(query);
+    const { vectorStore, indexName } = this.#vectorConfig;
+    const queryEmbedding = await this.#embedOne(query);
     const vectorResults = await vectorStore.query({
       indexName,
       queryVector: queryEmbedding,
@@ -6787,6 +6909,7 @@ Available files: ${allFiles.join(", ")}` : "";
 }
 
 // src/workspace/workspace.ts
+var FS_READ_CONCURRENCY = 8;
 var Workspace = class {
   id;
   name;
@@ -7120,20 +7243,15 @@ var Workspace = class {
           const alreadyCovered = directoryRoots.some((root) => entry.path === root || entry.path.startsWith(`${root}/`));
           if (!alreadyCovered) directoryRoots.push(entry.path);
         }
-        for (const filePath of filesToIndex) {
-          if (indexedPaths.has(filePath)) continue;
-          await this.indexFileForSearch(filePath);
-          indexedPaths.add(filePath);
-        }
+        const indexed = await this.indexFilesForSearch(
+          Array.from(filesToIndex).filter((filePath) => !indexedPaths.has(filePath))
+        );
+        for (const filePath of indexed) indexedPaths.add(filePath);
         for (const dir of directoryRoots) {
           try {
-            const files = await this.getAllFiles(dir);
-            for (const filePath of files) {
-              if (!indexedPaths.has(filePath)) {
-                await this.indexFileForSearch(filePath);
-                indexedPaths.add(filePath);
-              }
-            }
+            const files = (await this.getAllFiles(dir)).filter((filePath) => !indexedPaths.has(filePath));
+            const indexed2 = await this.indexFilesForSearch(files);
+            for (const filePath of indexed2) indexedPaths.add(filePath);
           } catch {
           }
         }
@@ -7141,10 +7259,69 @@ var Workspace = class {
       }
     }
   }
+  /**
+   * Load file contents for search indexing in parallel (bounded by {@link FS_READ_CONCURRENCY}).
+   * Paths that cannot be read as UTF-8 text are omitted (same behavior as {@link indexFileForSearch}).
+   */
+  async batchReadFiles(files) {
+    if (!this._fs || files.length === 0) {
+      return [];
+    }
+    const fs6 = this._fs;
+    return pMap__default.default(
+      files,
+      async (filePath) => {
+        try {
+          const content = await fs6.readFile(filePath, { encoding: "utf-8" });
+          const chunks = splitIntoChunks(content);
+          const docs = chunks.length === 1 ? [{ id: filePath, content }] : chunks.map((chunk, i) => ({
+            id: `${filePath}#chunk-${i}`,
+            content: chunk.content,
+            startLineOffset: chunk.startLine,
+            metadata: { sourceFile: filePath }
+          }));
+          return { filePath, docs };
+        } catch {
+          return pMap.pMapSkip;
+        }
+      },
+      { stopOnError: false, concurrency: FS_READ_CONCURRENCY }
+    );
+  }
+  /**
+   * Batch-read paths and {@link SearchEngine.indexMany}
+   *
+   * @returns paths that were indexed successfully.
+   * @remarks Falls back to one-at-a-time indexing on failure of {@link SearchEngine.indexMany}
+   */
+  async indexFilesForSearch(paths) {
+    const engine = this._searchEngine;
+    if (!engine) return [];
+    try {
+      const entries = await this.batchReadFiles(paths);
+      await pMap__default.default(entries, ({ filePath }) => engine.removeSource(filePath), {
+        concurrency: FS_READ_CONCURRENCY
+      });
+      const docs = entries.flatMap(({ docs: docs2 }) => docs2);
+      await engine.indexMany(docs);
+      return entries.map(({ filePath }) => filePath);
+    } catch {
+      const indexed = [];
+      for (const filePath of paths) {
+        const id = await this.indexFileForSearch(filePath);
+        if (id !== void 0) {
+          indexed.push(id);
+        }
+      }
+      return indexed;
+    }
+  }
   /**
    * Index a single file for search. Skips files that can't be read as text.
    * Large files are automatically split into chunks to stay within embedding
    * model token limits.
+   *
+   * @returns `filePath` when indexed, or `undefined` if read/index failed.
    */
   async indexFileForSearch(filePath) {
     let content;
@@ -7158,11 +7335,13 @@ var Workspace = class {
     if (chunks.length === 1) {
       try {
         await this._searchEngine.index({ id: filePath, content });
+        return filePath;
       } catch (error) {
         this._logger?.warn(`Failed to index file "${filePath}" for search`, { error });
+        return;
       }
-      return;
     }
+    let anyIndexed = false;
     for (let i = 0; i < chunks.length; i++) {
       const chunk = chunks[i];
       try {
@@ -7172,10 +7351,12 @@ var Workspace = class {
           startLineOffset: chunk.startLine,
           metadata: { sourceFile: filePath }
         });
+        anyIndexed = true;
       } catch (error) {
         this._logger?.warn(`Failed to index chunk ${i} of file "${filePath}" for search`, { error });
       }
     }
+    return anyIndexed ? filePath : void 0;
   }
   async getAllFiles(dir, depth = 0, maxDepth = 10, filesystem = this._fs) {
     if (!filesystem || depth >= maxDepth) return [];
@@ -7527,7 +7708,7 @@ function isAstGrepAvailable() {
     return astGrepModule !== null;
   }
   try {
-    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-OQ6ORZLY.cjs', document.baseURI).href)));
+    const req = module$1.createRequire((typeof document === 'undefined' ? require('u' + 'rl').pathToFileURL(__filename).href : (_documentCurrentScript && _documentCurrentScript.tagName.toUpperCase() === 'SCRIPT' && _documentCurrentScript.src || new URL('chunk-5ANWVE3P.cjs', document.baseURI).href)));
     req.resolve("@ast-grep/napi");
     return true;
   } catch {
@@ -9721,5 +9902,5 @@ exports.requireWorkspace = requireWorkspace;
 exports.resolveToolConfig = resolveToolConfig;
 exports.searchTool = searchTool;
 exports.writeFileTool = writeFileTool;
-//# sourceMappingURL=chunk-OQ6ORZLY.cjs.map
-//# sourceMappingURL=chunk-OQ6ORZLY.cjs.map
+//# sourceMappingURL=chunk-5ANWVE3P.cjs.map
+//# sourceMappingURL=chunk-5ANWVE3P.cjs.map