@mastra/core 1.31.0-alpha.3 → 1.31.0-alpha.5

package/dist/{chunk-5AFU6MDJ.js → chunk-F4PWGVHW.js}

@@ -4,6 +4,7 @@ import { MastraBase } from './chunk-WENZPAHS.js';
 import { RegisteredLogger } from './chunk-DBBWTK24.js';
 import * as nodePath from 'path';
 import nodePath__default, { parse, join, dirname } from 'path';
+import pMap, { pMapSkip } from 'p-map';
 import posixPath from 'path/posix';
 import { realpathSync, constants, existsSync } from 'fs';
 import * as fs2 from 'fs/promises';
@@ -4638,8 +4639,10 @@ var BM25Index = class _BM25Index {
     return idf * (numerator / denominator);
   }
 };
-
-// src/workspace/search/search-engine.ts
+function isBatchEmbedder(embedder) {
+  return typeof embedder === "function" && embedder.batch === true;
+}
+var DEFAULT_INDEX_MANY_CONCURRENCY = 8;
 var DEFAULT_MAX_CHUNK_CHARS = 4e3;
 var DEFAULT_OVERLAP_LINES = 3;
 function splitIntoChunks(text, options = {}) {
@@ -4731,12 +4734,15 @@ var SearchEngine = class {
     }
   }
   /**
-   * Index multiple documents
+   * Index multiple documents (up to `concurrency` at a time when async vector work runs).
+   *
+   * @param docs - Documents to index
+   * @param options - `p-map` options; `concurrency` defaults to 8
    */
-  async indexMany(docs) {
-    for (const doc of docs) {
-      await this.index(doc);
-    }
+  async indexMany(docs, options) {
+    const stopOnError = options?.stopOnError;
+    const concurrency = options?.concurrency ?? DEFAULT_INDEX_MANY_CONCURRENCY;
+    await pMap(docs, (doc) => this.index(doc), { stopOnError, concurrency });
   }
   /**
    * Remove a document from the index
@@ -4887,12 +4893,63 @@ var SearchEngine = class {
     throw new Error("No search configuration available. Provide bm25 or vector config.");
   }
   /**
-   * Index a single document in the vector store
+   * Embed a single text, dispatching to the batch path with a one-element array
+   * when the configured embedder is batch-capable.
+   */
+  async #embedOne(text) {
+    if (!this.#vectorConfig) {
+      throw new Error("Vector configuration is required to embed text.");
+    }
+    const { embedder } = this.#vectorConfig;
+    if (isBatchEmbedder(embedder)) {
+      const [embedding] = await embedder([text]);
+      if (!embedding) {
+        throw new Error("Batch embedder returned no embedding for input text.");
+      }
+      return embedding;
+    }
+    return embedder(text);
+  }
+  /**
+   * Embed many texts. Uses a single batched call (chunked by `maxBatchSize`)
+   * when the embedder is batch-capable; otherwise falls back to parallel
+   * single-text calls.
+   */
+  async #embedAll(texts) {
+    if (!this.#vectorConfig) {
+      throw new Error("Vector configuration is required to embed texts.");
+    }
+    if (texts.length === 0) return [];
+    const { embedder } = this.#vectorConfig;
+    if (isBatchEmbedder(embedder)) {
+      const max = embedder.maxBatchSize;
+      if (max === void 0 || texts.length <= max) {
+        return embedder(texts);
+      }
+      const chunks = [];
+      for (let i = 0; i < texts.length; i += max) {
+        chunks.push(texts.slice(i, i + max));
+      }
+      const results = await pMap(chunks, (chunk) => embedder(chunk), {
+        concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+      });
+      return results.flat();
+    }
+    return pMap(texts, (text) => embedder(text), {
+      concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+    });
+  }
+  /**
+   * Index a single document in the vector store.
+   *
+   * Used by the eager (non-lazy) write path. The lazy flush path embeds all
+   * pending docs together via {@link SearchEngine.#flushVectorBatch} for true
+   * batch embedding when supported.
    */
   async #indexVector(doc) {
     if (!this.#vectorConfig) return;
-    const { vectorStore, embedder, indexName } = this.#vectorConfig;
-    const embedding = await embedder(doc.content);
+    const { vectorStore, indexName } = this.#vectorConfig;
+    const embedding = await this.#embedOne(doc.content);
     if (!this.#vectorIndexReady) {
       try {
         await vectorStore.createIndex({ indexName, dimension: embedding.length });
@@ -4914,16 +4971,80 @@ var SearchEngine = class {
     this.#vectorIndexReady = true;
   }
   /**
-   * Ensure vector index is built (for lazy mode)
+   * Embed and upsert a batch of documents in as few provider calls as possible.
+   *
+   * - If the embedder is batch-capable, all texts go through a single embedder
+   *   call (chunked by `maxBatchSize`), then a single `upsert` with all vectors.
+   * - Otherwise falls back to per-doc embedding via {@link SearchEngine.#indexVector}.
+   */
+  async #flushVectorBatch(docs) {
+    if (!this.#vectorConfig || docs.length === 0) return;
+    const { vectorStore, embedder, indexName } = this.#vectorConfig;
+    if (!isBatchEmbedder(embedder)) {
+      await pMap(docs, (doc) => this.#indexVector(doc), {
+        concurrency: DEFAULT_INDEX_MANY_CONCURRENCY
+      });
+      return;
+    }
+    const embeddings = await this.#embedAll(docs.map((d) => d.content));
+    if (embeddings.length !== docs.length) {
+      throw new Error(`Batch embedder returned ${embeddings.length} embeddings for ${docs.length} inputs.`);
+    }
+    if (!this.#vectorIndexReady) {
+      const dim = embeddings[0].length;
+      try {
+        await vectorStore.createIndex({ indexName, dimension: dim });
+      } catch {
+      }
+    }
+    await vectorStore.upsert({
+      indexName,
+      vectors: embeddings,
+      metadata: docs.map((doc) => ({
+        id: doc.id,
+        text: doc.content,
+        ...doc.metadata
+      })),
+      ids: docs.map((d) => d.id)
+    });
+    this.#vectorIndexReady = true;
+  }
+  /**
+   * Collapse duplicate document ids so a single deterministic upsert runs per id (last queue entry wins).
+   */
+  #dedupePendingVectorDocsLastWins(docs) {
+    const byId = /* @__PURE__ */ new Map();
+    for (const doc of docs) {
+      byId.set(doc.id, doc);
+    }
+    return [...byId.values()];
+  }
+  /**
+   * Ensure vector index is built (for lazy mode).
+   *
+   * Drains the pending queue into a local batch before awaiting upserts so concurrent `index()` calls
+   * append to a fresh queue and are not wiped by a blanket clear. Loops until the queue is empty so
+   * documents added mid-flush are indexed before search runs. Re-queues the batch on flush failure.
    */
   async #ensureVectorIndex() {
-    if (!this.#lazyVectorIndex || this.#vectorIndexBuilt || this.#pendingVectorDocs.length === 0) {
+    if (!this.#lazyVectorIndex) {
       return;
     }
-    for (const doc of this.#pendingVectorDocs) {
-      await this.#indexVector(doc);
+    if (this.#pendingVectorDocs.length === 0) {
+      this.#vectorIndexBuilt = true;
+      return;
+    }
+    while (this.#pendingVectorDocs.length > 0) {
+      const batch = this.#pendingVectorDocs;
+      this.#pendingVectorDocs = [];
+      const uniqueDocs = this.#dedupePendingVectorDocsLastWins(batch);
+      try {
+        await this.#flushVectorBatch(uniqueDocs);
+      } catch (error) {
+        this.#pendingVectorDocs = [...uniqueDocs, ...this.#pendingVectorDocs];
+        throw error;
+      }
     }
-    this.#pendingVectorDocs = [];
     this.#vectorIndexBuilt = true;
   }
   /**
@@ -4957,8 +5078,8 @@ var SearchEngine = class {
       throw new Error("Vector search requires vector configuration.");
     }
     await this.#ensureVectorIndex();
-    const { vectorStore, embedder, indexName } = this.#vectorConfig;
-    const queryEmbedding = await embedder(query);
+    const { vectorStore, indexName } = this.#vectorConfig;
+    const queryEmbedding = await this.#embedOne(query);
     const vectorResults = await vectorStore.query({
       indexName,
       queryVector: queryEmbedding,
@@ -6759,6 +6880,7 @@ Available files: ${allFiles.join(", ")}` : "";
 }
 
 // src/workspace/workspace.ts
+var FS_READ_CONCURRENCY = 8;
 var Workspace = class {
   id;
   name;
@@ -7092,20 +7214,15 @@ var Workspace = class {
           const alreadyCovered = directoryRoots.some((root) => entry.path === root || entry.path.startsWith(`${root}/`));
           if (!alreadyCovered) directoryRoots.push(entry.path);
         }
-        for (const filePath of filesToIndex) {
-          if (indexedPaths.has(filePath)) continue;
-          await this.indexFileForSearch(filePath);
-          indexedPaths.add(filePath);
-        }
+        const indexed = await this.indexFilesForSearch(
+          Array.from(filesToIndex).filter((filePath) => !indexedPaths.has(filePath))
+        );
+        for (const filePath of indexed) indexedPaths.add(filePath);
         for (const dir of directoryRoots) {
           try {
-            const files = await this.getAllFiles(dir);
-            for (const filePath of files) {
-              if (!indexedPaths.has(filePath)) {
-                await this.indexFileForSearch(filePath);
-                indexedPaths.add(filePath);
-              }
-            }
+            const files = (await this.getAllFiles(dir)).filter((filePath) => !indexedPaths.has(filePath));
+            const indexed2 = await this.indexFilesForSearch(files);
+            for (const filePath of indexed2) indexedPaths.add(filePath);
           } catch {
           }
         }
@@ -7113,10 +7230,69 @@ var Workspace = class {
       }
     }
   }
+  /**
+   * Load file contents for search indexing in parallel (bounded by {@link FS_READ_CONCURRENCY}).
+   * Paths that cannot be read as UTF-8 text are omitted (same behavior as {@link indexFileForSearch}).
+   */
+  async batchReadFiles(files) {
+    if (!this._fs || files.length === 0) {
+      return [];
+    }
+    const fs6 = this._fs;
+    return pMap(
+      files,
+      async (filePath) => {
+        try {
+          const content = await fs6.readFile(filePath, { encoding: "utf-8" });
+          const chunks = splitIntoChunks(content);
+          const docs = chunks.length === 1 ? [{ id: filePath, content }] : chunks.map((chunk, i) => ({
+            id: `${filePath}#chunk-${i}`,
+            content: chunk.content,
+            startLineOffset: chunk.startLine,
+            metadata: { sourceFile: filePath }
+          }));
+          return { filePath, docs };
+        } catch {
+          return pMapSkip;
+        }
+      },
+      { stopOnError: false, concurrency: FS_READ_CONCURRENCY }
+    );
+  }
+  /**
+   * Batch-read paths and {@link SearchEngine.indexMany}
+   *
+   * @returns paths that were indexed successfully.
+   * @remarks Falls back to one-at-a-time indexing on failure of {@link SearchEngine.indexMany}
+   */
+  async indexFilesForSearch(paths) {
+    const engine = this._searchEngine;
+    if (!engine) return [];
+    try {
+      const entries = await this.batchReadFiles(paths);
+      await pMap(entries, ({ filePath }) => engine.removeSource(filePath), {
+        concurrency: FS_READ_CONCURRENCY
+      });
+      const docs = entries.flatMap(({ docs: docs2 }) => docs2);
+      await engine.indexMany(docs);
+      return entries.map(({ filePath }) => filePath);
+    } catch {
+      const indexed = [];
+      for (const filePath of paths) {
+        const id = await this.indexFileForSearch(filePath);
+        if (id !== void 0) {
+          indexed.push(id);
+        }
+      }
+      return indexed;
+    }
+  }
   /**
    * Index a single file for search. Skips files that can't be read as text.
    * Large files are automatically split into chunks to stay within embedding
    * model token limits.
+   *
+   * @returns `filePath` when indexed, or `undefined` if read/index failed.
    */
   async indexFileForSearch(filePath) {
     let content;
@@ -7130,11 +7306,13 @@ var Workspace = class {
     if (chunks.length === 1) {
       try {
         await this._searchEngine.index({ id: filePath, content });
+        return filePath;
       } catch (error) {
         this._logger?.warn(`Failed to index file "${filePath}" for search`, { error });
+        return;
       }
-      return;
     }
+    let anyIndexed = false;
     for (let i = 0; i < chunks.length; i++) {
       const chunk = chunks[i];
       try {
@@ -7144,10 +7322,12 @@ var Workspace = class {
           startLineOffset: chunk.startLine,
           metadata: { sourceFile: filePath }
         });
+        anyIndexed = true;
       } catch (error) {
         this._logger?.warn(`Failed to index chunk ${i} of file "${filePath}" for search`, { error });
       }
     }
+    return anyIndexed ? filePath : void 0;
   }
   async getAllFiles(dir, depth = 0, maxDepth = 10, filesystem = this._fs) {
     if (!filesystem || depth >= maxDepth) return [];
@@ -9623,5 +9803,5 @@ async function createWorkspaceTools(workspace, configContext) {
 }
 
 export { BM25Index, BrowserCliHandler, CompositeFilesystem, CompositeVersionedSkillSource, DirectoryNotEmptyError, DirectoryNotFoundError, FileExistsError, FileNotFoundError, FileReadRequiredError, FilesystemError, FilesystemNotAvailableError, FilesystemNotMountableError, FilesystemNotReadyError, IsDirectoryError, IsolationUnavailableError, LocalFilesystem, LocalSandbox, LocalSkillSource, MastraFilesystem, MastraSandbox, MountError, MountManager, MountNotSupportedError, NotDirectoryError, PermissionError, ProcessHandle, SandboxError, SandboxExecutionError, SandboxFeatureNotSupportedError, SandboxNotAvailableError, SandboxNotReadyError, SandboxProcessManager, SandboxTimeoutError, SearchNotAvailableError, StaleFileError, VersionedSkillSource, WORKSPACE_TOOLS, WORKSPACE_TOOLS_PREFIX, Workspace, WorkspaceError, WorkspaceNotAvailableError, WorkspaceNotReadyError, WorkspaceReadOnlyError, browserCliHandler, callLifecycle, collectSkillForPublish, createGlobMatcher, createSkillTools, createWorkspaceTools, deleteFileTool, detectIsolation, editFileTool, executeCommandTool, extractGlobBase, fileStatTool, getRecommendedIsolation, getTiktoken, indexContentTool, isGlobPattern, isIsolationAvailable, listFilesTool, matchGlob, mkdirTool, publishSkillFromSource, readFileTool, requireFilesystem, requireSandbox, requireWorkspace, resolveToolConfig, searchTool, writeFileTool };
-//# sourceMappingURL=chunk-5AFU6MDJ.js.map
-//# sourceMappingURL=chunk-5AFU6MDJ.js.map
+//# sourceMappingURL=chunk-F4PWGVHW.js.map
+//# sourceMappingURL=chunk-F4PWGVHW.js.map