agentlang 0.10.2 → 0.10.3

package/src/language/parser.ts

@@ -1,5 +1,6 @@
 import { createAgentlangServices } from '../language/agentlang-module.js';
 import { AstNode, EmptyFileSystem, LangiumCoreServices, LangiumDocument, URI } from 'langium';
+import { getFormattedErrors, collectErrors, formatErrors } from './error-reporter.js';
 import {
   CrudMap,
   Delete,
@@ -19,12 +20,14 @@ import {
   isPrimExpr,
   isWorkflowDefinition,
   JoinSpec,
+  LimitClause,
   Literal,
   MapEntry,
   MapLiteral,
   ModuleDefinition,
   NegExpr,
   NotExpr,
+  OffsetClause,
   OrderByClause,
   Pattern,
   PrimExpr,
@@ -124,113 +127,25 @@ export async function parseWorkflow(workflowDef: string): Promise<WorkflowDefini
   }
 }
 
-const ErrorIndicator = '<-- ERROR';
-
 export function maybeGetValidationErrors(
   document: LangiumDocument,
-  lines?: string[]
-): string[] | undefined {
-  if (lines === undefined) {
-    lines = document.textDocument.getText().split('\n');
-  }
+  _lines?: string[]
+): string | undefined {
   const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
+  if (validationErrors.length === 0) return undefined;
 
-  const sls = new Set<number>();
-  const scs = new Set<number>();
-  if (validationErrors.length > 0) {
-    for (const validationError of validationErrors) {
-      if (
-        !sls.has(validationError.range.start.line) &&
-        !scs.has(validationError.range.start.character)
-      ) {
-        const t = document.textDocument.getText(validationError.range);
-        const s = `(${validationError.range.start.line + 1}:${validationError.range.start.character + 1}) unexpected token(s) '${t}'`;
-        const ln = lines[validationError.range.start.line];
-        if (ln.indexOf(ErrorIndicator) > 0) {
-          lines[validationError.range.start.line] = `${ln}, ${s}`;
-        } else {
-          lines[validationError.range.start.line] = `${ln}    ${ErrorIndicator} ${s}`;
-        }
-        sls.add(validationError.range.start.line);
-        scs.add(validationError.range.start.character);
-      }
-    }
-    return trimErrorLines(lines);
-  } else {
-    return undefined;
-  }
-}
-
-function trimErrorLines(lines: string[]): string[] {
-  let startidx = 0;
-  for (let i = 0; i < lines.length; ++i) {
-    if (lines[i].indexOf(ErrorIndicator) > 0) {
-      startidx = i;
-      break;
-    }
-  }
-  let endidx = startidx;
-  for (let i = startidx + 1; i < lines.length; ++i) {
-    if (lines[i].indexOf(ErrorIndicator) > 0) {
-      endidx = i;
-      break;
-    }
-  }
-  if (startidx > 0) {
-    --startidx;
-  }
-  if (endidx != lines.length) {
-    ++endidx;
-  }
-  return lines.slice(startidx, endidx);
-}
+  // Collect only validation errors (not lexer/parser)
+  const errors = collectErrors(document).filter(e => e.category === 'VALIDATION ERROR');
+  if (errors.length === 0) return undefined;
 
-function trimErrorMessage(s: string): string {
-  const start = s.indexOf('Expecting:');
-  if (start >= 0) {
-    const end = s.indexOf('but found:');
-    if (end > 0) {
-      return `Expecting a valid token sequence, ${s.substring(end)}`;
-    }
-  }
-  return s;
+  const source = document.textDocument.getText();
+  return formatErrors(errors, source);
 }
 
 export function maybeRaiseParserErrors(document: LangiumDocument) {
-  const code = document.textDocument.getText();
-  const lines = code.split('\n');
-  let hasErrors = false;
-  const errLines = new Set<number>();
-  if (document.parseResult.lexerErrors.length > 0) {
-    document.parseResult.lexerErrors.forEach((err: any) => {
-      if (!errLines.has(err.line)) {
-        const errMsg = trimErrorMessage(err.message);
-        const s = `${ErrorIndicator} (${err.line}:${err.column}) ${errMsg}`;
-        lines[err.line - 1] = `${lines[err.line - 1]}    ${s}`;
-        errLines.add(err.line);
-      }
-    });
-    hasErrors = true;
-  }
-  if (document.parseResult.parserErrors.length > 0) {
-    document.parseResult.parserErrors.forEach((err: any) => {
-      const errMsg = trimErrorMessage(err.message);
-      if (err.token.startLine && err.token.endLine) {
-        if (!errLines.has(err.token.startLine)) {
-          const s = `${ErrorIndicator} (${err.token.startLine}:${err.token.startColumn}) ${errMsg}`;
-          lines[err.token.endLine - 1] = `${lines[err.token.endLine - 1]}    ${s}`;
-          lines.join('\n');
-          errLines.add(err.token.startLine);
-        }
-      } else {
-        lines.push(`ERROR: ${errMsg}`);
-      }
-    });
-    hasErrors = true;
-  }
-  const errs = maybeGetValidationErrors(document, lines);
-  if (hasErrors || errs !== undefined) {
-    throw new Error(lines.join('\n'));
+  const formatted = getFormattedErrors(document);
+  if (formatted) {
+    throw new Error('\n' + formatted);
   }
 }
 
@@ -411,6 +326,12 @@ function introspectQueryPattern(crudMap: CrudMap): CrudPattern {
     if (opts.orderByClause) {
       cp.orderBy = opts.orderByClause.colNames;
     }
+    if (opts.limitClause) {
+      cp.limit = opts.limitClause.value;
+    }
+    if (opts.offsetClause) {
+      cp.offset = opts.offsetClause.value;
+    }
     cp.isCreate = false;
     cp.isQueryUpdate = false;
     cp.isQuery = true;
@@ -425,6 +346,8 @@ export type ExtractedQueryOptions = {
   where: WhereSpec | undefined;
   groupByClause: GroupByClause | undefined;
   orderByClause: OrderByClause | undefined;
+  limitClause: LimitClause | undefined;
+  offsetClause: OffsetClause | undefined;
   upsert: '@upsert' | undefined;
   distinct: '@distinct' | undefined;
 };
@@ -436,6 +359,8 @@ export function extractQueryOptions(crudMap: CrudMap): ExtractedQueryOptions {
     where: undefined,
     groupByClause: undefined,
     orderByClause: undefined,
+    limitClause: undefined,
+    offsetClause: undefined,
     upsert: undefined,
     distinct: undefined,
   };
@@ -453,6 +378,10 @@ export function extractQueryOptions(crudMap: CrudMap): ExtractedQueryOptions {
       r.groupByClause = qo.groupByClause;
     } else if (qo.orderByClause) {
       r.orderByClause = qo.orderByClause;
+    } else if (qo.limitClause) {
+      r.limitClause = qo.limitClause;
+    } else if (qo.offsetClause) {
+      r.offsetClause = qo.offsetClause;
     } else if (qo.upsert) {
       r.upsert = qo.upsert;
     } else if (qo.distinct) {
@@ -596,7 +525,7 @@ export async function introspectCase(caseStr: string): Promise<CasePattern> {
 
 export function canParse(s: string): boolean {
   const ts = s.trim();
-  if (ts) {
+  if (ts && ts.includes('{')) {
     const contents = ts.substring(1, ts.length - 1).trim();
     return contents.length > 0;
   }
@@ -614,6 +543,8 @@ export function objectToQueryPattern(obj: any): string {
         });
       } else if (k === '@groupBy' || k === '@orderBy') {
         strs.push(`${k} ( ${xs.join(', ')} )`);
+      } else if (k === '@limit' || k === '@offset') {
+        strs.push(`${k}(${xs})`);
       } else {
         strs.push(`${k} ${objectToQuerySpecPattern(xs, true)}`);
       }

package/src/language/syntax.ts

@@ -413,6 +413,8 @@ export class CrudPattern extends BasePattern {
   where: WhereSpecClausePattern[] | undefined;
   groupBy: string[] | undefined;
   orderBy: string[] | undefined;
+  limit: number | undefined;
+  offset: number | undefined;
   isQuery: boolean = false;
   isQueryUpdate: boolean = false;
   isCreate: boolean = false;
@@ -592,6 +594,12 @@ export class CrudPattern extends BasePattern {
     if (this.orderBy) {
       s = s.concat(`,\n${this.orderByAsString()}`);
     }
+    if (this.limit !== undefined) {
+      s = s.concat(`,\n@limit(${this.limit})`);
+    }
+    if (this.offset !== undefined) {
+      s = s.concat(`,\n@offset(${this.offset})`);
+    }
 
     return s.concat('}', this.hintsAsString());
   }

package/src/runtime/api.ts

@@ -1,4 +1,5 @@
 import { fetchConfig as al_fetchConfig } from './interpreter.js';
+import { defaultDataSource } from './resolvers/sqldb/database.js';
 import {
   makeInstance as al_makeInstance,
   isInstanceOfType as al_isInstanceOfType,
@@ -6,6 +7,13 @@ import {
 import { getLocalEnv as al_getLocalEnv, setLocalEnv as al_setLocalEnv } from './auth/defs.js';
 import { now } from './util.js';
 import { initDateFns } from './datefns.js';
+import {
+  integrationAuthFetch as al_authFetch,
+  getIntegrationAuthHeaders as al_getAuthHeaders,
+  getOAuthAuthorizeUrl as al_getOAuthAuthorizeUrl,
+  exchangeOAuthCode as al_exchangeOAuthCode,
+  getIntegrationAccessToken as al_getAccessToken,
+} from './integration-client.js';
 
 declare global {
   var agentlang: any | undefined;
@@ -37,6 +45,29 @@ export function initGlobalApi() {
     // Expose date-fns functions globally as dateFns.*
     globalThis.dateFns = initDateFns();
 
+    // Expose credential auth helpers globally
+    globalThis.agentlang.authFetch = al_authFetch;
+    globalThis.agentlang.getAuthHeaders = al_getAuthHeaders;
+
+    // Expose OAuth consent flow helpers globally
+    globalThis.agentlang.getOAuthAuthorizeUrl = al_getOAuthAuthorizeUrl;
+    globalThis.agentlang.exchangeOAuthCode = al_exchangeOAuthCode;
+    globalThis.agentlang.getAccessToken = al_getAccessToken;
+
+    // Expose raw SQL query for resolvers that need direct database access (e.g. pgvector)
+    globalThis.agentlang.rawQuery = async (sql: string, params?: any[]) => {
+      if (!defaultDataSource || !defaultDataSource.isInitialized) {
+        throw new Error('Database not initialized');
+      }
+      return defaultDataSource.query(sql, params);
+    };
+
+    // Expose database type detection for resolvers
+    globalThis.agentlang.getDbType = () => {
+      if (!defaultDataSource || !defaultDataSource.isInitialized) return 'unknown';
+      return defaultDataSource.options.type;
+    };
+
     ApiInited = true;
   }
 }

package/src/runtime/defs.ts

@@ -4,7 +4,8 @@ export const PathAttributeName: string = '__path__';
 export const PathAttributeNameQuery: string = '__path__?';
 export const ParentAttributeName: string = '__parent__';
 export const DeletedFlagAttributeName: string = '__is_deleted__';
-export const TenantAttributeName: string = '__tenant__';
+export const AgentIdAttributeName: string = 'agentId';
+export const TenantAttributeName: string = 'agentId';
 
 export function isPathAttribute(n: string): boolean {
   return n.startsWith(PathAttributeName);

package/src/runtime/document-retriever.ts

@@ -0,0 +1,311 @@
+import { logger } from './logger.js';
+import { AppConfig } from './state.js';
+import { TextChunker } from './embeddings/chunker.js';
+import { OpenAIEmbeddingProvider } from './embeddings/openai.js';
+import { LanceDBVectorStore } from './resolvers/vector/lancedb-store.js';
+import type { VectorStore } from './resolvers/vector/types.js';
+import crypto from 'crypto';
+import { readFileSync } from 'fs';
+import { resolve as pathResolve } from 'path';
+
+const VECTOR_DIMENSION = 1536;
+
+interface LocalChunk {
+  id: string;
+  content: string;
+  documentTitle: string;
+  chunkIndex: number;
+}
+
+function usePgvector(): boolean {
+  if (AppConfig?.vectorStore?.type === 'pgvector') return true;
+  if (AppConfig?.vectorStore?.type === 'lancedb') return false;
+  return AppConfig?.store?.type === 'postgres';
+}
+
+/**
+ * Local document retriever — embeds documents into pgvector or LanceDB
+ * and retrieves relevant chunks via vector similarity search.
+ */
+class DocumentRetriever {
+  private vectorStore: VectorStore | null = null;
+  private embeddingProvider: OpenAIEmbeddingProvider | null = null;
+  private chunker: TextChunker | null = null;
+  private localChunks: Map<string, LocalChunk> = new Map();
+  private processedDocuments: Set<string> = new Set();
+  private initialized = false;
+
+  private async ensureInit(): Promise<void> {
+    if (this.initialized) return;
+
+    this.chunker = new TextChunker(1000, 200);
+    this.embeddingProvider = new OpenAIEmbeddingProvider({
+      model: 'text-embedding-3-small',
+    });
+
+    if (!usePgvector()) {
+      const dbPath =
+        AppConfig?.vectorStore?.type === 'lancedb'
+          ? (AppConfig.vectorStore as any).dbname || './data/document-vectors.lance'
+          : './data/document-vectors.lance';
+
+      this.vectorStore = new LanceDBVectorStore({
+        moduleName: 'documents',
+        vectorDimension: VECTOR_DIMENSION,
+        dbname: dbPath,
+      });
+      await this.vectorStore.init();
+      logger.info(`[DOCUMENT-RETRIEVER] LanceDB vector store initialized at ${dbPath}`);
+    } else {
+      try {
+        const ag = (globalThis as any).agentlang;
+        if (ag?.rawQuery) {
+          await ag.rawQuery(`
+            CREATE TABLE IF NOT EXISTS document_local_chunks (
+              id TEXT PRIMARY KEY,
+              content TEXT NOT NULL,
+              document_title TEXT NOT NULL,
+              chunk_index INTEGER NOT NULL,
+              embedding vector(${VECTOR_DIMENSION})
+            )
+          `);
+          try {
+            await ag.rawQuery(`
+              CREATE INDEX IF NOT EXISTS idx_document_local_chunks_embedding
+              ON document_local_chunks USING hnsw (embedding vector_cosine_ops)
+            `);
+          } catch {
+            // Index may already exist or pgvector extension not loaded
+          }
+          logger.info('[DOCUMENT-RETRIEVER] pgvector local chunks table initialized');
+        }
+      } catch (err) {
+        logger.warn(`[DOCUMENT-RETRIEVER] Failed to initialize pgvector table: ${err}`);
+      }
+    }
+
+    this.initialized = true;
+  }
+
+  async processDocument(title: string, url: string): Promise<void> {
+    if (this.processedDocuments.has(title)) return;
+
+    await this.ensureInit();
+
+    try {
+      let content: string;
+      if (url.startsWith('http://') || url.startsWith('https://')) {
+        const resp = await fetch(url);
+        if (!resp.ok) {
+          logger.warn(
+            `[DOCUMENT-RETRIEVER] Failed to fetch "${title}" from ${url}: ${resp.status}`
+          );
+          return;
+        }
+        content = await resp.text();
+      } else {
+        const filePath = pathResolve(url);
+        try {
+          content = readFileSync(filePath, 'utf-8');
+        } catch (err) {
+          logger.warn(`[DOCUMENT-RETRIEVER] Failed to read "${title}" from ${filePath}: ${err}`);
+          return;
+        }
+      }
+
+      if (!content || content.trim().length === 0) {
+        logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}" is empty, skipping`);
+        this.processedDocuments.add(title);
+        return;
+      }
+
+      const chunks = this.chunker!.splitText(content);
+      logger.debug(`[DOCUMENT-RETRIEVER] Document "${title}": ${chunks.length} chunks`);
+
+      if (chunks.length === 0) {
+        this.processedDocuments.add(title);
+        return;
+      }
+
+      const embeddings = await this.embeddingProvider!.embedTexts(chunks);
+
+      if (usePgvector()) {
+        await this.storePgvectorChunks(title, chunks, embeddings);
+      } else {
+        await this.storeLanceDBChunks(title, chunks, embeddings);
+      }
+
+      this.processedDocuments.add(title);
+      logger.info(
+        `[DOCUMENT-RETRIEVER] Processed "${title}": ${chunks.length} chunks embedded and stored`
+      );
+    } catch (err) {
+      logger.warn(`[DOCUMENT-RETRIEVER] Error processing "${title}": ${err}`);
+    }
+  }
+
+  private async storePgvectorChunks(
+    title: string,
+    chunks: string[],
+    embeddings: number[][]
+  ): Promise<void> {
+    const ag = (globalThis as any).agentlang;
+    if (!ag?.rawQuery) return;
+
+    for (let i = 0; i < chunks.length; i++) {
+      const id = crypto.randomUUID();
+      const embeddingStr = `[${embeddings[i].join(',')}]`;
+      await ag.rawQuery(
+        `INSERT INTO document_local_chunks (id, content, document_title, chunk_index, embedding)
+         VALUES ($1, $2, $3, $4, $5::vector)
+         ON CONFLICT (id) DO NOTHING`,
+        [id, chunks[i], title, i, embeddingStr]
+      );
+      this.localChunks.set(id, {
+        id,
+        content: chunks[i],
+        documentTitle: title,
+        chunkIndex: i,
+      });
+    }
+  }
+
+  private async storeLanceDBChunks(
+    title: string,
+    chunks: string[],
+    embeddings: number[][]
+  ): Promise<void> {
+    if (!this.vectorStore) return;
+
+    for (let i = 0; i < chunks.length; i++) {
+      const id = crypto.randomUUID();
+      await this.vectorStore.addEmbedding({
+        id,
+        embedding: embeddings[i],
+        documentId: title,
+      });
+      this.localChunks.set(id, {
+        id,
+        content: chunks[i],
+        documentTitle: title,
+        chunkIndex: i,
+      });
+    }
+  }
+
+  async query(queryText: string, documentTitles?: string[], limit: number = 10): Promise<string> {
+    await this.ensureInit();
+
+    try {
+      const results = usePgvector()
+        ? await this.queryPgvector(queryText, documentTitles, limit)
+        : await this.queryLanceDB(queryText, documentTitles, limit);
+
+      if (results.length === 0) return '';
+
+      return results.map(r => r.content).join('\n\n---\n\n');
+    } catch (err) {
+      logger.debug(`[DOCUMENT-RETRIEVER] Query failed: ${err}`);
+      return '';
+    }
+  }
+
+  private async queryPgvector(
+    queryText: string,
+    documentTitles?: string[],
+    limit: number = 10
+  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
+    const ag = (globalThis as any).agentlang;
+    if (!ag?.rawQuery) return [];
+
+    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
+    const embeddingStr = `[${queryEmbedding.join(',')}]`;
+
+    let sql: string;
+    let params: any[];
+
+    if (documentTitles && documentTitles.length > 0) {
+      const placeholders = documentTitles.map((_, i) => `$${i + 2}`).join(', ');
+      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
+             FROM document_local_chunks
+             WHERE document_title IN (${placeholders})
+             ORDER BY embedding <=> $1::vector
+             LIMIT ${limit}`;
+      params = [embeddingStr, ...documentTitles];
+    } else {
+      sql = `SELECT id, content, document_title, 1 - (embedding <=> $1::vector) AS similarity
+             FROM document_local_chunks
+             ORDER BY embedding <=> $1::vector
+             LIMIT ${limit}`;
+      params = [embeddingStr];
+    }
+
+    const rows: any[] = await ag.rawQuery(sql, params);
+    return (rows || []).map((r: any) => ({
+      id: r.id,
+      content: r.content,
+      similarity: parseFloat(r.similarity) || 0,
+    }));
+  }
+
+  private async queryLanceDB(
+    queryText: string,
+    documentTitles?: string[],
+    limit: number = 10
+  ): Promise<Array<{ id: string; content: string; similarity: number }>> {
+    if (!this.vectorStore) return [];
+
+    const queryEmbedding = await this.embeddingProvider!.embedText(queryText);
+    const searchResults = await this.vectorStore.search(
+      queryEmbedding,
+      undefined,
+      undefined,
+      limit
+    );
+
+    const results: Array<{ id: string; content: string; similarity: number }> = [];
+
+    for (const sr of searchResults) {
+      const chunk = this.localChunks.get(sr.id);
+      if (!chunk) continue;
+
+      if (documentTitles && documentTitles.length > 0) {
+        if (!documentTitles.includes(chunk.documentTitle)) continue;
+      }
+
+      results.push({
+        id: sr.id,
+        content: chunk.content,
+        similarity: 1 - (sr.distance || 0),
+      });
+    }
+
+    return results.slice(0, limit);
+  }
+
+  async close(): Promise<void> {
+    if (this.vectorStore) {
+      await this.vectorStore.close();
+      this.vectorStore = null;
+    }
+    this.localChunks.clear();
+    this.processedDocuments.clear();
+    this.initialized = false;
+  }
+}
+
+let retrieverInstance: DocumentRetriever | null = null;
+
+export function getDocumentRetriever(): DocumentRetriever {
+  if (!retrieverInstance) {
+    retrieverInstance = new DocumentRetriever();
+  }
+  return retrieverInstance;
+}
+
+export function resetDocumentRetriever(): void {
+  if (retrieverInstance) {
+    retrieverInstance.close().catch(() => {});
+    retrieverInstance = null;
+  }
+}

package/src/runtime/embeddings/chunker.ts

@@ -1,41 +1,79 @@
+/**
+ * Streaming text chunker - yields chunks one at a time without storing all in memory.
+ */
 export class TextChunker {
   private chunkSize: number;
   private chunkOverlap: number;
   private separators: string[] = ['\n\n', '\n', '. ', ' ', ''];
 
   constructor(chunkSize: number = 1000, chunkOverlap: number = 200) {
-    this.chunkSize = chunkSize;
-    this.chunkOverlap = chunkOverlap;
+    // Ensure valid values - overlap must be less than chunk size to avoid infinite loop
+    this.chunkSize = Math.max(100, chunkSize || 1000);
+    // Cap overlap to at most 20% of chunk size to ensure progress
+    this.chunkOverlap = Math.max(
+      0,
+      Math.min(chunkOverlap || 200, Math.floor(this.chunkSize * 0.2))
+    );
   }
 
-  splitText(text: string): string[] {
+  /**
+   * Calculate total chunks without creating them all in memory.
+   * Used for logging/progress tracking.
+   */
+  estimateChunks(text: string): number {
+    if (text.length <= this.chunkSize) {
+      return 1;
+    }
+    // Rough estimate: (text length / effective chunk size) + 1
+    const effectiveChunkSize = this.chunkSize - this.chunkOverlap;
+    return Math.ceil(text.length / effectiveChunkSize);
+  }
+
+  /**
+   * Streaming generator that yields chunks one at a time.
+   * Memory-efficient: doesn't store all chunks in an array.
+   */
+  *streamChunks(text: string): Generator<string, void, unknown> {
     if (text.length <= this.chunkSize) {
-      return [text];
+      yield text;
+      return;
     }
 
-    const chunks: string[] = [];
     let start = 0;
+    const minAdvance = Math.max(50, this.chunkSize - this.chunkOverlap); // Ensure we always advance
 
     while (start < text.length) {
       let end = Math.min(start + this.chunkSize, text.length);
 
       if (end < text.length) {
-        end = this.findBestSplitPoint(text, start, end);
+        // Try to find a good split point, but ensure we advance by at least minAdvance
+        const splitPoint = this.findBestSplitPoint(text, start, end);
+        // Only use split point if it gives us reasonable progress
+        if (splitPoint - start >= minAdvance * 0.5) {
+          end = splitPoint;
+        }
+        // Otherwise use the hard end to ensure progress
       }
 
-      chunks.push(text.substring(start, end));
-      start = end - this.chunkOverlap;
+      yield text.substring(start, end);
 
-      if (start < 0) start = 0;
-      if (start >= text.length - this.chunkOverlap) {
-        if (start < text.length) {
-          chunks.push(text.substring(start));
-        }
+      // Advance by at least minAdvance characters to avoid infinite loops
+      const nextStart = end - this.chunkOverlap;
+      start = Math.max(nextStart, start + minAdvance * 0.5);
+
+      if (start >= text.length) {
         break;
       }
     }
+  }
 
-    return chunks;
+  /**
+   * Legacy method for backwards compatibility.
+   * ⚠️ WARNING: This creates all chunks in memory and can cause OOM on large documents.
+   * Prefer streamChunks() for large documents.
+   */
+  splitText(text: string): string[] {
+    return Array.from(this.streamChunks(text));
   }
 
   private findBestSplitPoint(text: string, start: number, end: number): number {