@mastra/rag 2.1.1 → 2.1.2

package/dist/docs/references/reference-tools-graph-rag-tool.md

@@ -0,0 +1,182 @@
+# createGraphRAGTool()
+
+The `createGraphRAGTool()` creates a tool that enhances RAG by building a graph of semantic relationships between documents. It uses the `GraphRAG` system under the hood to provide graph-based retrieval, finding relevant content through both direct similarity and connected relationships.
+
+## Usage Example
+
+```typescript
+import { createGraphRAGTool } from '@mastra/rag'
+import { ModelRouterEmbeddingModel } from '@mastra/core/llm'
+
+const graphTool = createGraphRAGTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  graphOptions: {
+    dimension: 1536,
+    threshold: 0.7,
+    randomWalkSteps: 100,
+    restartProb: 0.15,
+  },
+})
+```
+
+## Parameters
+
+> **Note:** **Parameter Requirements:** Most fields can be set at creation as defaults. Some fields can be overridden at runtime via the request context or input. If a required field is missing from both creation and runtime, an error will be thrown. Note that `model`, `id`, and `description` can only be set at creation time.
+
+**id?:** (`string`): Custom ID for the tool. By default: 'GraphRAG {vectorStoreName} {indexName} Tool'. (Set at creation only.)
+
+**description?:** (`string`): Custom description for the tool. By default: 'Access and analyze relationships between information in the knowledge base to answer complex questions about connections and patterns.' (Set at creation only.)
+
+**vectorStoreName:** (`string`): Name of the vector store to query. (Can be set at creation or overridden at runtime.)
+
+**indexName:** (`string`): Name of the index within the vector store. (Can be set at creation or overridden at runtime.)
+
+**model:** (`EmbeddingModel`): Embedding model to use for vector search. (Set at creation only.)
+
+**enableFilter?:** (`boolean`): Enable filtering of results based on metadata. (Set at creation only, but will be automatically enabled if a filter is provided in the request context.) (Default: `false`)
+
+**includeSources?:** (`boolean`): Include the full retrieval objects in the results. (Can be set at creation or overridden at runtime.) (Default: `true`)
+
+**graphOptions?:** (`GraphOptions`): Configuration for the graph-based retrieval (Default: `Default graph options`)
+
+**providerOptions?:** (`Record<string, Record<string, any>>`): Provider-specific options for the embedding model (e.g., outputDimensionality). \*\*Important\*\*: Only works with AI SDK EmbeddingModelV2 models. For V1 models, configure options when creating the model itself.
+
+**vectorStore?:** (`MastraVector | VectorStoreResolver`): Direct vector store instance or a resolver function for dynamic selection. Use a function for multi-tenant applications where the vector store is selected based on request context. When provided, \`vectorStoreName\` becomes optional.
+
+### GraphOptions
+
+**dimension?:** (`number`): Dimension of the embedding vectors (Default: `1536`)
+
+**threshold?:** (`number`): Similarity threshold for creating edges between nodes (0-1) (Default: `0.7`)
+
+**randomWalkSteps?:** (`number`): Number of steps in random walk for graph traversal. (Can be set at creation or overridden at runtime.) (Default: `100`)
+
+**restartProb?:** (`number`): Probability of restarting random walk from query node. (Can be set at creation or overridden at runtime.) (Default: `0.15`)
+
+## Returns
+
+The tool returns an object with:
+
+**relevantContext:** (`string`): Combined text from the most relevant document chunks, retrieved using graph-based ranking
+
+**sources:** (`QueryResult[]`): Array of full retrieval result objects. Each object contains all information needed to reference the original document, chunk, and similarity score.
+
+### QueryResult object structure
+
+```typescript
+{
+  id: string;         // Unique chunk/document identifier
+  metadata: any;      // All metadata fields (document ID, etc.)
+  vector: number[];   // Embedding vector (if available)
+  score: number;      // Similarity score for this retrieval
+  document: string;   // Full chunk/document text (if available)
+}
+```
+
+## Default Tool Description
+
+The default description focuses on:
+
+- Analyzing relationships between documents
+- Finding patterns and connections
+- Answering complex queries
+
+## Advanced Example
+
+```typescript
+const graphTool = createGraphRAGTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  graphOptions: {
+    dimension: 1536,
+    threshold: 0.8, // Higher similarity threshold
+    randomWalkSteps: 200, // More exploration steps
+    restartProb: 0.2, // Higher restart probability
+  },
+})
+```
+
+## Example with Custom Description
+
+```typescript
+const graphTool = createGraphRAGTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: 'openai/text-embedding-3-small	',
+  description:
+    "Analyze document relationships to find complex patterns and connections in our company's historical data",
+})
+```
+
+This example shows how to customize the tool description for a specific use case while maintaining its core purpose of relationship analysis.
+
+## Example: Using Request Context
+
+```typescript
+const graphTool = createGraphRAGTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: 'openai/text-embedding-3-small	',
+})
+```
+
+When using request context, provide required parameters at execution time via the request context:
+
+```typescript
+const requestContext = new RequestContext<{
+  vectorStoreName: string
+  indexName: string
+  topK: number
+  filter: any
+}>()
+requestContext.set('vectorStoreName', 'my-store')
+requestContext.set('indexName', 'my-index')
+requestContext.set('topK', 5)
+requestContext.set('filter', { category: 'docs' })
+requestContext.set('randomWalkSteps', 100)
+requestContext.set('restartProb', 0.15)
+
+const response = await agent.generate('Find documentation from the knowledge base.', {
+  requestContext,
+})
+```
+
+For more information on request context, please see:
+
+- [Agent Request Context](https://mastra.ai/docs/server/request-context)
+- [Request Context](https://mastra.ai/docs/server/request-context)
+
+## Dynamic Vector Store for Multi-Tenant Applications
+
+For multi-tenant applications where each tenant has isolated data, you can pass a resolver function instead of a static vector store:
+
+```typescript
+import { createGraphRAGTool, VectorStoreResolver } from '@mastra/rag'
+import { PgVector } from '@mastra/pg'
+
+const vectorStoreResolver: VectorStoreResolver = async ({ requestContext }) => {
+  const tenantId = requestContext?.get('tenantId')
+
+  return new PgVector({
+    id: `pg-vector-${tenantId}`,
+    connectionString: process.env.POSTGRES_CONNECTION_STRING!,
+    schemaName: `tenant_${tenantId}`,
+  })
+}
+
+const graphTool = createGraphRAGTool({
+  indexName: 'embeddings',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  vectorStore: vectorStoreResolver,
+})
+```
+
+See [createVectorQueryTool - Dynamic Vector Store](https://mastra.ai/reference/tools/vector-query-tool) for more details.
+
+## Related
+
+- [createVectorQueryTool](https://mastra.ai/reference/tools/vector-query-tool)
+- [GraphRAG](https://mastra.ai/reference/rag/graph-rag)

package/dist/docs/references/reference-tools-vector-query-tool.md

@@ -0,0 +1,459 @@
+# createVectorQueryTool()
+
+The `createVectorQueryTool()` function creates a tool for semantic search over vector stores. It supports filtering, reranking, database-specific configurations, and integrates with various vector store backends.
+
+## Basic Usage
+
+```typescript
+import { createVectorQueryTool } from '@mastra/rag'
+import { ModelRouterEmbeddingModel } from '@mastra/core/llm'
+
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+})
+```
+
+## Parameters
+
+> **Note:** **Parameter Requirements:** Most fields can be set at creation as defaults. Some fields can be overridden at runtime via the request context or input. If a required field is missing from both creation and runtime, an error will be thrown. Note that `model`, `id`, and `description` can only be set at creation time.
+
+**id?:** (`string`): Custom ID for the tool. By default: 'VectorQuery {vectorStoreName} {indexName} Tool'. (Set at creation only.)
+
+**description?:** (`string`): Custom description for the tool. By default: 'Access the knowledge base to find information needed to answer user questions' (Set at creation only.)
+
+**model:** (`EmbeddingModel`): Embedding model to use for vector search. (Set at creation only.)
+
+**vectorStoreName:** (`string`): Name of the vector store to query. (Can be set at creation or overridden at runtime.)
+
+**indexName:** (`string`): Name of the index within the vector store. (Can be set at creation or overridden at runtime.)
+
+**enableFilter?:** (`boolean`): Enable filtering of results based on metadata. (Set at creation only, but will be automatically enabled if a filter is provided in the request context.) (Default: `false`)
+
+**includeVectors?:** (`boolean`): Include the embedding vectors in the results. (Can be set at creation or overridden at runtime.) (Default: `false`)
+
+**includeSources?:** (`boolean`): Include the full retrieval objects in the results. (Can be set at creation or overridden at runtime.) (Default: `true`)
+
+**reranker?:** (`RerankConfig`): Options for reranking results. (Can be set at creation or overridden at runtime.)
+
+**databaseConfig?:** (`DatabaseConfig`): Database-specific configuration options for optimizing queries. (Can be set at creation or overridden at runtime.)
+
+**providerOptions?:** (`Record<string, Record<string, any>>`): Provider-specific options for the embedding model (e.g., outputDimensionality). \*\*Important\*\*: Only works with AI SDK EmbeddingModelV2 models. For V1 models, configure options when creating the model itself.
+
+**vectorStore?:** (`MastraVector | VectorStoreResolver`): Direct vector store instance or a resolver function for dynamic selection. Use a function for multi-tenant applications where the vector store is selected based on request context. When provided, \`vectorStoreName\` becomes optional.
+
+### DatabaseConfig
+
+The `DatabaseConfig` type allows you to specify database-specific configurations that are automatically applied to query operations. This enables you to take advantage of unique features and optimizations offered by different vector stores.
+
+**pinecone?:** (`PineconeConfig`): objectnamespace?:stringPinecone namespace for organizing vectorssparseVector?:{ indices: number\[]; values: number\[]; }Sparse vector for hybrid search
+
+**pgvector?:** (`PgVectorConfig`): objectminScore?:numberMinimum similarity score threshold for resultsef?:numberHNSW search parameter - controls accuracy vs speed tradeoffprobes?:numberIVFFlat probe parameter - number of cells to visit during search
+
+**chroma?:** (`ChromaConfig`): objectwhere?:Record\<string, any>Metadata filtering conditionswhereDocument?:Record\<string, any>Document content filtering conditions
+
+### RerankConfig
+
+**model:** (`MastraLanguageModel`): Language model to use for reranking
+
+**options?:** (`RerankerOptions`): objectweights?:WeightConfigWeights for scoring components (semantic: 0.4, vector: 0.4, position: 0.2)topK?:numberNumber of top results to return
+
+## Returns
+
+The tool returns an object with:
+
+**relevantContext:** (`string`): Combined text from the most relevant document chunks
+
+**sources:** (`QueryResult[]`): Array of full retrieval result objects. Each object contains all information needed to reference the original document, chunk, and similarity score.
+
+### QueryResult object structure
+
+```typescript
+{
+  id: string;         // Unique chunk/document identifier
+  metadata: any;      // All metadata fields (document ID, etc.)
+  vector: number[];   // Embedding vector (if available)
+  score: number;      // Similarity score for this retrieval
+  document: string;   // Full chunk/document text (if available)
+}
+```
+
+## Default Tool Description
+
+The default description focuses on:
+
+- Finding relevant information in stored knowledge
+- Answering user questions
+- Retrieving factual content
+
+## Result Handling
+
+The tool determines the number of results to return based on the user's query, with a default of 10 results. This can be adjusted based on the query requirements.
+
+## Example with Filters
+
+```typescript
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  enableFilter: true,
+})
+```
+
+With filtering enabled, the tool processes queries to construct metadata filters that combine with semantic search. The process works as follows:
+
+1. A user makes a query with specific filter requirements like "Find content where the 'version' field is greater than 2.0"
+2. The agent analyzes the query and constructs the appropriate filters:
+   ```typescript
+   {
+      "version": { "$gt": 2.0 }
+   }
+   ```
+
+This agent-driven approach:
+
+- Processes natural language queries into filter specifications
+- Implements vector store-specific filter syntax
+- Translates query terms to filter operators
+
+For detailed filter syntax and store-specific capabilities, see the [Metadata Filters](https://mastra.ai/reference/rag/metadata-filters) documentation.
+
+For an example of how agent-driven filtering works, see the [Agent-Driven Metadata Filtering](https://github.com/mastra-ai/mastra/tree/main/examples/basics/rag/filter-rag) example.
+
+## Example with Reranking
+
+```typescript
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'milvus',
+  indexName: 'documentation',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  reranker: {
+    model: 'openai/gpt-5.1',
+    options: {
+      weights: {
+        semantic: 0.5, // Semantic relevance weight
+        vector: 0.3, // Vector similarity weight
+        position: 0.2, // Original position weight
+      },
+      topK: 5,
+    },
+  },
+})
+```
+
+Reranking improves result quality by combining:
+
+- Semantic relevance: Using LLM-based scoring of text similarity
+- Vector similarity: Original vector distance scores
+- Position bias: Consideration of original result ordering
+- Query analysis: Adjustments based on query characteristics
+
+The reranker processes the initial vector search results and returns a reordered list optimized for relevance.
+
+## Example with Custom Description
+
+```typescript
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  description:
+    'Search through document archives to find relevant information for answering questions about company policies and procedures',
+})
+```
+
+This example shows how to customize the tool description for a specific use case while maintaining its core purpose of information retrieval.
+
+## Database-Specific Configuration Examples
+
+The `databaseConfig` parameter allows you to leverage unique features and optimizations specific to each vector database. These configurations are automatically applied during query execution.
+
+**Pinecone**:
+
+### Pinecone Configuration
+
+```typescript
+const pineconeQueryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  databaseConfig: {
+    pinecone: {
+      namespace: 'production', // Organize vectors by environment
+      sparseVector: {
+        // Enable hybrid search
+        indices: [0, 1, 2, 3],
+        values: [0.1, 0.2, 0.15, 0.05],
+      },
+    },
+  },
+})
+```
+
+**Pinecone Features:**
+
+- **Namespace**: Isolate different data sets within the same index
+- **Sparse Vector**: Combine dense and sparse embeddings for improved search quality
+- **Use Cases**: Multi-tenant applications, hybrid semantic search
+
+**pgVector**:
+
+### pgVector Configuration
+
+```typescript
+const pgVectorQueryTool = createVectorQueryTool({
+  vectorStoreName: 'postgres',
+  indexName: 'embeddings',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  databaseConfig: {
+    pgvector: {
+      minScore: 0.7, // Only return results above 70% similarity
+      ef: 200, // Higher value = better accuracy, slower search
+      probes: 10, // For IVFFlat: more probes = better recall
+    },
+  },
+})
+```
+
+**pgVector Features:**
+
+- **minScore**: Filter out low-quality matches
+- **ef (HNSW)**: Control accuracy vs speed for HNSW indexes
+- **probes (IVFFlat)**: Control recall vs speed for IVFFlat indexes
+- **Use Cases**: Performance tuning, quality filtering
+
+**Chroma**:
+
+### Chroma Configuration
+
+```typescript
+const chromaQueryTool = createVectorQueryTool({
+  vectorStoreName: 'chroma',
+  indexName: 'documents',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  databaseConfig: {
+    chroma: {
+      where: {
+        // Metadata filtering
+        category: 'technical',
+        status: 'published',
+      },
+      whereDocument: {
+        // Document content filtering
+        $contains: 'API',
+      },
+    },
+  },
+})
+```
+
+**Chroma Features:**
+
+- **where**: Filter by metadata fields
+- **whereDocument**: Filter by document content
+- **Use Cases**: Advanced filtering, content-based search
+
+**Multiple Configs**:
+
+### Multiple Database Configurations
+
+```typescript
+// Configure for multiple databases (useful for dynamic stores)
+const multiDbQueryTool = createVectorQueryTool({
+  vectorStoreName: 'dynamic-store', // Will be set at runtime
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  databaseConfig: {
+    pinecone: {
+      namespace: 'default',
+    },
+    pgvector: {
+      minScore: 0.8,
+      ef: 150,
+    },
+    chroma: {
+      where: { type: 'documentation' },
+    },
+  },
+})
+```
+
+**Multi-Config Benefits:**
+
+- Support multiple vector stores with one tool
+- Database-specific optimizations are automatically applied
+- Flexible deployment scenarios
+
+### Runtime Configuration Override
+
+You can override database configurations at runtime to adapt to different scenarios:
+
+```typescript
+import { RequestContext } from '@mastra/core/request-context'
+
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  databaseConfig: {
+    pinecone: {
+      namespace: 'development',
+    },
+  },
+})
+
+// Override at runtime
+const requestContext = new RequestContext()
+requestContext.set('databaseConfig', {
+  pinecone: {
+    namespace: 'production', // Switch to production namespace
+  },
+})
+
+const response = await agent.generate('Find information about deployment', {
+  requestContext,
+})
+```
+
+This approach allows you to:
+
+- Switch between environments (dev/staging/prod)
+- Adjust performance parameters based on load
+- Apply different filtering strategies per request
+
+## Example: Using Request Context
+
+```typescript
+const queryTool = createVectorQueryTool({
+  vectorStoreName: 'pinecone',
+  indexName: 'docs',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+})
+```
+
+When using request context, provide required parameters at execution time via the request context:
+
+```typescript
+const requestContext = new RequestContext<{
+  vectorStoreName: string
+  indexName: string
+  topK: number
+  filter: VectorFilter
+  databaseConfig: DatabaseConfig
+}>()
+requestContext.set('vectorStoreName', 'my-store')
+requestContext.set('indexName', 'my-index')
+requestContext.set('topK', 5)
+requestContext.set('filter', { category: 'docs' })
+requestContext.set('databaseConfig', {
+  pinecone: { namespace: 'runtime-namespace' },
+})
+requestContext.set('model', 'openai/text-embedding-3-small')
+
+const response = await agent.generate('Find documentation from the knowledge base.', {
+  requestContext,
+})
+```
+
+For more information on request context, please see:
+
+- [Agent Request Context](https://mastra.ai/docs/server/request-context)
+- [Request Context](https://mastra.ai/docs/server/request-context)
+
+## Usage Without a Mastra Server
+
+The tool can be used by itself to retrieve documents matching a query:
+
+```typescript
+import { RequestContext } from '@mastra/core/request-context'
+import { createVectorQueryTool } from '@mastra/rag'
+import { PgVector } from '@mastra/pg'
+
+const pgVector = new PgVector({
+  id: 'pg-vector',
+  connectionString: process.env.POSTGRES_CONNECTION_STRING!,
+})
+
+const vectorQueryTool = createVectorQueryTool({
+  vectorStoreName: 'pgVector', // optional since we're passing in a store
+  vectorStore: pgVector,
+  indexName: 'embeddings',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+})
+
+const requestContext = new RequestContext()
+const queryResult = await vectorQueryTool.execute({ queryText: 'foo', topK: 1 }, { requestContext })
+
+console.log(queryResult.sources)
+```
+
+## Dynamic Vector Store for Multi-Tenant Applications
+
+For multi-tenant applications where each tenant has isolated data (e.g., separate PostgreSQL schemas), you can pass a resolver function instead of a static vector store instance. The function receives the request context and can return the appropriate vector store for the current tenant:
+
+```typescript
+import { createVectorQueryTool, VectorStoreResolver } from '@mastra/rag'
+import { PgVector } from '@mastra/pg'
+
+// Cache for tenant-specific vector stores
+const vectorStoreCache = new Map<string, PgVector>()
+
+// Resolver function that returns the correct vector store based on tenant
+const vectorStoreResolver: VectorStoreResolver = async ({ requestContext }) => {
+  const tenantId = requestContext?.get('tenantId')
+
+  if (!tenantId) {
+    throw new Error('tenantId is required in request context')
+  }
+
+  // Return cached instance or create new one
+  if (!vectorStoreCache.has(tenantId)) {
+    vectorStoreCache.set(
+      tenantId,
+      new PgVector({
+        id: `pg-vector-${tenantId}`,
+        connectionString: process.env.POSTGRES_CONNECTION_STRING!,
+        schemaName: `tenant_${tenantId}`, // Each tenant has their own schema
+      }),
+    )
+  }
+
+  return vectorStoreCache.get(tenantId)!
+}
+
+const vectorQueryTool = createVectorQueryTool({
+  indexName: 'embeddings',
+  model: new ModelRouterEmbeddingModel('openai/text-embedding-3-small'),
+  vectorStore: vectorStoreResolver, // Dynamic resolution!
+})
+
+// Usage with tenant context
+const requestContext = new RequestContext()
+requestContext.set('tenantId', 'acme-corp')
+
+const result = await vectorQueryTool.execute(
+  { queryText: 'company policies', topK: 5 },
+  { requestContext },
+)
+```
+
+This pattern is similar to how `Agent.memory` supports dynamic configuration and enables:
+
+- **Schema isolation**: Each tenant's data in separate PostgreSQL schemas
+- **Database isolation**: Route to different database instances per tenant
+- **Dynamic configuration**: Adjust vector store settings based on request context
+
+## Tool Details
+
+The tool is created with:
+
+- **ID**: `VectorQuery {vectorStoreName} {indexName} Tool`
+- **Input Schema**: Requires queryText and filter objects
+- **Output Schema**: Returns relevantContext string
+
+## Related
+
+- [rerank()](https://mastra.ai/reference/rag/rerank)
+- [createGraphRAGTool](https://mastra.ai/reference/tools/graph-rag-tool)

package/dist/document/transformers/semantic-markdown.d.ts

@@ -1,13 +1,15 @@
-import type { TiktokenModel, TiktokenEncoding } from 'js-tiktoken';
+import type { TiktokenModel, TiktokenEncoding, Tiktoken } from 'js-tiktoken';
 import { Document } from '../schema/index.js';
 import type { SemanticMarkdownChunkOptions } from '../types.js';
 import { TextTransformer } from './text.js';
 export declare class SemanticMarkdownTransformer extends TextTransformer {
     private tokenizer;
     private joinThreshold;
-    private allowedSpecial;
-    private disallowedSpecial;
-    constructor({ joinThreshold, encodingName, modelName, allowedSpecial, disallowedSpecial, ...baseOptions }?: SemanticMarkdownChunkOptions);
+    private allowedArray;
+    private disallowedArray;
+    constructor({ joinThreshold, encodingName, modelName, tokenizer: existingTokenizer, allowedSpecial, disallowedSpecial, ...baseOptions }?: SemanticMarkdownChunkOptions & {
+        tokenizer?: Tiktoken;
+    });
     private countTokens;
     private splitMarkdownByHeaders;
     private mergeSemanticSections;

package/dist/document/transformers/semantic-markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/semantic-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAY,MAAM,aAAa,CAAC;AAE7E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,UAAU,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AASzC,qBAAa,2BAA4B,SAAQ,eAAe;IAC9D,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,aAAmB,EACnB,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,GAAG,WAAW,EACf,GAAE,4BAAiC;IAcpC,OAAO,CAAC,WAAW;IAQnB,OAAO,CAAC,sBAAsB;IA2D9B,OAAO,CAAC,qBAAqB;IA+B7B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAgB/C,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAuB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,4BAA4B,CAAC;KACxC,GAAG,2BAA2B;CA4BhC"}
+{"version":3,"file":"semantic-markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/semantic-markdown.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAE7E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,KAAK,EAAE,4BAA4B,EAAE,MAAM,UAAU,CAAC;AAE7D,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AASzC,qBAAa,2BAA4B,SAAQ,eAAe;IAC9D,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAmB;IACvC,OAAO,CAAC,eAAe,CAAmB;gBAE9B,EACV,aAAmB,EACnB,YAA4B,EAC5B,SAAS,EACT,SAAS,EAAE,iBAAiB,EAC5B,cAA0B,EAC1B,iBAAyB,EACzB,GAAG,WAAW,EACf,GAAE,4BAA4B,GAAG;QAAE,SAAS,CAAC,EAAE,QAAQ,CAAA;KAAO;IAkB/D,OAAO,CAAC,WAAW;IAKnB,OAAO,CAAC,sBAAsB;IA2D9B,OAAO,CAAC,qBAAqB;IAqC7B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAgB/C,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAuB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE,4BAA4B,CAAC;KACxC,GAAG,2BAA2B;CA6BhC"}

package/dist/document/transformers/token.d.ts

@@ -1,4 +1,4 @@
-import type { TiktokenModel, TiktokenEncoding } from 'js-tiktoken';
+import type { TiktokenModel, TiktokenEncoding, Tiktoken } from 'js-tiktoken';
 import type { TokenChunkOptions } from '../types.js';
 import { TextTransformer } from './text.js';
 interface Tokenizer {
@@ -13,11 +13,12 @@ export declare function splitTextOnTokens({ text, tokenizer }: {
 }): string[];
 export declare class TokenTransformer extends TextTransformer {
     private tokenizer;
-    private allowedSpecial;
-    private disallowedSpecial;
-    constructor({ encodingName, modelName, allowedSpecial, disallowedSpecial, options, }: {
+    private allowedArray;
+    private disallowedArray;
+    constructor({ encodingName, modelName, tokenizer: existingTokenizer, allowedSpecial, disallowedSpecial, options, }: {
         encodingName?: TiktokenEncoding;
         modelName?: TiktokenModel;
+        tokenizer?: Tiktoken;
         allowedSpecial?: Set<string> | 'all';
         disallowedSpecial?: Set<string> | 'all';
         options: TokenChunkOptions;