@mastra/rag 0.0.2-alpha.70 → 0.0.2-alpha.71

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @mastra/rag
 
+## 0.0.2-alpha.71
+
+### Patch Changes
+
+- a944f1a: Bundle @mastra/rag package with tsup
+
 ## 0.0.2-alpha.70
 
 ### Patch Changes

package/dist/index.d.ts

@@ -1,4 +1,447 @@
-export * from './document/document';
-export * from './embeddings';
-export * from './utils';
-//# sourceMappingURL=index.d.ts.map
+import { LLM, TitleExtractorPrompt, TitleCombinePrompt, SummaryPrompt, QuestionExtractPrompt, Document } from 'llamaindex';
+import { TiktokenEncoding, TiktokenModel } from 'js-tiktoken';
+import * as _mastra_core from '@mastra/core';
+import { EmbeddingOptions, EmbedResult, EmbedManyResult, QueryResult, ModelConfig } from '@mastra/core';
+import { z } from 'zod';
+
+declare enum Language {
+    CPP = "cpp",
+    GO = "go",
+    JAVA = "java",
+    KOTLIN = "kotlin",
+    JS = "js",
+    TS = "ts",
+    PHP = "php",
+    PROTO = "proto",
+    PYTHON = "python",
+    RST = "rst",
+    RUBY = "ruby",
+    RUST = "rust",
+    SCALA = "scala",
+    SWIFT = "swift",
+    MARKDOWN = "markdown",
+    LATEX = "latex",
+    HTML = "html",
+    SOL = "sol",
+    CSHARP = "csharp",
+    COBOL = "cobol",
+    C = "c",
+    LUA = "lua",
+    PERL = "perl",
+    HASKELL = "haskell",
+    ELIXIR = "elixir",
+    POWERSHELL = "powershell"
+}
+type ExtractParams = {
+    title?: TitleExtractorsArgs | boolean;
+    summary?: SummaryExtractArgs | boolean;
+    questions?: QuestionAnswerExtractArgs | boolean;
+    keywords?: boolean | Record<string, any>;
+};
+type ChunkOptions = {
+    headers?: [string, string][];
+    returnEachLine?: boolean;
+    sections?: [string, string][];
+    separator?: string;
+    separators?: string[];
+    isSeparatorRegex?: boolean;
+    size?: number;
+    maxSize?: number;
+    minSize?: number;
+    overlap?: number;
+    lengthFunction?: (text: string) => number;
+    keepSeparator?: boolean | 'start' | 'end';
+    addStartIndex?: boolean;
+    stripWhitespace?: boolean;
+    language?: Language;
+    ensureAscii?: boolean;
+    convertLists?: boolean;
+    encodingName?: TiktokenEncoding;
+    modelName?: TiktokenModel;
+    allowedSpecial?: Set<string> | 'all';
+    disallowedSpecial?: Set<string> | 'all';
+    stripHeaders?: boolean;
+};
+type TitleExtractorsArgs = {
+    llm?: LLM;
+    nodes?: number;
+    nodeTemplate?: TitleExtractorPrompt['template'];
+    combineTemplate?: TitleCombinePrompt['template'];
+};
+type SummaryExtractArgs = {
+    llm?: LLM;
+    summaries?: string[];
+    promptTemplate?: SummaryPrompt['template'];
+};
+type QuestionAnswerExtractArgs = {
+    llm?: LLM;
+    questions?: number;
+    promptTemplate?: QuestionExtractPrompt['template'];
+    embeddingOnly?: boolean;
+};
+type ChunkStrategy = 'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex';
+interface ChunkParams extends ChunkOptions {
+    strategy?: ChunkStrategy;
+    extract?: ExtractParams;
+}
+
+declare class MDocument {
+    private chunks;
+    private type;
+    constructor({ docs, type }: {
+        docs: {
+            text: string;
+            metadata?: Record<string, any>;
+        }[];
+        type: string;
+    });
+    extractMetadata({ title, summary, questions, keywords }: ExtractParams): Promise<MDocument>;
+    static fromText(text: string, metadata?: Record<string, any>): MDocument;
+    static fromHTML(html: string, metadata?: Record<string, any>): MDocument;
+    static fromMarkdown(markdown: string, metadata?: Record<string, any>): MDocument;
+    static fromJSON(jsonString: string, metadata?: Record<string, any>): MDocument;
+    private defaultStrategy;
+    private chunkBy;
+    chunkRecursive(options?: ChunkOptions): Promise<void>;
+    chunkCharacter(options?: ChunkOptions): Promise<void>;
+    chunkHTML(options?: ChunkOptions): Promise<void>;
+    chunkJSON(options?: ChunkOptions): Promise<void>;
+    chunkLatex(options?: ChunkOptions): Promise<void>;
+    chunkToken(options?: ChunkOptions): Promise<void>;
+    chunkMarkdown(options?: ChunkOptions): Promise<void>;
+    chunk(params?: ChunkParams): Promise<Document[]>;
+    getDocs(): Document[];
+    getText(): string[];
+    getMetadata(): Record<string, any>[];
+}
+
+declare function embed(chunk: Document | string, options: EmbeddingOptions): Promise<EmbedResult<string>>;
+declare function embedMany(chunks: (Document | string)[], options: EmbeddingOptions): Promise<EmbedManyResult<string>>;
+
+type WeightConfig = {
+    semantic?: number;
+    vector?: number;
+    position?: number;
+};
+interface ScoringDetails {
+    semantic: number;
+    vector: number;
+    position: number;
+    queryAnalysis?: {
+        magnitude: number;
+        dominantFeatures: number[];
+    };
+}
+interface RerankResult {
+    result: QueryResult;
+    score: number;
+    details: ScoringDetails;
+}
+interface RerankerOptions {
+    weights?: WeightConfig;
+    topK?: number;
+}
+interface RerankerFunctionOptions {
+    weights?: WeightConfig;
+    queryEmbedding?: number[];
+    topK?: number;
+}
+interface RerankConfig {
+    options?: RerankerOptions;
+    model: ModelConfig;
+}
+declare function rerank(results: QueryResult[], query: string, modelConfig: ModelConfig, options: RerankerFunctionOptions): Promise<RerankResult[]>;
+
+type VectorFilterType = 'pg' | 'astra' | 'qdrant' | 'upstash' | 'pinecone' | 'chroma' | '';
+declare const createVectorQueryTool: ({ vectorStoreName, indexName, topK, options, vectorFilterType, reranker, }: {
+    vectorStoreName: string;
+    indexName: string;
+    options: EmbeddingOptions;
+    topK?: number;
+    vectorFilterType?: VectorFilterType;
+    reranker?: RerankConfig;
+}) => _mastra_core.Tool<`VectorQuery ${string} ${string} Tool`, z.ZodObject<{
+    queryText: z.ZodString;
+    filter: z.ZodUnion<[z.ZodObject<{
+        keyword: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }>, z.ZodObject<{
+        type: z.ZodEnum<["$and", "$or"]>;
+        filters: z.ZodArray<z.ZodObject<{
+            keyword: z.ZodString;
+            operator: z.ZodString;
+            value: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            value: string;
+            keyword: string;
+            operator: string;
+        }, {
+            value: string;
+            keyword: string;
+            operator: string;
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    }, {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    }>]>;
+}, "strip", z.ZodTypeAny, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    } | {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    };
+    queryText: string;
+}, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    } | {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    };
+    queryText: string;
+}>, z.ZodObject<{
+    relevantContext: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    relevantContext: string;
+}, {
+    relevantContext: string;
+}>, _mastra_core.ToolExecutionContext<z.ZodObject<{
+    queryText: z.ZodString;
+    filter: z.ZodUnion<[z.ZodObject<{
+        keyword: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }>, z.ZodObject<{
+        type: z.ZodEnum<["$and", "$or"]>;
+        filters: z.ZodArray<z.ZodObject<{
+            keyword: z.ZodString;
+            operator: z.ZodString;
+            value: z.ZodString;
+        }, "strip", z.ZodTypeAny, {
+            value: string;
+            keyword: string;
+            operator: string;
+        }, {
+            value: string;
+            keyword: string;
+            operator: string;
+        }>, "many">;
+    }, "strip", z.ZodTypeAny, {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    }, {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    }>]>;
+}, "strip", z.ZodTypeAny, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    } | {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    };
+    queryText: string;
+}, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    } | {
+        type: "$and" | "$or";
+        filters: {
+            value: string;
+            keyword: string;
+            operator: string;
+        }[];
+    };
+    queryText: string;
+}>, _mastra_core.WorkflowContext<any>>>;
+declare const createGraphRAGTool: ({ vectorStoreName, indexName, topK, options, vectorFilterType, graphOptions, }: {
+    vectorStoreName: string;
+    indexName: string;
+    options: EmbeddingOptions;
+    topK?: number;
+    vectorFilterType?: VectorFilterType;
+    graphOptions?: {
+        dimension?: number;
+        randomWalkSteps?: number;
+        restartProb?: number;
+        threshold?: number;
+    };
+}) => _mastra_core.Tool<`GraphRAG ${string} ${string} Tool`, z.ZodObject<{
+    queryText: z.ZodString;
+    filter: z.ZodObject<{
+        keyword: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    };
+    queryText: string;
+}, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    };
+    queryText: string;
+}>, z.ZodObject<{
+    relevantContext: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    relevantContext: string;
+}, {
+    relevantContext: string;
+}>, _mastra_core.ToolExecutionContext<z.ZodObject<{
+    queryText: z.ZodString;
+    filter: z.ZodObject<{
+        keyword: z.ZodString;
+        operator: z.ZodString;
+        value: z.ZodString;
+    }, "strip", z.ZodTypeAny, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }, {
+        value: string;
+        keyword: string;
+        operator: string;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    };
+    queryText: string;
+}, {
+    filter: {
+        value: string;
+        keyword: string;
+        operator: string;
+    };
+    queryText: string;
+}>, _mastra_core.WorkflowContext<any>>>;
+declare const createDocumentChunkerTool: ({ doc, params, }: {
+    doc: MDocument;
+    params?: ChunkParams;
+}) => _mastra_core.Tool<"Document Chunker undefined undefined" | `Document Chunker undefined ${number}` | "Document Chunker markdown undefined" | `Document Chunker markdown ${number}` | "Document Chunker latex undefined" | `Document Chunker latex ${number}` | "Document Chunker html undefined" | `Document Chunker html ${number}` | "Document Chunker recursive undefined" | `Document Chunker recursive ${number}` | "Document Chunker character undefined" | `Document Chunker character ${number}` | "Document Chunker token undefined" | `Document Chunker token ${number}` | "Document Chunker json undefined" | `Document Chunker json ${number}`, z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>, undefined, _mastra_core.ToolExecutionContext<z.ZodObject<{}, "strip", z.ZodTypeAny, {}, {}>, _mastra_core.WorkflowContext<any>>>;
+
+/**
+ * TODO: GraphRAG Enhancements
+ *  - Add support for more edge types (sequential, hierarchical, citation, etc)
+ *  - Allow for custom edge types
+ *  - Utilize metadata for richer connections
+ *  - Improve graph traversal and querying using types
+ */
+type SupportedEdgeType = 'semantic';
+interface GraphNode {
+    id: string;
+    content: string;
+    embedding?: number[];
+    metadata?: Record<string, any>;
+}
+interface RankedNode extends GraphNode {
+    score: number;
+}
+interface GraphEdge {
+    source: string;
+    target: string;
+    weight: number;
+    type: SupportedEdgeType;
+}
+interface GraphChunk {
+    text: string;
+    metadata: Record<string, any>;
+}
+interface GraphEmbedding {
+    vector: number[];
+}
+declare class GraphRAG {
+    private nodes;
+    private edges;
+    private dimension;
+    private threshold;
+    constructor(dimension?: number, threshold?: number);
+    addNode(node: GraphNode): void;
+    addEdge(edge: GraphEdge): void;
+    getNodes(): GraphNode[];
+    getEdges(): GraphEdge[];
+    getEdgesByType(type: string): GraphEdge[];
+    clear(): void;
+    updateNodeContent(id: string, newContent: string): void;
+    private getNeighbors;
+    private cosineSimilarity;
+    createGraph(chunks: GraphChunk[], embeddings: GraphEmbedding[]): void;
+    private selectWeightedNeighbor;
+    private randomWalkWithRestart;
+    query(query: number[], topK?: number, randomWalkSteps?: number, restartProb?: number): RankedNode[];
+}
+
+export { GraphRAG, MDocument, type RerankConfig, type RerankResult, type RerankerFunctionOptions, type RerankerOptions, createDocumentChunkerTool, createGraphRAGTool, createVectorQueryTool, embed, embedMany, rerank };