@mastra/rag 0.0.2-alpha.12 → 0.0.2-alpha.13

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @mastra/rag
 
+## 0.0.2-alpha.13
+
+### Patch Changes
+
+- Updated dependencies [963c15a]
+  - @mastra/core@0.1.27-alpha.30
+
 ## 0.0.2-alpha.12
 
 ### Patch Changes

package/dist/astra-db/index.d.ts

@@ -0,0 +1,53 @@
+import { MastraVector, QueryResult, IndexStats } from '@mastra/core';
+export interface AstraDbOptions {
+    token: string;
+    endpoint: string;
+    keyspace?: string;
+}
+export declare class AstraVector extends MastraVector {
+    #private;
+    constructor({ token, endpoint, keyspace }: AstraDbOptions);
+    /**
+     * Creates a new collection with the specified configuration.
+     *
+     * @param {string} indexName - The name of the collection to create.
+     * @param {number} dimension - The dimension of the vectors to be stored in the collection.
+     * @param {'cosine' | 'euclidean' | 'dotproduct'} [metric=cosine] - The metric to use to sort vectors in the collection.
+     * @returns {Promise<void>} A promise that resolves when the collection is created.
+     */
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    /**
+     * Inserts or updates vectors in the specified collection.
+     *
+     * @param {string} indexName - The name of the collection to upsert into.
+     * @param {number[][]} vectors - An array of vectors to upsert.
+     * @param {Record<string, any>[]} [metadata] - An optional array of metadata objects corresponding to each vector.
+     * @param {string[]} [ids] - An optional array of IDs corresponding to each vector. If not provided, new IDs will be generated.
+     * @returns {Promise<string[]>} A promise that resolves to an array of IDs of the upserted vectors.
+     */
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    /**
+     * Queries the specified collection using a vector and optional filter.
+     *
+     * @param {string} indexName - The name of the collection to query.
+     * @param {number[]} queryVector - The vector to query with.
+     * @param {number} [topK] - The maximum number of results to return.
+     * @param {Record<string, any>} [filter] - An optional filter to apply to the query. For more on filters in Astra DB, see the filtering reference: https://docs.datastax.com/en/astra-db-serverless/api-reference/documents.html#operators
+     * @returns {Promise<QueryResult[]>} A promise that resolves to an array of query results.
+     */
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>): Promise<QueryResult[]>;
+    /**
+     * Lists all collections in the database.
+     *
+     * @returns {Promise<string[]>} A promise that resolves to an array of collection names.
+     */
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    /**
+     * Deletes the specified collection.
+     *
+     * @param {string} indexName - The name of the collection to delete.
+     * @returns {Promise<void>} A promise that resolves when the collection is deleted.
+     */
+    deleteIndex(indexName: string): Promise<void>;
+}

package/dist/document/document.d.ts

@@ -0,0 +1,35 @@
+import { Document } from 'llamaindex';
+import { ChunkOptions, ChunkStrategy, ExtractParams } from './types';
+export declare class MastraDocument {
+    private docs;
+    private type;
+    constructor({ docs, type }: {
+        docs: {
+            text: string;
+            metadata?: Record<string, any>;
+        }[];
+        type: string;
+    });
+    private extract;
+    static fromText(text: string, metadata?: Record<string, any>): MastraDocument;
+    static fromHTML(html: string, metadata?: Record<string, any>): MastraDocument;
+    static fromMarkdown(markdown: string, metadata?: Record<string, any>): MastraDocument;
+    static fromJSON(jsonString: string, metadata?: Record<string, any>): MastraDocument;
+    private defaultStrategy;
+    private chunkBy;
+    chunkRecursive(options?: ChunkOptions): Promise<void>;
+    chunkCharacter(options?: ChunkOptions): Promise<void>;
+    chunkHTML(options?: ChunkOptions): Promise<void>;
+    chunkJSON(options?: ChunkOptions): Promise<void>;
+    chunkLatex(options?: ChunkOptions): Promise<void>;
+    chunkToken(options?: ChunkOptions): Promise<void>;
+    chunkMarkdown(options?: ChunkOptions): Promise<void>;
+    chunk(params?: {
+        strategy?: ChunkStrategy;
+        options?: ChunkOptions;
+        extract?: ExtractParams;
+    }): Promise<MastraDocument>;
+    getDocs(): Document[];
+    getText(): string[];
+    getMetadata(): Record<string, any>[];
+}

package/dist/document/index.d.ts

@@ -0,0 +1,2 @@
+export * from './document';
+export * from './types';

package/dist/document/transformers/character.d.ts

@@ -0,0 +1,44 @@
+import { ChunkOptions, Language } from '../types';
+import { TextTransformer } from './text';
+export declare class CharacterTransformer extends TextTransformer {
+    protected separator: string;
+    protected isSeparatorRegex: boolean;
+    constructor({ separator, isSeparatorRegex, options, }: {
+        separator?: string;
+        isSeparatorRegex?: boolean;
+        options?: {
+            chunkSize?: number;
+            chunkOverlap?: number;
+            lengthFunction?: (text: string) => number;
+            keepSeparator?: boolean | 'start' | 'end';
+            addStartIndex?: boolean;
+            stripWhitespace?: boolean;
+        };
+    });
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    private __splitChunk;
+}
+export declare class RecursiveCharacterTransformer extends TextTransformer {
+    protected separators: string[];
+    protected isSeparatorRegex: boolean;
+    constructor({ separators, isSeparatorRegex, options, }: {
+        separators?: string[];
+        isSeparatorRegex?: boolean;
+        options?: ChunkOptions;
+    });
+    private _splitText;
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    static fromLanguage(language: Language, options?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    }): RecursiveCharacterTransformer;
+    static getSeparatorsForLanguage(language: Language): string[];
+}

package/dist/document/transformers/html.d.ts

@@ -0,0 +1,23 @@
+import { Document } from 'llamaindex';
+export declare class HTMLHeaderTransformer {
+    private headersToSplitOn;
+    private returnEachElement;
+    constructor(headersToSplitOn: [string, string][], returnEachElement?: boolean);
+    splitText({ text }: {
+        text: string;
+    }): Document[];
+    private getXPath;
+    private aggregateElementsToChunks;
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+}
+export declare class HTMLSectionTransformer {
+    private headersToSplitOn;
+    private options;
+    constructor(headersToSplitOn: [string, string][], options?: Record<string, any>);
+    splitText(text: string): Document[];
+    splitDocuments(documents: Document[]): Promise<Document[]>;
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    private splitHtmlByHeaders;
+    transformDocuments(documents: Document[]): Document[];
+}

package/dist/document/transformers/json.d.ts

@@ -0,0 +1,56 @@
+import { Document } from 'llamaindex';
+export declare class RecursiveJsonTransformer {
+    private maxChunkSize;
+    private minChunkSize;
+    constructor({ maxChunkSize, minChunkSize }: {
+        maxChunkSize: number;
+        minChunkSize?: number;
+    });
+    private static jsonSize;
+    /**
+     * Transform JSON data while handling circular references
+     */
+    transform(data: Record<string, any>): Record<string, any>;
+    /**
+     * Set a value in a nested dictionary based on the given path
+     */
+    private static setNestedDict;
+    /**
+     * Convert lists in the JSON structure to dictionaries with index-based keys
+     */
+    private listToDictPreprocessing;
+    /**
+     * Split json into maximum size dictionaries while preserving structure
+     */
+    private jsonSplit;
+    /**
+     * Splits JSON into a list of JSON chunks
+     */
+    splitJson({ jsonData, convertLists, }: {
+        jsonData: Record<string, any>;
+        convertLists?: boolean;
+    }): Record<string, any>[];
+    private escapeNonAscii;
+    /**
+     * Splits JSON into a list of JSON formatted strings
+     */
+    splitText({ jsonData, convertLists, ensureAscii, }: {
+        jsonData: Record<string, any>;
+        convertLists?: boolean;
+        ensureAscii?: boolean;
+    }): string[];
+    /**
+     * Create documents from a list of json objects
+     */
+    createDocuments({ texts, convertLists, ensureAscii, metadatas, }: {
+        texts: string[];
+        convertLists?: boolean;
+        ensureAscii?: boolean;
+        metadatas?: Record<string, any>[];
+    }): Document[];
+    transformDocuments({ ensureAscii, documents, convertLists, }: {
+        ensureAscii?: boolean;
+        convertLists?: boolean;
+        documents: Document[];
+    }): Document[];
+}

package/dist/document/transformers/latex.d.ts

@@ -0,0 +1,11 @@
+import { RecursiveCharacterTransformer } from './character';
+export declare class LatexTransformer extends RecursiveCharacterTransformer {
+    constructor(options?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    });
+}

package/dist/document/transformers/markdown.d.ts

@@ -0,0 +1,24 @@
+import { Document } from 'llamaindex';
+import { RecursiveCharacterTransformer } from './character';
+export declare class MarkdownTransformer extends RecursiveCharacterTransformer {
+    constructor(options?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    });
+}
+export declare class MarkdownHeaderTransformer {
+    private headersToSplitOn;
+    private returnEachLine;
+    private stripHeaders;
+    constructor(headersToSplitOn: [string, string][], returnEachLine?: boolean, stripHeaders?: boolean);
+    private aggregateLinesToChunks;
+    splitText({ text }: {
+        text: string;
+    }): Document[];
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+}

package/dist/document/transformers/text.d.ts

@@ -0,0 +1,21 @@
+import { Document } from 'llamaindex';
+import { ChunkOptions } from '../types';
+import { Transformer } from './transformer';
+export declare abstract class TextTransformer implements Transformer {
+    protected chunkSize: number;
+    protected chunkOverlap: number;
+    protected lengthFunction: (text: string) => number;
+    protected keepSeparator: boolean | 'start' | 'end';
+    protected addStartIndex: boolean;
+    protected stripWhitespace: boolean;
+    constructor({ chunkSize, chunkOverlap, lengthFunction, keepSeparator, addStartIndex, stripWhitespace, }: ChunkOptions);
+    setAddStartIndex(value: boolean): void;
+    abstract splitText({ text }: {
+        text: string;
+    }): string[];
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    splitDocuments(documents: Document[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+    protected joinDocs(docs: string[], separator: string): string | null;
+    protected mergeSplits(splits: string[], separator: string): string[];
+}

package/dist/document/transformers/token.d.ts

@@ -0,0 +1,47 @@
+import { TiktokenModel, TiktokenEncoding } from '@dqbd/tiktoken';
+import { TextTransformer } from './text';
+interface Tokenizer {
+    chunkOverlap: number;
+    tokensPerChunk: number;
+    decode: (tokens: number[]) => string;
+    encode: (text: string) => number[];
+}
+export declare function splitTextOnTokens({ text, tokenizer }: {
+    text: string;
+    tokenizer: Tokenizer;
+}): string[];
+export declare class TokenTransformer extends TextTransformer {
+    private tokenizer;
+    private allowedSpecial;
+    private disallowedSpecial;
+    private textDecoder;
+    constructor({ encodingName, modelName, allowedSpecial, disallowedSpecial, options, }: {
+        encodingName: TiktokenEncoding;
+        modelName?: TiktokenModel;
+        allowedSpecial?: Set<string> | 'all';
+        disallowedSpecial?: Set<string> | 'all';
+        options: {
+            chunkSize?: number;
+            chunkOverlap?: number;
+            lengthFunction?: (text: string) => number;
+            keepSeparator?: boolean | 'start' | 'end';
+            addStartIndex?: boolean;
+            stripWhitespace?: boolean;
+        };
+    });
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    dispose(): void;
+    static fromTikToken({ encodingName, modelName, options, }: {
+        encodingName?: TiktokenEncoding;
+        modelName?: TiktokenModel;
+        options?: {
+            chunkSize?: number;
+            chunkOverlap?: number;
+            allowedSpecial?: Set<string> | 'all';
+            disallowedSpecial?: Set<string> | 'all';
+        };
+    }): TokenTransformer;
+}
+export {};

package/dist/document/transformers/transformer.d.ts

@@ -0,0 +1,4 @@
+import { Document } from 'llamaindex';
+export interface Transformer {
+    transformDocuments(documents: Document[]): Document[];
+}

package/dist/document/types.d.ts

@@ -0,0 +1,83 @@
+import { TiktokenEncoding, TiktokenModel } from '@dqbd/tiktoken';
+import { LLM, TitleCombinePrompt, TitleExtractorPrompt, SummaryPrompt, QuestionExtractPrompt, KeywordExtractPrompt } from 'llamaindex';
+export declare enum Language {
+    CPP = "cpp",
+    GO = "go",
+    JAVA = "java",
+    KOTLIN = "kotlin",
+    JS = "js",
+    TS = "ts",
+    PHP = "php",
+    PROTO = "proto",
+    PYTHON = "python",
+    RST = "rst",
+    RUBY = "ruby",
+    RUST = "rust",
+    SCALA = "scala",
+    SWIFT = "swift",
+    MARKDOWN = "markdown",
+    LATEX = "latex",
+    HTML = "html",
+    SOL = "sol",
+    CSHARP = "csharp",
+    COBOL = "cobol",
+    C = "c",
+    LUA = "lua",
+    PERL = "perl",
+    HASKELL = "haskell",
+    ELIXIR = "elixir",
+    POWERSHELL = "powershell"
+}
+export type ExtractParams = {
+    title?: TitleExtractorsArgs | boolean;
+    summary?: SummaryExtractArgs | boolean;
+    questions?: QuestionAnswerExtractArgs | boolean;
+    keywords?: boolean | Record<string, any>;
+};
+export type ChunkOptions = {
+    headers?: [string, string][];
+    returnEachLine?: boolean;
+    sections?: [string, string][];
+    separator?: string;
+    separators?: string[];
+    isSeparatorRegex?: boolean;
+    chunkSize?: number;
+    maxChunkSize?: number;
+    minChunkSize?: number;
+    chunkOverlap?: number;
+    lengthFunction?: (text: string) => number;
+    keepSeparator?: boolean | 'start' | 'end';
+    addStartIndex?: boolean;
+    stripWhitespace?: boolean;
+    language?: Language;
+    ensureAscii?: boolean;
+    convertLists?: boolean;
+    encodingName?: TiktokenEncoding;
+    modelName?: TiktokenModel;
+    allowedSpecial?: Set<string> | 'all';
+    disallowedSpecial?: Set<string> | 'all';
+    stripHeaders?: boolean;
+};
+export type TitleExtractorsArgs = {
+    llm?: LLM;
+    nodes?: number;
+    nodeTemplate?: TitleExtractorPrompt['template'];
+    combineTemplate?: TitleCombinePrompt['template'];
+};
+export type SummaryExtractArgs = {
+    llm?: LLM;
+    summaries?: string[];
+    promptTemplate?: SummaryPrompt['template'];
+};
+export type QuestionAnswerExtractArgs = {
+    llm?: LLM;
+    questions?: number;
+    promptTemplate?: QuestionExtractPrompt['template'];
+    embeddingOnly?: boolean;
+};
+export type KeywordExtractArgs = {
+    llm?: LLM;
+    keywords?: number;
+    promptTemplate?: KeywordExtractPrompt['template'];
+};
+export type ChunkStrategy = 'recursive' | 'character' | 'token' | 'markdown' | 'html' | 'json' | 'latex';

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+export * from './document/document';
+export * from './pg';
+export * from './pinecone';
+export * from './astra-db';
+export * from './qdrant';
+export * from './upstash';

package/dist/index.js

@@ -0,0 +1,8 @@
+
+'use strict'
+
+if (process.env.NODE_ENV === 'production') {
+  module.exports = require('./rag.cjs.production.min.js')
+} else {
+  module.exports = require('./rag.cjs.development.js')
+}

package/dist/pg/index.d.ts

@@ -0,0 +1,13 @@
+import { IndexStats, QueryResult, MastraVector } from '@mastra/core';
+export declare class PgVector extends MastraVector {
+    private pool;
+    constructor(connectionString: string);
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>, minScore?: number): Promise<QueryResult[]>;
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    deleteIndex(indexName: string): Promise<void>;
+    truncateIndex(indexName: string): Promise<void>;
+    disconnect(): Promise<void>;
+}

package/dist/pg/index_test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/pinecone/index.d.ts

@@ -0,0 +1,11 @@
+import { MastraVector, QueryResult, IndexStats } from '@mastra/core';
+export declare class PineconeVector extends MastraVector {
+    private client;
+    constructor(apiKey: string, environment?: string);
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>): Promise<QueryResult[]>;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    deleteIndex(indexName: string): Promise<void>;
+}

package/dist/qdrant/index.d.ts

@@ -0,0 +1,11 @@
+import { MastraVector, QueryResult, IndexStats } from '@mastra/core';
+export declare class QdrantVector extends MastraVector {
+    private client;
+    constructor(url: string, apiKey?: string, https?: boolean);
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>): Promise<QueryResult[]>;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    deleteIndex(indexName: string): Promise<void>;
+}