@mastra/rag 0.0.2-alpha.3 → 0.0.2-alpha.31

package/CHANGELOG.md

@@ -1,5 +1,219 @@
 # @mastra/rag
 
+## 0.0.2-alpha.31
+
+### Patch Changes
+
+- Updated dependencies [f6ba259]
+  - @mastra/core@0.1.27-alpha.47
+
+## 0.0.2-alpha.30
+
+### Patch Changes
+
+- 8ae2bbc: Dane publishing
+- 0bd142c: Fixes learned from docs
+- ee4de15: Dane fixes
+- Updated dependencies [8ae2bbc]
+- Updated dependencies [0bd142c]
+- Updated dependencies [ee4de15]
+  - @mastra/core@0.1.27-alpha.46
+
+## 0.0.2-alpha.29
+
+### Patch Changes
+
+- Updated dependencies [e608d8c]
+- Updated dependencies [002d6d8]
+  - @mastra/core@0.1.27-alpha.45
+
+## 0.0.2-alpha.28
+
+### Patch Changes
+
+- Updated dependencies [2fa7f53]
+  - @mastra/core@0.1.27-alpha.44
+
+## 0.0.2-alpha.27
+
+### Patch Changes
+
+- Updated dependencies [2e099d2]
+- Updated dependencies [d6d8159]
+  - @mastra/core@0.1.27-alpha.43
+
+## 0.0.2-alpha.26
+
+### Patch Changes
+
+- Updated dependencies [4a54c82]
+  - @mastra/core@0.1.27-alpha.42
+
+## 0.0.2-alpha.25
+
+### Patch Changes
+
+- Updated dependencies [5cdfb88]
+  - @mastra/core@0.1.27-alpha.41
+
+## 0.0.2-alpha.24
+
+### Patch Changes
+
+- Updated dependencies [9029796]
+  - @mastra/core@0.1.27-alpha.40
+
+## 0.0.2-alpha.23
+
+### Patch Changes
+
+- 6101f2d: Fix module incompatibility, and dane cli build
+- Updated dependencies [2b01511]
+  - @mastra/core@0.1.27-alpha.39
+
+## 0.0.2-alpha.22
+
+### Patch Changes
+
+- f031a1f: expose embed from rag, and refactor embed
+- Updated dependencies [f031a1f]
+  - @mastra/core@0.1.27-alpha.38
+
+## 0.0.2-alpha.21
+
+### Patch Changes
+
+- 45fd5b8: rename MastraDocument to MDocument
+- Updated dependencies [c872875]
+- Updated dependencies [f6da688]
+- Updated dependencies [b5393f1]
+  - @mastra/core@0.1.27-alpha.37
+
+## 0.0.2-alpha.20
+
+### Patch Changes
+
+- Updated dependencies [f537e33]
+- Updated dependencies [bc40916]
+- Updated dependencies [f7d1131]
+- Updated dependencies [75bf3f0]
+- Updated dependencies [3c4488b]
+- Updated dependencies [d38f7a6]
+  - @mastra/core@0.1.27-alpha.36
+
+## 0.0.2-alpha.19
+
+### Patch Changes
+
+- 033eda6: More fixes for refactor
+- Updated dependencies [033eda6]
+  - @mastra/core@0.1.27-alpha.35
+
+## 0.0.2-alpha.18
+
+### Patch Changes
+
+- 837a288: MAJOR Revamp of tools, workflows, syncs.
+- 5811de6: Updates spec-writer example to use new workflows constructs. Small improvements to workflow internals. Switch transformer tokenizer for js compatible one.
+- Updated dependencies [837a288]
+- Updated dependencies [5811de6]
+  - @mastra/core@0.1.27-alpha.34
+
+## 0.0.2-alpha.17
+
+### Patch Changes
+
+- e1dd94a: update the api for embeddings
+- Updated dependencies [e1dd94a]
+  - @mastra/core@0.1.27-alpha.33
+
+## 0.0.2-alpha.16
+
+### Patch Changes
+
+- Updated dependencies [2712098]
+  - @mastra/core@0.1.27-alpha.32
+
+## 0.0.2-alpha.15
+
+### Patch Changes
+
+- Updated dependencies [c2dd6b5]
+  - @mastra/core@0.1.27-alpha.31
+
+## 0.0.2-alpha.14
+
+### Patch Changes
+
+- 1c3232a: ChromaDB
+
+## 0.0.2-alpha.13
+
+### Patch Changes
+
+- Updated dependencies [963c15a]
+  - @mastra/core@0.1.27-alpha.30
+
+## 0.0.2-alpha.12
+
+### Patch Changes
+
+- Updated dependencies [7d87a15]
+  - @mastra/core@0.1.27-alpha.29
+
+## 0.0.2-alpha.11
+
+### Patch Changes
+
+- Updated dependencies [1ebd071]
+  - @mastra/core@0.1.27-alpha.28
+
+## 0.0.2-alpha.10
+
+### Patch Changes
+
+- 779702b: Upstash vector
+
+## 0.0.2-alpha.9
+
+### Patch Changes
+
+- Updated dependencies [cd02c56]
+  - @mastra/core@0.1.27-alpha.27
+
+## 0.0.2-alpha.8
+
+### Patch Changes
+
+- Updated dependencies [d5e12de]
+  - @mastra/core@0.1.27-alpha.26
+
+## 0.0.2-alpha.7
+
+### Patch Changes
+
+- 24fe87e: Change document semantics
+
+## 0.0.2-alpha.6
+
+### Patch Changes
+
+- Updated dependencies [01502b0]
+  - @mastra/core@0.1.27-alpha.25
+
+## 0.0.2-alpha.5
+
+### Patch Changes
+
+- 036ee5e: Add astra-db to rag
+
+## 0.0.2-alpha.4
+
+### Patch Changes
+
+- Updated dependencies [836f4e3]
+  - @mastra/core@0.1.27-alpha.24
+
 ## 0.0.2-alpha.3
 
 ### Patch Changes

package/dist/astra-db/index.d.ts

@@ -0,0 +1,54 @@
+import { MastraVector, QueryResult, IndexStats } from '@mastra/core';
+export interface AstraDbOptions {
+    token: string;
+    endpoint: string;
+    keyspace?: string;
+}
+export declare class AstraVector extends MastraVector {
+    #private;
+    constructor({ token, endpoint, keyspace }: AstraDbOptions);
+    /**
+     * Creates a new collection with the specified configuration.
+     *
+     * @param {string} indexName - The name of the collection to create.
+     * @param {number} dimension - The dimension of the vectors to be stored in the collection.
+     * @param {'cosine' | 'euclidean' | 'dotproduct'} [metric=cosine] - The metric to use to sort vectors in the collection.
+     * @returns {Promise<void>} A promise that resolves when the collection is created.
+     */
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    /**
+     * Inserts or updates vectors in the specified collection.
+     *
+     * @param {string} indexName - The name of the collection to upsert into.
+     * @param {number[][]} vectors - An array of vectors to upsert.
+     * @param {Record<string, any>[]} [metadata] - An optional array of metadata objects corresponding to each vector.
+     * @param {string[]} [ids] - An optional array of IDs corresponding to each vector. If not provided, new IDs will be generated.
+     * @returns {Promise<string[]>} A promise that resolves to an array of IDs of the upserted vectors.
+     */
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    /**
+     * Queries the specified collection using a vector and optional filter.
+     *
+     * @param {string} indexName - The name of the collection to query.
+     * @param {number[]} queryVector - The vector to query with.
+     * @param {number} [topK] - The maximum number of results to return.
+     * @param {Record<string, any>} [filter] - An optional filter to apply to the query. For more on filters in Astra DB, see the filtering reference: https://docs.datastax.com/en/astra-db-serverless/api-reference/documents.html#operators
+     * @returns {Promise<QueryResult[]>} A promise that resolves to an array of query results.
+     */
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>): Promise<QueryResult[]>;
+    /**
+     * Lists all collections in the database.
+     *
+     * @returns {Promise<string[]>} A promise that resolves to an array of collection names.
+     */
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    /**
+     * Deletes the specified collection.
+     *
+     * @param {string} indexName - The name of the collection to delete.
+     * @returns {Promise<void>} A promise that resolves when the collection is deleted.
+     */
+    deleteIndex(indexName: string): Promise<void>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/astra-db/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/astra-db/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AASrE,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,WAAY,SAAQ,YAAY;;gBAG/B,EAAE,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,cAAc;IAMzD;;;;;;;OAOG;IACG,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,QAAQ,GAAG,WAAW,GAAG,YAAuB,GACvD,OAAO,CAAC,IAAI,CAAC;IAUhB;;;;;;;;OAQG;IACG,MAAM,CACV,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAChC,GAAG,CAAC,EAAE,MAAM,EAAE,GACb,OAAO,CAAC,MAAM,EAAE,CAAC;IAgBpB;;;;;;;;OAQG;IACG,KAAK,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,CAAC,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC3B,OAAO,CAAC,WAAW,EAAE,CAAC;IAkBzB;;;;OAIG;IACH,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAI1B,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAiB3D;;;;;OAKG;IACG,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIpD"}

package/dist/chroma/index.d.ts

@@ -0,0 +1,21 @@
+import { MastraVector, QueryResult, IndexStats } from '@mastra/core';
+export declare class ChromaVector extends MastraVector {
+    private client;
+    private collections;
+    constructor({ path, auth, }: {
+        path: string;
+        auth?: {
+            provider: string;
+            credentials: string;
+        };
+    });
+    private getCollection;
+    private validateVectorDimensions;
+    upsert(indexName: string, vectors: number[][], metadata?: Record<string, any>[], ids?: string[]): Promise<string[]>;
+    createIndex(indexName: string, dimension: number, metric?: 'cosine' | 'euclidean' | 'dotproduct'): Promise<void>;
+    query(indexName: string, queryVector: number[], topK?: number, filter?: Record<string, any>): Promise<QueryResult[]>;
+    listIndexes(): Promise<string[]>;
+    describeIndex(indexName: string): Promise<IndexStats>;
+    deleteIndex(indexName: string): Promise<void>;
+}
+//# sourceMappingURL=index.d.ts.map

package/dist/chroma/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/chroma/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAGrE,qBAAa,YAAa,SAAQ,YAAY;IAC5C,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,WAAW,CAAmB;gBAE1B,EACV,IAAI,EACJ,IAAI,GACL,EAAE;QACD,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE;YACL,QAAQ,EAAE,MAAM,CAAC;YACjB,WAAW,EAAE,MAAM,CAAC;SACrB,CAAC;KACH;YASa,aAAa;IAa3B,OAAO,CAAC,wBAAwB;IAU1B,MAAM,CACV,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EAAE,EAAE,EACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,EAChC,GAAG,CAAC,EAAE,MAAM,EAAE,GACb,OAAO,CAAC,MAAM,EAAE,CAAC;IAwBd,WAAW,CACf,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,MAAM,GAAE,QAAQ,GAAG,WAAW,GAAG,YAAuB,GACvD,OAAO,CAAC,IAAI,CAAC;IAUV,KAAK,CACT,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,MAAM,EAAE,EACrB,IAAI,GAAE,MAAW,EACjB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC3B,OAAO,CAAC,WAAW,EAAE,CAAC;IAiBnB,WAAW,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IAKhC,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAYrD,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIpD"}

package/dist/document/document.d.ts

@@ -0,0 +1,32 @@
+import { Document as Chunk } from 'llamaindex';
+import { ChunkOptions, ChunkParams, ExtractParams } from './types';
+export declare class MDocument {
+    private chunks;
+    private type;
+    constructor({ docs, type }: {
+        docs: {
+            text: string;
+            metadata?: Record<string, any>;
+        }[];
+        type: string;
+    });
+    extractMetadata({ title, summary, questions, keywords }: ExtractParams): Promise<MDocument>;
+    static fromText(text: string, metadata?: Record<string, any>): MDocument;
+    static fromHTML(html: string, metadata?: Record<string, any>): MDocument;
+    static fromMarkdown(markdown: string, metadata?: Record<string, any>): MDocument;
+    static fromJSON(jsonString: string, metadata?: Record<string, any>): MDocument;
+    private defaultStrategy;
+    private chunkBy;
+    chunkRecursive(options?: ChunkOptions): Promise<void>;
+    chunkCharacter(options?: ChunkOptions): Promise<void>;
+    chunkHTML(options?: ChunkOptions): Promise<void>;
+    chunkJSON(options?: ChunkOptions): Promise<void>;
+    chunkLatex(options?: ChunkOptions): Promise<void>;
+    chunkToken(options?: ChunkOptions): Promise<void>;
+    chunkMarkdown(options?: ChunkOptions): Promise<void>;
+    chunk(params?: ChunkParams): Promise<MDocument['chunks']>;
+    getDocs(): Chunk[];
+    getText(): string[];
+    getMetadata(): Record<string, any>[];
+}
+//# sourceMappingURL=document.d.ts.map

package/dist/document/document.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/document/document.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,QAAQ,IAAI,KAAK,EAMlB,MAAM,YAAY,CAAC;AAQpB,OAAO,EAAE,YAAY,EAAE,WAAW,EAAiB,aAAa,EAAE,MAAM,SAAS,CAAC;AAElF,qBAAa,SAAS;IACpB,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,IAAI,CAAS;gBAET,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAOhG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC;IAwCjG,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYxE,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAYhF,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS;IAY9E,OAAO,CAAC,eAAe;YAeT,OAAO;IA4Bf,cAAc,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBrD,cAAc,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAUrD,SAAS,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBhD,SAAS,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBhD,UAAU,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAMjD,UAAU,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAUjD,aAAa,CAAC,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC;IAapD,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAe/D,OAAO,IAAI,KAAK,EAAE;IAIlB,OAAO,IAAI,MAAM,EAAE;IAInB,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;CAGrC"}

package/dist/document/index.d.ts

@@ -0,0 +1,3 @@
+export * from './document';
+export * from './types';
+//# sourceMappingURL=index.d.ts.map

package/dist/document/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/document/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,SAAS,CAAC"}

package/dist/document/transformers/character.d.ts

@@ -0,0 +1,45 @@
+import { ChunkOptions, Language } from '../types';
+import { TextTransformer } from './text';
+export declare class CharacterTransformer extends TextTransformer {
+    protected separator: string;
+    protected isSeparatorRegex: boolean;
+    constructor({ separator, isSeparatorRegex, options, }: {
+        separator?: string;
+        isSeparatorRegex?: boolean;
+        options?: {
+            size?: number;
+            overlap?: number;
+            lengthFunction?: (text: string) => number;
+            keepSeparator?: boolean | 'start' | 'end';
+            addStartIndex?: boolean;
+            stripWhitespace?: boolean;
+        };
+    });
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    private __splitChunk;
+}
+export declare class RecursiveCharacterTransformer extends TextTransformer {
+    protected separators: string[];
+    protected isSeparatorRegex: boolean;
+    constructor({ separators, isSeparatorRegex, options, }: {
+        separators?: string[];
+        isSeparatorRegex?: boolean;
+        options?: ChunkOptions;
+    });
+    private _splitText;
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    static fromLanguage(language: Language, options?: {
+        size?: number;
+        chunkOverlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    }): RecursiveCharacterTransformer;
+    static getSeparatorsForLanguage(language: Language): string[];
+}
+//# sourceMappingURL=character.d.ts.map

package/dist/document/transformers/character.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"character.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/character.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AA+CzC,qBAAa,oBAAqB,SAAQ,eAAe;IACvD,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EACV,SAAkB,EAClB,gBAAwB,EACxB,OAAY,GACb,EAAE;QACD,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;YACxB,eAAe,CAAC,EAAE,OAAO,CAAC;SAC3B,CAAC;KACH;IAMD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAqB/C,OAAO,CAAC,YAAY;CAwBrB;AAED,qBAAa,6BAA8B,SAAQ,eAAe;IAChE,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;IAC/B,SAAS,CAAC,gBAAgB,EAAE,OAAO,CAAC;gBAExB,EACV,UAAU,EACV,gBAAwB,EACxB,OAAY,GACb,EAAE;QACD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;QACtB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,OAAO,CAAC,EAAE,YAAY,CAAC;KACxB;IAMD,OAAO,CAAC,UAAU;IAuDlB,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAI/C,MAAM,CAAC,YAAY,CACjB,QAAQ,EAAE,QAAQ,EAClB,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB,GACL,6BAA6B;IAKhC,MAAM,CAAC,wBAAwB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,EAAE;CAgE9D"}

package/dist/document/transformers/html.d.ts

@@ -0,0 +1,24 @@
+import { Document } from 'llamaindex';
+export declare class HTMLHeaderTransformer {
+    private headersToSplitOn;
+    private returnEachElement;
+    constructor(headersToSplitOn: [string, string][], returnEachElement?: boolean);
+    splitText({ text }: {
+        text: string;
+    }): Document[];
+    private getXPath;
+    private aggregateElementsToChunks;
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+}
+export declare class HTMLSectionTransformer {
+    private headersToSplitOn;
+    private options;
+    constructor(headersToSplitOn: [string, string][], options?: Record<string, any>);
+    splitText(text: string): Document[];
+    splitDocuments(documents: Document[]): Promise<Document[]>;
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    private splitHtmlByHeaders;
+    transformDocuments(documents: Document[]): Document[];
+}
+//# sourceMappingURL=html.d.ts.map

package/dist/document/transformers/html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/html.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAWtC,qBAAa,qBAAqB;IAChC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,iBAAiB,CAAU;gBAEvB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,iBAAiB,GAAE,OAAe;IAKpF,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAwCjD,OAAO,CAAC,QAAQ;IAkBhB,OAAO,CAAC,yBAAyB;IAyBjC,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD;AAED,qBAAa,sBAAsB;IACjC,OAAO,CAAC,gBAAgB,CAAyB;IACjD,OAAO,CAAC,OAAO,CAAsB;gBAEzB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAM;IAKnF,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,EAAE;IAc7B,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAchE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA+B/E,OAAO,CAAC,kBAAkB;IAuD1B,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/document/transformers/json.d.ts

@@ -0,0 +1,57 @@
+import { Document } from 'llamaindex';
+export declare class RecursiveJsonTransformer {
+    private maxSize;
+    private minSize;
+    constructor({ maxSize, minSize }: {
+        maxSize: number;
+        minSize?: number;
+    });
+    private static jsonSize;
+    /**
+     * Transform JSON data while handling circular references
+     */
+    transform(data: Record<string, any>): Record<string, any>;
+    /**
+     * Set a value in a nested dictionary based on the given path
+     */
+    private static setNestedDict;
+    /**
+     * Convert lists in the JSON structure to dictionaries with index-based keys
+     */
+    private listToDictPreprocessing;
+    /**
+     * Split json into maximum size dictionaries while preserving structure
+     */
+    private jsonSplit;
+    /**
+     * Splits JSON into a list of JSON chunks
+     */
+    splitJson({ jsonData, convertLists, }: {
+        jsonData: Record<string, any>;
+        convertLists?: boolean;
+    }): Record<string, any>[];
+    private escapeNonAscii;
+    /**
+     * Splits JSON into a list of JSON formatted strings
+     */
+    splitText({ jsonData, convertLists, ensureAscii, }: {
+        jsonData: Record<string, any>;
+        convertLists?: boolean;
+        ensureAscii?: boolean;
+    }): string[];
+    /**
+     * Create documents from a list of json objects
+     */
+    createDocuments({ texts, convertLists, ensureAscii, metadatas, }: {
+        texts: string[];
+        convertLists?: boolean;
+        ensureAscii?: boolean;
+        metadatas?: Record<string, any>[];
+    }): Document[];
+    transformDocuments({ ensureAscii, documents, convertLists, }: {
+        ensureAscii?: boolean;
+        convertLists?: boolean;
+        documents: Document[];
+    }): Document[];
+}
+//# sourceMappingURL=json.d.ts.map

package/dist/document/transformers/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/json.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,qBAAa,wBAAwB;IACnC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,EAAE,OAAc,EAAE,OAAO,EAAE,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE;IAK9E,OAAO,CAAC,MAAM,CAAC,QAAQ;IAmCvB;;OAEG;IACI,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAmChE;;OAEG;IACH,OAAO,CAAC,MAAM,CAAC,aAAa;IAS5B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAU/B;;OAEG;IACH,OAAO,CAAC,SAAS;IAsCjB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,GACrB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;IAYzB,OAAO,CAAC,cAAc;IAatB;;OAEG;IACH,SAAS,CAAC,EACR,QAAQ,EACR,YAAoB,EACpB,WAAkB,GACnB,EAAE;QACD,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;KACvB,GAAG,MAAM,EAAE;IAWZ;;OAEG;IACH,eAAe,CAAC,EACd,KAAK,EACL,YAAoB,EACpB,WAAkB,EAClB,SAAS,GACV,EAAE;QACD,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;KACnC,GAAG,QAAQ,EAAE;IAoBd,kBAAkB,CAAC,EACjB,WAAW,EACX,SAAS,EACT,YAAY,GACb,EAAE;QACD,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,YAAY,CAAC,EAAE,OAAO,CAAC;QACvB,SAAS,EAAE,QAAQ,EAAE,CAAC;KACvB,GAAG,QAAQ,EAAE;CAiBf"}

package/dist/document/transformers/latex.d.ts

@@ -0,0 +1,12 @@
+import { RecursiveCharacterTransformer } from './character';
+export declare class LatexTransformer extends RecursiveCharacterTransformer {
+    constructor(options?: {
+        size?: number;
+        overlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    });
+}
+//# sourceMappingURL=latex.d.ts.map

package/dist/document/transformers/latex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"latex.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/latex.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAE5D,qBAAa,gBAAiB,SAAQ,6BAA6B;gBAE/D,OAAO,GAAE;QACP,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;CAKT"}

package/dist/document/transformers/markdown.d.ts

@@ -0,0 +1,25 @@
+import { Document } from 'llamaindex';
+import { RecursiveCharacterTransformer } from './character';
+export declare class MarkdownTransformer extends RecursiveCharacterTransformer {
+    constructor(options?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        lengthFunction?: (text: string) => number;
+        keepSeparator?: boolean | 'start' | 'end';
+        addStartIndex?: boolean;
+        stripWhitespace?: boolean;
+    });
+}
+export declare class MarkdownHeaderTransformer {
+    private headersToSplitOn;
+    private returnEachLine;
+    private stripHeaders;
+    constructor(headersToSplitOn: [string, string][], returnEachLine?: boolean, stripHeaders?: boolean);
+    private aggregateLinesToChunks;
+    splitText({ text }: {
+        text: string;
+    }): Document[];
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+}
+//# sourceMappingURL=markdown.d.ts.map

package/dist/document/transformers/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAItC,OAAO,EAAE,6BAA6B,EAAE,MAAM,aAAa,CAAC;AAa5D,qBAAa,mBAAoB,SAAQ,6BAA6B;gBAElE,OAAO,GAAE;QACP,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;QAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;CAKT;AAED,qBAAa,yBAAyB;IACpC,OAAO,CAAC,gBAAgB,CAAqB;IAC7C,OAAO,CAAC,cAAc,CAAU;IAChC,OAAO,CAAC,YAAY,CAAU;gBAElB,gBAAgB,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,cAAc,GAAE,OAAe,EAAE,YAAY,GAAE,OAAc;IAM/G,OAAO,CAAC,sBAAsB;IAsD9B,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,QAAQ,EAAE;IAmHjD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IAmB/E,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;CAWtD"}

package/dist/document/transformers/text.d.ts

@@ -0,0 +1,22 @@
+import { Document } from 'llamaindex';
+import { ChunkOptions } from '../types';
+import { Transformer } from './transformer';
+export declare abstract class TextTransformer implements Transformer {
+    protected size: number;
+    protected overlap: number;
+    protected lengthFunction: (text: string) => number;
+    protected keepSeparator: boolean | 'start' | 'end';
+    protected addStartIndex: boolean;
+    protected stripWhitespace: boolean;
+    constructor({ size, overlap, lengthFunction, keepSeparator, addStartIndex, stripWhitespace, }: ChunkOptions);
+    setAddStartIndex(value: boolean): void;
+    abstract splitText({ text }: {
+        text: string;
+    }): string[];
+    createDocuments(texts: string[], metadatas?: Record<string, any>[]): Document[];
+    splitDocuments(documents: Document[]): Document[];
+    transformDocuments(documents: Document[]): Document[];
+    protected joinDocs(docs: string[], separator: string): string | null;
+    protected mergeSplits(splits: string[], separator: string): string[];
+}
+//# sourceMappingURL=text.d.ts.map

package/dist/document/transformers/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/text.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,8BAAsB,eAAgB,YAAW,WAAW;IAC1D,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC;IAC1B,SAAS,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;IACnD,SAAS,CAAC,aAAa,EAAE,OAAO,CAAC;IACjC,SAAS,CAAC,eAAe,EAAE,OAAO,CAAC;gBAEvB,EACV,IAAW,EACX,OAAa,EACb,cAA8C,EAC9C,aAAqB,EACrB,aAAqB,EACrB,eAAsB,GACvB,EAAE,YAAY;IAYf,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAItC,QAAQ,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IAExD,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE;IA4B/E,cAAc,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAUjD,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE;IAYrD,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAQpE,SAAS,CAAC,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;CAoCrE"}

package/dist/document/transformers/token.d.ts

@@ -0,0 +1,46 @@
+import { TiktokenModel, TiktokenEncoding } from 'js-tiktoken';
+import { TextTransformer } from './text';
+interface Tokenizer {
+    overlap: number;
+    tokensPerChunk: number;
+    decode: (tokens: number[]) => string;
+    encode: (text: string) => number[];
+}
+export declare function splitTextOnTokens({ text, tokenizer }: {
+    text: string;
+    tokenizer: Tokenizer;
+}): string[];
+export declare class TokenTransformer extends TextTransformer {
+    private tokenizer;
+    private allowedSpecial;
+    private disallowedSpecial;
+    constructor({ encodingName, modelName, allowedSpecial, disallowedSpecial, options, }: {
+        encodingName: TiktokenEncoding;
+        modelName?: TiktokenModel;
+        allowedSpecial?: Set<string> | 'all';
+        disallowedSpecial?: Set<string> | 'all';
+        options: {
+            size?: number;
+            overlap?: number;
+            lengthFunction?: (text: string) => number;
+            keepSeparator?: boolean | 'start' | 'end';
+            addStartIndex?: boolean;
+            stripWhitespace?: boolean;
+        };
+    });
+    splitText({ text }: {
+        text: string;
+    }): string[];
+    static fromTikToken({ encodingName, modelName, options, }: {
+        encodingName?: TiktokenEncoding;
+        modelName?: TiktokenModel;
+        options?: {
+            size?: number;
+            overlap?: number;
+            allowedSpecial?: Set<string> | 'all';
+            disallowedSpecial?: Set<string> | 'all';
+        };
+    }): TokenTransformer;
+}
+export {};
+//# sourceMappingURL=token.d.ts.map

package/dist/document/transformers/token.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAA2C,aAAa,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEvG,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAC;AAEzC,UAAU,SAAS;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,iBAAiB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,MAAM,EAAE,CAkBvG;AAED,qBAAa,gBAAiB,SAAQ,eAAe;IACnD,OAAO,CAAC,SAAS,CAAW;IAC5B,OAAO,CAAC,cAAc,CAAsB;IAC5C,OAAO,CAAC,iBAAiB,CAAsB;gBAEnC,EACV,YAA4B,EAC5B,SAAS,EACT,cAA0B,EAC1B,iBAAyB,EACzB,OAAY,GACb,EAAE;QACD,YAAY,EAAE,gBAAgB,CAAC;QAC/B,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;QACxC,OAAO,EAAE;YACP,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC;YAC1C,aAAa,CAAC,EAAE,OAAO,CAAC;YACxB,eAAe,CAAC,EAAE,OAAO,CAAC;SAC3B,CAAC;KACH;IAaD,SAAS,CAAC,EAAE,IAAI,EAAE,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,EAAE;IA0B/C,MAAM,CAAC,YAAY,CAAC,EAClB,YAA4B,EAC5B,SAAS,EACT,OAAY,GACb,EAAE;QACD,YAAY,CAAC,EAAE,gBAAgB,CAAC;QAChC,SAAS,CAAC,EAAE,aAAa,CAAC;QAC1B,OAAO,CAAC,EAAE;YACR,IAAI,CAAC,EAAE,MAAM,CAAC;YACd,OAAO,CAAC,EAAE,MAAM,CAAC;YACjB,cAAc,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;YACrC,iBAAiB,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC;SACzC,CAAC;KACH,GAAG,gBAAgB;CAuCrB"}

package/dist/document/transformers/transformer.d.ts

@@ -0,0 +1,5 @@
+import { Document } from 'llamaindex';
+export interface Transformer {
+    transformDocuments(documents: Document[]): Document[];
+}
+//# sourceMappingURL=transformer.d.ts.map

package/dist/document/transformers/transformer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transformer.d.ts","sourceRoot":"","sources":["../../../src/document/transformers/transformer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,MAAM,WAAW,WAAW;IAC1B,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAAC;CACvD"}