@exulu/backend 0.3.4 → 0.3.6

package/dist/index.d.cts

@@ -5,7 +5,8 @@ import { ZodSchema, z } from 'zod';
 import { Tool, LanguageModelV1 } from 'ai';
 import { Express } from 'express';
 import { Knex } from 'knex';
-import { SentenceChunker, RecursiveChunker, RecursiveRules } from 'chonkie';
+import { Tiktoken } from 'tiktoken/lite';
+import models from 'tiktoken/model_to_encoding.json';
 
 declare function redisClient(): Promise<{
     client: RedisClientType | null;
@@ -526,6 +527,863 @@ declare class ExuluQueues {
 }
 declare const queues: ExuluQueues;
 
+/**
+ * Represents the data structure for a chunk object.
+ *
+ * @property {string} text - The text of the chunk.
+ * @property {number} startIndex - The starting index of the chunk in the original text.
+ * @property {number} endIndex - The ending index of the chunk in the original text.
+ * @property {number} tokenCount - The number of tokens in the chunk.
+ */
+interface ChunkData {
+    text: string;
+    startIndex: number;
+    endIndex: number;
+    tokenCount: number;
+    embedding?: number[];
+}
+/**
+ * Represents a chunk of text with associated metadata.
+ *
+ * @property {string} text - The text of the chunk.
+ * @property {number} startIndex - The starting index of the chunk in the original text.
+ * @property {number} endIndex - The ending index of the chunk in the original text.
+ * @property {number} tokenCount - The number of tokens in the chunk.
+ * @property {number[]} [embedding] - The embedding for the chunk.
+ */
+declare class Chunk {
+    /** The text of the chunk. */
+    text: string;
+    /** The starting index of the chunk in the original text. */
+    startIndex: number;
+    /** The ending index of the chunk in the original text. */
+    endIndex: number;
+    /** The number of tokens in the chunk. */
+    tokenCount: number;
+    /** Optional embedding for the chunk. */
+    embedding?: number[];
+    /**
+     * Constructs a new Chunk object.
+     *
+     * @param {ChunkData} data - The data to construct the Chunk from.
+     */
+    constructor(data: {
+        text: string;
+        startIndex: number;
+        endIndex: number;
+        tokenCount: number;
+        embedding?: number[];
+    });
+    /** Return a string representation of the Chunk.
+     *
+     * @returns {string} The text of the chunk.
+     */
+    toString(): string;
+    /** Return a detailed string representation of the Chunk.
+     *
+     * @returns {string} The detailed string representation of the Chunk.
+     */
+    toRepresentation(): string;
+    /** Return a slice of the chunk's text.
+     *
+     * @param {number} [start] - The starting index of the slice.
+     * @param {number} [end] - The ending index of the slice.
+     * @returns {string} The slice of the chunk's text.
+     */
+    slice(start?: number, end?: number): string;
+    /** Return the Chunk as a dictionary-like object.
+     *
+     * @returns {ChunkData} The dictionary-like object.
+     */
+    toDict(): ChunkData;
+    /** Create a Chunk object from a dictionary-like object.
+     *
+     * @param {ChunkData} data - The dictionary-like object.
+     * @returns {Chunk} The Chunk object.
+     */
+    static fromDict(data: ChunkData): Chunk;
+    /** Return a deep copy of the chunk.
+     *
+     * @returns {Chunk} The deep copy of the chunk.
+     */
+    copy(): Chunk;
+}
+
+/** Type for include delimiter options
+ *
+ * @enum {string}
+ */
+type IncludeDelim = 'prev' | 'next';
+/** Interface for RecursiveLevel data
+ *
+ * @interface RecursiveLevelData
+ * @property {string | string[]} [delimiters] - The delimiters to use for chunking.
+ * @property {boolean} [whitespace] - Whether to use whitespace as a delimiter.
+ * @property {IncludeDelim} [includeDelim] - Whether to include the delimiter in the previous or next chunk.
+ */
+interface RecursiveLevelData {
+    delimiters?: string | string[];
+    whitespace?: boolean;
+    includeDelim?: IncludeDelim;
+}
+/** Class to represent recursive chunking rules at a specific level
+ *
+ * @class RecursiveLevel
+ * @property {string | string[]} [delimiters] - The delimiters to use for chunking.
+ * @property {boolean} [whitespace] - Whether to use whitespace as a delimiter.
+ * @property {IncludeDelim} [includeDelim] - Whether to include the delimiter in the previous or next chunk.
+ */
+declare class RecursiveLevel {
+    /** Custom delimiters for chunking */
+    delimiters?: string | string[];
+    /** Whether to use whitespace as a delimiter */
+    whitespace: boolean;
+    /** Whether to include the delimiter in the previous or next chunk */
+    includeDelim: IncludeDelim;
+    /**
+     * Constructs a new RecursiveLevel object.
+     *
+     * @param {RecursiveLevelData} data - The data to construct the RecursiveLevel from.
+     */
+    constructor(data?: RecursiveLevelData);
+    /**
+     * Validates the RecursiveLevel object.
+     *
+     * @private
+     */
+    private validate;
+    /** Return a string representation of the RecursiveLevel
+     *
+     * @returns {string} The string representation of the RecursiveLevel.
+     */
+    toString(): string;
+    /** Return the RecursiveLevel as a dictionary-like object
+     *
+     * @returns {RecursiveLevelData} The dictionary-like object.
+     */
+    toDict(): RecursiveLevelData;
+    /** Create RecursiveLevel object from a dictionary
+     *
+     * @param {RecursiveLevelData} data - The dictionary-like object.
+     * @returns {RecursiveLevel} The RecursiveLevel object.
+     */
+    static fromDict(data: RecursiveLevelData): RecursiveLevel;
+    /** Create RecursiveLevel object from a recipe
+     *
+     * @param {string} name - The name of the recipe.
+     * @param {string} lang - The language of the recipe.
+     * @returns {Promise<RecursiveLevel>} The RecursiveLevel object.
+     */
+    static fromRecipe(name: string, lang?: string): Promise<RecursiveLevel>;
+}
+/** Interface for RecursiveRules data
+ *
+ * @interface RecursiveRulesData
+ * @property {RecursiveLevelData[]} [levels] - The recursive levels.
+ */
+interface RecursiveRulesData {
+    levels?: RecursiveLevelData[];
+}
+/** Class to represent recursive chunking rules
+ *
+ * @class RecursiveRules
+ * @property {RecursiveLevel[]} [levels] - The recursive levels.
+ */
+declare class RecursiveRules {
+    /** List of recursive levels */
+    levels: RecursiveLevel[];
+    constructor(data?: RecursiveRulesData);
+    /** Return a string representation of the RecursiveRules
+     *
+     * @returns {string} The string representation of the RecursiveRules.
+     */
+    toString(): string;
+    /** Return the number of levels
+     *
+     * @returns {number} The number of levels.
+     */
+    get length(): number;
+    /** Get a level by index
+     *
+     * @param {number} index - The index of the level.
+     * @returns {RecursiveLevel | undefined} The level.
+     */
+    getLevel(index: number): RecursiveLevel | undefined;
+    /** Return an iterator over the levels
+     *
+     * @returns {Iterator<RecursiveLevel>} The iterator over the levels.
+     */
+    [Symbol.iterator](): Iterator<RecursiveLevel>;
+    /** Create a RecursiveRules object from a dictionary
+     *
+     * @param {RecursiveRulesData} data - The dictionary-like object.
+     * @returns {RecursiveRules} The RecursiveRules object.
+     */
+    static fromDict(data: RecursiveRulesData): RecursiveRules;
+    /** Return the RecursiveRules as a dictionary-like object
+     *
+     * @returns {RecursiveRulesData} The dictionary-like object.
+     */
+    toDict(): RecursiveRulesData;
+    /** Create a RecursiveRules object from a recipe
+     *
+     * @param {string} name - The name of the recipe.
+     * @param {string} lang - The language of the recipe.
+     * @param {string} path - The path to the recipe.
+     * @returns {Promise<RecursiveRules>} The RecursiveRules object.
+     */
+    static fromRecipe(name?: string, lang?: string, path?: string): Promise<RecursiveRules>;
+}
+/** Interface for RecursiveChunk data
+ *
+ * @interface RecursiveChunkData
+ * @property {string} text - The text of the chunk.
+ * @property {number} startIndex - The starting index of the chunk.
+ * @property {number} endIndex - The ending index of the chunk.
+ * @property {number} tokenCount - The number of tokens in the chunk.
+ * @property {number} [level] - The level of recursion for the chunk.
+ */
+interface RecursiveChunkData {
+    text: string;
+    startIndex: number;
+    endIndex: number;
+    tokenCount: number;
+    level?: number;
+    embedding?: number[];
+}
+/** Class to represent recursive chunks
+ *
+ * @class RecursiveChunk
+ * @property {number} [level] - The level of recursion for the chunk.
+ */
+declare class RecursiveChunk extends Chunk {
+    /** The level of recursion for the chunk */
+    level?: number;
+    constructor(data: {
+        text: string;
+        startIndex: number;
+        endIndex: number;
+        tokenCount: number;
+        level?: number;
+        embedding?: number[];
+    });
+    /** Return a string representation of the RecursiveChunk
+     *
+     * @returns {string} The string representation of the RecursiveChunk.
+     */
+    toString(): string;
+    /** Return the RecursiveChunk as a dictionary-like object
+     *
+     * @returns {RecursiveChunkData} The dictionary-like object.
+     */
+    toDict(): RecursiveChunkData;
+    /** Create a RecursiveChunk object from a dictionary
+     *
+     * @param {RecursiveChunkData} data - The dictionary-like object.
+     * @returns {RecursiveChunk} The RecursiveChunk object.
+     */
+    static fromDict(data: RecursiveChunkData): RecursiveChunk;
+}
+
+type TokenizerModelName = keyof typeof models;
+declare class ExuluTokenizer {
+    constructor();
+    encoder: Tiktoken | null;
+    create(modelName: TokenizerModelName): Promise<Tiktoken>;
+    decode(tokens: Uint32Array): Promise<string>;
+    decodeBatch(tokenSequences: Uint32Array[]): Promise<string[]>;
+    encode(text: string): Uint32Array;
+    countTokensBatch(texts: string[]): Promise<number[]>;
+    countTokens(text: string): number;
+    free(): Promise<void>;
+}
+
+/** Base Chunking Class. **/
+
+/**
+ * Base class for all chunking classes.
+ *
+ * This abstract class provides a common interface and shared logic for all chunking implementations.
+ * It supports chunking a single text or a batch of texts, with optional concurrency and progress reporting.
+ *
+ * Subclasses must implement the `chunk` method to define how a single text is chunked.
+ *
+ * @template T - The type of chunk produced (usually `Chunk[]` or `string[]`).
+ *
+ * @property {Tokenizer} tokenizer - The tokenizer instance used for chunking operations.
+ * @property {boolean} _useConcurrency - Whether to use concurrent processing for batch chunking (default: true).
+ *
+ * @example
+ * class MyChunker extends BaseChunker {
+ *   async chunk(text: string): Promise<Chunk[]> {
+ *     // ... implementation ...
+ *   }
+ * }
+ *
+ * const chunker = new MyChunker(tokenizer);
+ * const chunks = await chunker.call("Some text");
+ * const batchChunks = await chunker.call(["Text 1", "Text 2"], true);
+ */
+declare abstract class BaseChunker {
+    protected tokenizer: ExuluTokenizer;
+    protected _useConcurrency: boolean;
+    constructor(tokenizer: ExuluTokenizer);
+    /**
+     * Returns a string representation of the chunker instance.
+     *
+     * @returns {string} The class name and constructor signature.
+     */
+    toString(): string;
+    /**
+     * Call the chunker with a single string or an array of strings.
+     *
+     * If a single string is provided, returns the result of `chunk(text)`.
+     * If an array of strings is provided, returns the result of `chunkBatch(texts, showProgress)`.
+     *
+     * @param {string | string[]} textOrTexts - The text or array of texts to chunk.
+     * @param {boolean} [showProgress=false] - Whether to display progress for batch operations (only applies to arrays).
+     * @returns {Promise<Chunk[] | Chunk[][]>} The chunked result(s).
+     * @throws {Error} If input is not a string or array of strings.
+     */
+    call(text: string, showProgress?: boolean): Promise<Chunk[]>;
+    call(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
+    /**
+     * Process a batch of texts sequentially (one after another).
+     *
+     * @protected
+     * @param {string[]} texts - The texts to chunk.
+     * @param {boolean} [showProgress=false] - Whether to display progress in the console.
+     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+     */
+    protected _sequential_batch_processing(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
+    /**
+     * Process a batch of texts concurrently using Promise.all.
+     *
+     * @protected
+     * @param {string[]} texts - The texts to chunk.
+     * @param {boolean} [showProgress=false] - Whether to display progress in the console.
+     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+     */
+    protected _concurrent_batch_processing(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
+    /**
+     * Abstract method to chunk a single text. Must be implemented by subclasses.
+     *
+     * @param {string} text - The text to chunk.
+     * @returns {Promise<Chunk[]>} The chunked representation of the input text.
+     * @abstract
+     */
+    abstract chunk(text: string): Promise<Chunk[]>;
+    /**
+     * Chunk a batch of texts, using either concurrent or sequential processing.
+     *
+     * If only one text is provided, processes it directly without batch overhead.
+     *
+     * @param {string[]} texts - The texts to chunk.
+     * @param {boolean} [showProgress=true] - Whether to display progress in the console.
+     * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+     */
+    chunkBatch(texts: string[], showProgress?: boolean): Promise<Chunk[][]>;
+}
+
+/** Module containing RecursiveChunker class. */
+
+/**
+ * Configuration options for creating a RecursiveChunker instance.
+ * All options are optional and have sensible defaults.
+ *
+ * @interface RecursiveChunkerOptions
+ * @property {string | Tokenizer} [tokenizer] - The tokenizer to use for text processing. Can be a string identifier (default: "Xenova/gpt2") or a Tokenizer instance.
+ * @property {number} [chunkSize] - The maximum number of tokens per chunk. Must be greater than 0. Default: 512.
+ * @property {RecursiveRules} [rules] - The rules that define how text should be recursively chunked. Default: new RecursiveRules().
+ * @property {number} [minCharactersPerChunk] - The minimum number of characters that should be in each chunk. Must be greater than 0. Default: 24.
+ */
+interface RecursiveChunkerOptions {
+    tokenizer?: TokenizerModelName;
+    chunkSize?: number;
+    rules?: RecursiveRules;
+    minCharactersPerChunk?: number;
+}
+/**
+ * Represents a RecursiveChunker instance that is also directly callable as a function.
+ *
+ * This type combines all properties and methods of {@link RecursiveChunker} with callable signatures for chunking text(s).
+ *
+ * Calling the instance executes its `call` method (from {@link BaseChunker}), which in turn calls `chunk` or `chunkBatch`.
+ *
+ * @typedef {Object} CallableRecursiveChunker
+ * @property {number} chunkSize - The maximum number of tokens per chunk.
+ * @property {number} minCharactersPerChunk - The minimum number of characters per chunk.
+ * @property {RecursiveRules} rules - The rules that define how text should be recursively chunked.
+ * @property {string} sep - The separator string used for internal splitting (usually "✄").
+ * @property {Tokenizer} tokenizer - The tokenizer instance used for chunking operations (inherited from BaseChunker).
+ *
+ * @method chunk - Recursively chunk a single text into chunks or strings.
+ * @method chunkBatch - Recursively chunk a batch of texts.
+ * @method toString - Returns a string representation of the RecursiveChunker instance.
+ * @method call - Call the chunker with a single string or an array of strings. (see callable signatures)
+ *
+ * @static
+ * @method create
+ * @memberof CallableRecursiveChunker
+ * @param {RecursiveChunkerOptions} [options] - Configuration options for the RecursiveChunker.
+ * @returns {Promise<CallableRecursiveChunker>} A Promise that resolves to a callable RecursiveChunker instance.
+ *
+ * @example
+ * const chunker = await RecursiveChunker.create({ chunkSize: 256 });
+ * const chunks = await chunker("Some text to chunk");
+ * const batchChunks = await chunker(["Text 1", "Text 2"]);
+ */
+type CallableRecursiveChunker = RecursiveChunker & {
+    (text: string, showProgress?: boolean): Promise<RecursiveChunk[]>;
+    (texts: string[], showProgress?: boolean): Promise<(RecursiveChunk[])[]>;
+};
+/**
+ * Recursively chunk text using a set of rules.
+ *
+ * This class extends the BaseChunker class and implements the chunk method.
+ * It provides a flexible way to chunk text based on custom rules, including
+ * delimiters, whitespace, and token-based chunking.
+ *
+ * @extends BaseChunker
+ * @property {number} chunkSize - The maximum number of tokens per chunk.
+ * @property {number} minCharactersPerChunk - The minimum number of characters per chunk.
+ * @property {RecursiveRules} rules - The rules that define how text should be recursively chunked.
+ * @property {string} sep - The separator string used for internal splitting (usually "✄").
+ *
+ * @method chunk - Recursively chunk a single text into chunks or strings.
+ * @method chunkBatch - Recursively chunk a batch of texts.
+ * @method toString - Returns a string representation of the RecursiveChunker instance.
+ * @method call - Call the chunker with a single string or an array of strings. (see callable signatures)
+ *
+ * @static
+ * @method create
+ * @memberof RecursiveChunker
+ * @param {RecursiveChunkerOptions} [options] - Configuration options for the RecursiveChunker.
+ * @returns {Promise<RecursiveChunker>} A Promise that resolves to a RecursiveChunker instance.
+ *
+ * @example
+ * const chunker = await RecursiveChunker.create({ chunkSize: 256 });
+ * const chunks = await chunker("Some text to chunk");
+ * const batchChunks = await chunker(["Text 1", "Text 2"]);
+ */
+declare class RecursiveChunker extends BaseChunker {
+    readonly chunkSize: number;
+    readonly minCharactersPerChunk: number;
+    readonly rules: RecursiveRules;
+    readonly sep: string;
+    private readonly _CHARS_PER_TOKEN;
+    /**
+     * Private constructor. Use `RecursiveChunker.create()` to instantiate.
+     */
+    private constructor();
+    /**
+     * Creates and initializes a directly callable RecursiveChunker instance.
+     *
+     * This static factory method constructs a RecursiveChunker with the provided options and returns a callable function object.
+     * The returned instance can be used as both a function (to chunk text(s)) and as an object (with all RecursiveChunker methods and properties).
+     *
+     * @param {RecursiveChunkerOptions} [options] - Configuration options for the chunker. All options are optional:
+     *   @param {string|Tokenizer} [options.tokenizer="Xenova/gpt2"] - Tokenizer to use for text processing. Can be a string identifier (e.g., "Xenova/gpt2") or a Tokenizer instance. If a string is provided, Tokenizer.create() is called internally.
+     *   @param {number} [options.chunkSize=512] - Maximum number of tokens per chunk. Must be > 0.
+     *   @param {RecursiveRules} [options.rules=new RecursiveRules()] - Rules for recursive chunking. See {@link RecursiveRules} for customization.
+     *   @param {number} [options.minCharactersPerChunk=24] - Minimum number of characters per chunk. Must be > 0.
+     *
+     * @returns {Promise<CallableRecursiveChunker>} Promise resolving to a callable RecursiveChunker instance.
+     *
+     * @throws {Error} If any option is invalid (e.g., chunkSize <= 0).
+     *
+     * @see CallableRecursiveChunker for the callable interface and available properties/methods.
+     *
+     * @example <caption>Basic usage with default options</caption>
+     * const chunker = await RecursiveChunker.create();
+     * const chunks = await chunker("Some text to chunk");
+     *
+     * @example <caption>Custom options and batch chunking</caption>
+     * const chunker = await RecursiveChunker.create({ chunkSize: 256 });
+     * const batchChunks = await chunker(["Text 1", "Text 2"]);
+     *
+     * @example <caption>Accessing properties and methods</caption>
+     * const chunker = await RecursiveChunker.create();
+     * console.log(chunker.chunkSize); // 512
+     * console.log(chunker.rules); // RecursiveRules instance
+     * const chunks = await chunker.chunk("Some text"); // Use as object method
+     *
+     * @note
+     * The returned instance is both callable (like a function) and has all properties/methods of RecursiveChunker.
+     * You can use it as a drop-in replacement for a function or a class instance.
+     *
+     * @note
+     * For advanced customization, pass a custom RecursiveRules object to the rules option.
+     * See {@link RecursiveRules} and {@link RecursiveLevel} for rule structure.
+     */
+    static create(options?: RecursiveChunkerOptions): Promise<CallableRecursiveChunker>;
+    /**
+     * Estimates the number of tokens in a given text.
+     *
+     * This method uses a character-to-token ratio (default: 6.5 characters per token) for quick estimation.
+     * If the estimated token count exceeds the chunk size, it performs an actual token count.
+     *
+     * @param {string} text - The text to estimate token count for
+     * @returns {Promise<number>} A promise that resolves to the estimated number of tokens
+     * @private
+     */
+    private _estimateTokenCount;
+    /**
+     * Split the text into chunks based on the provided recursive level rules.
+     *
+     * This method handles three different splitting strategies:
+     * 1. Whitespace-based splitting: Splits text on spaces
+     * 2. Delimiter-based splitting: Splits text on specified delimiters with options to include delimiters
+     * 3. Token-based splitting: Splits text into chunks of maximum token size
+     *
+     * @param {string} text - The text to be split into chunks
+     * @param {RecursiveLevel} recursiveLevel - The rules defining how to split the text
+     * @returns {Promise<string[]>} A promise that resolves to an array of text chunks
+     * @private
+     */
+    private _splitText;
+    /**
+     * Create a RecursiveChunk object with indices based on the current offset.
+     *
+     * This method constructs a RecursiveChunk object that contains metadata about the chunk,
+     * including the text content, its start and end indices, token count, and the level of recursion.
+     *
+     * @param {string} text - The text content of the chunk
+     * @param {number} tokenCount - The number of tokens in the chunk
+     */
+    private _makeChunks;
+    /**
+     * Merge short splits.
+     */
+    private _mergeSplits;
+    /**
+     * Binary search to find the leftmost position where value should be inserted to maintain order.
+     *
+     * @param {number[]} arr - The array to search
+     * @param {number} value - The value to insert
+     * @param {number} [lo=0] - The starting index for the search
+     * @returns {number} The index where the value should be inserted
+     * @private
+     */
+    private _bisectLeft;
+    /**
+     * Recursive helper for core chunking.
+     */
+    private _recursiveChunk;
+    /**
+     * Recursively chunk text.
+     *
+     * This method is the main entry point for chunking text using the RecursiveChunker.
+     * It takes a single text string and returns an array of RecursiveChunk objects.
+     *
+     * @param {string} text - The text to be chunked
+     * @returns {Promise<RecursiveChunk[]>} A promise that resolves to an array of RecursiveChunk objects
+     */
+    chunk(text: string): Promise<RecursiveChunk[]>;
+    /**
+     * Return a string representation of the RecursiveChunker.
+     *
+     * This method provides a string representation of the RecursiveChunker instance,
+     * including its tokenizer, rules, chunk size, minimum characters per chunk, and return type.
+     *
+     * @returns {string} A string representation of the RecursiveChunker
+     */
+    toString(): string;
+}
+
+/**
+ * Represents the essential data for a sentence within a text.
+ *
+ * @property text - The actual sentence string as it appears in the source text.
+ * @property startIndex - The zero-based index indicating where the sentence starts in the original text.
+ * @property endIndex - The zero-based index indicating where the sentence ends in the original text (inclusive).
+ * @property tokenCount - The number of tokens (words or subwords) in the sentence, useful for NLP tasks.
+ */
+interface SentenceData {
+    text: string;
+    startIndex: number;
+    endIndex: number;
+    tokenCount: number;
+}
+/**
+ * Class to represent a sentence.
+ *
+ * Represents a single sentence within a text, including its text, position, and token count.
+ *
+ * @class
+ * @param {SentenceData} data - The data required to construct a Sentence instance.
+ * @property {string} text - The text of the sentence.
+ * @property {number} startIndex - The starting index of the sentence in the original text.
+ * @property {number} endIndex - The ending index of the sentence in the original text.
+ * @property {number} tokenCount - The number of tokens in the sentence.
+ * @property {number[]} [embedding] - The embedding vector for the sentence (array of numbers, or null if not present).
+ *
+ * @method toString Returns a string representation of the Sentence.
+ * @returns {string}
+ *
+ * @method toDict Returns the Sentence as a dictionary-like object.
+ * @returns {SentenceData}
+ *
+ * @method static fromDict Creates a Sentence object from a dictionary-like object.
+ * @param {SentenceData} data - The data to create the Sentence from.
+ * @returns {Sentence}
+ */
+declare class Sentence {
+    /** The text of the sentence */
+    text: string;
+    /** The starting index of the sentence in the original text */
+    startIndex: number;
+    /** The ending index of the sentence in the original text */
+    endIndex: number;
+    /** The number of tokens in the sentence */
+    tokenCount: number;
+    constructor(data: SentenceData);
+    /** Return a string representation of the Sentence */
+    toString(): string;
+    /** Return the Sentence as a dictionary-like object */
+    toDict(): SentenceData;
+    /** Create a Sentence object from a dictionary-like object */
+    static fromDict(data: SentenceData): Sentence;
+}
+/**
+ * Represents the essential data for a chunk of sentences within a text.
+ *
+ * @property text - The combined text of all sentences in the chunk as it appears in the source text.
+ * @property startIndex - The zero-based index indicating where the chunk starts in the original text.
+ * @property endIndex - The zero-based index indicating where the chunk ends in the original text (inclusive).
+ * @property tokenCount - The total number of tokens (words or subwords) in the chunk, useful for NLP tasks.
+ * @property sentences - An array of SentenceData objects, each representing an individual sentence within the chunk.
+ */
+interface SentenceChunkData {
+    text: string;
+    startIndex: number;
+    endIndex: number;
+    tokenCount: number;
+    sentences: SentenceData[];
+    embedding?: number[];
+}
+/**
+ * Represents a chunk of one or more sentences within a text.
+ *
+ * A SentenceChunk groups together multiple {@link Sentence} objects, providing their combined text, position, and token count within the original text.
+ *
+ * @class
+ * @extends Chunk
+ *
+ * @param {Object} data - Data to construct a SentenceChunk instance.
+ * @param {string} data.text - Combined text of all sentences in the chunk.
+ * @param {number} data.startIndex - Zero-based index where the chunk starts in the original text.
+ * @param {number} data.endIndex - Zero-based index where the chunk ends in the original text (inclusive).
+ * @param {number} data.tokenCount - Total number of tokens in the chunk.
+ * @param {Sentence[]} data.sentences - Array of {@link Sentence} objects in the chunk.
+ *
+ * @property {string} text - Combined text of all sentences in the chunk.
+ * @property {number} startIndex - Starting index of the chunk in the original text.
+ * @property {number} endIndex - Ending index of the chunk in the original text.
+ * @property {number} tokenCount - Total number of tokens in the chunk.
+ * @property {Sentence[]} sentences - List of {@link Sentence} objects in the chunk.
+ *
+ * @method toString Returns a detailed string representation of the SentenceChunk, including its text, start and end indices, token count, and a list of all contained sentences with their metadata.
+ * @method toDict Returns the SentenceChunk as a plain object (see {@link SentenceChunkData}).
+ * @method static fromDict Creates a SentenceChunk from a {@link SentenceChunkData} object.
+ */
+declare class SentenceChunk extends Chunk {
+    /** List of sentences in the chunk */
+    sentences: Sentence[];
+    constructor(data: {
+        text: string;
+        startIndex: number;
+        endIndex: number;
+        tokenCount: number;
+        sentences: Sentence[];
+        embedding?: number[];
+    });
+    /**
+     * Returns a detailed string representation of the SentenceChunk, including its text, start and end indices, token count, and a list of all contained sentences with their metadata.
+     *
+     * This method overrides the base {@link Chunk} toString method to provide a more informative output, which is especially useful for debugging and logging. Each sentence in the chunk is represented using its own toString method, and all sentences are included in the output.
+     *
+     * @returns {string} A string describing the SentenceChunk and all its sentences, e.g.,
+     *   SentenceChunk(text=..., startIndex=..., endIndex=..., tokenCount=..., sentences=[Sentence(...), ...])
+     */
+    toString(): string;
+    /**
+     * Returns the SentenceChunk as a dictionary-like object.
+     *
+     * This method extends the base {@link Chunk} toDict method to include the sentences in the chunk.
+     *
+     * @returns {SentenceChunkData} A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
+    /** Return the SentenceChunk as a dictionary-like object */
+    toDict(): SentenceChunkData;
+    /**
+     * Creates a SentenceChunk object from a dictionary-like object.
+     *
+     * This method extends the base {@link Chunk} fromDict method to include the sentences in the chunk.
+     *
+     * @param {SentenceChunkData} data - A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
+     * @returns {SentenceChunk} A new SentenceChunk object created from the provided dictionary-like object.
+     */
+    static fromDict(data: SentenceChunkData): SentenceChunk;
+}
+
+/** Module containing SentenceChunker class. */
+
+/**
+ * Options for creating a SentenceChunker instance.
+ *
+ * @property {string | Tokenizer} [tokenizer] - The tokenizer to use for token counting. Can be a string (model name) or a Tokenizer instance. Default: 'Xenova/gpt2'.
+ * @property {number} [chunkSize] - Maximum number of tokens per chunk. Must be > 0. Default: 512.
+ * @property {number} [chunkOverlap] - Number of tokens to overlap between consecutive chunks. Must be >= 0 and < chunkSize. Default: 0.
+ * @property {number} [minSentencesPerChunk] - Minimum number of sentences per chunk. Must be > 0. Default: 1.
+ * @property {number} [minCharactersPerSentence] - Minimum number of characters for a valid sentence. Sentences shorter than this are merged. Must be > 0. Default: 12.
+ * @property {boolean} [approximate] - (Deprecated) Whether to use approximate token counting. Default: false. Will be removed in future versions.
+ * @property {string[]} [delim] - List of sentence delimiters to use for splitting. Default: ['. ', '! ', '? ', '\n'].
+ * @property {('prev' | 'next' | null)} [includeDelim] - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null). Default: 'prev'.
+ */
+interface SentenceChunkerOptions {
+    tokenizer?: TokenizerModelName;
+    chunkSize?: number;
+    chunkOverlap?: number;
+    minSentencesPerChunk?: number;
+    minCharactersPerSentence?: number;
+    approximate?: boolean;
+    delim?: string[];
+    includeDelim?: "prev" | "next" | null;
+}
+/**
+ * Represents a SentenceChunker instance that is also directly callable.
+ * This type combines the SentenceChunker class with a function interface,
+ * allowing the instance to be called directly like a function.
+ *
+ * When called, it executes the `call` method inherited from BaseChunker,
+ * which in turn calls either `chunk` (for single text) or `chunkBatch` (for multiple texts).
+ *
+ * @example
+ * const chunker = await SentenceChunker.create();
+ * // Single text processing
+ * const chunks = await chunker("This is a sample text.");
+ * // Batch processing
+ * const batchChunks = await chunker(["Text 1", "Text 2"]);
+ *
+ * @type {SentenceChunker & {
+ *   (text: string, showProgress?: boolean): Promise<SentenceChunk[]>;
+ *   (texts: string[], showProgress?: boolean): Promise<SentenceChunk[][]>;
+ * }}
+ */
+type CallableSentenceChunker = SentenceChunker & {
+    (text: string, showProgress?: boolean): Promise<SentenceChunk[]>;
+    (texts: string[], showProgress?: boolean): Promise<SentenceChunk[][]>;
+};
+/**
+ * SentenceChunker is a class that implements the BaseChunker interface.
+ * It uses a tokenizer to split text into sentences and then creates chunks of text.
+ *
+ * @extends BaseChunker
+ *
+ * @property {number} chunkSize - Maximum number of tokens per chunk.
+ * @property {number} chunkOverlap - Number of tokens to overlap between consecutive chunks.
+ * @property {number} minSentencesPerChunk - Minimum number of sentences per chunk.
+ * @property {number} minCharactersPerSentence - Minimum number of characters for a valid sentence.
+ * @property {boolean} approximate - Whether to use approximate token counting.
+ * @property {string[]} delim - List of sentence delimiters to use for splitting.
+ * @property {('prev' | 'next' | null)} includeDelim - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null).
+ *
+ * @method chunk - Chunk a single text string.
+ * @method chunkBatch - Chunk an array of text strings.
+ * @method call - (Inherited from BaseChunker) Chunk a single text string or an array of text strings.
+ * @method toString - Return a string representation of the SentenceChunker.
+ *
+ * @example
+ * const chunker = await SentenceChunker.create();
+ * const chunks = await chunker("This is a sample text.");
+ * const batchChunks = await chunker(["Text 1", "Text 2"]);
+ *
+ * @see BaseChunker
+ */
+declare class SentenceChunker extends BaseChunker {
+    readonly chunkSize: number;
+    readonly chunkOverlap: number;
+    readonly minSentencesPerChunk: number;
+    readonly minCharactersPerSentence: number;
+    readonly approximate: boolean;
+    readonly delim: string[];
+    readonly includeDelim: "prev" | "next" | null;
+    readonly sep: string;
+    /**
+     * Private constructor. Use `SentenceChunker.create()` to instantiate.
+     *
+     * @param {Tokenizer} tokenizer - The tokenizer to use for token counting.
+     * @param {number} chunkSize - Maximum number of tokens per chunk.
+     * @param {number} chunkOverlap - Number of tokens to overlap between consecutive chunks.
+     * @param {number} minSentencesPerChunk - Minimum number of sentences per chunk.
+     * @param {number} minCharactersPerSentence - Minimum number of characters for a valid sentence.
+     * @param {boolean} approximate - Whether to use approximate token counting.
+     * @param {string[]} delim - List of sentence delimiters to use for splitting.
+     * @param {('prev' | 'next' | null)} includeDelim - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null).
+     */
+    private constructor();
+    /**
+     * Creates and initializes a SentenceChunker instance that is directly callable.
+     *
+     * This method is a static factory function that returns a Promise resolving to a CallableSentenceChunker instance.
+     * The returned instance is a callable function that can be used to chunk text strings or arrays of text strings.
+     *
+     * @param {SentenceChunkerOptions} [options] - Options for configuring the SentenceChunker.
+     * @returns {Promise<CallableSentenceChunker>} A promise that resolves to a callable SentenceChunker instance.
+     *
+     * @example
+     * const chunker = await SentenceChunker.create();
+     * const chunks = await chunker("This is a sample text.");
+     * const batchChunks = await chunker(["Text 1", "Text 2"]);
+     *
+     * @see SentenceChunkerOptions
+     */
+    static create(options?: SentenceChunkerOptions): Promise<CallableSentenceChunker>;
+    /**
+     * Fast sentence splitting while maintaining accuracy.
+     *
+     * @param {string} text - The text to split into sentences.
+     * @returns {string[]} An array of sentences.
+     */
+    private _splitText;
+    /**
+     * Split text into sentences and calculate token counts for each sentence.
+     *
+     * @param {string} text - The text to split into sentences.
+     * @returns {Promise<Sentence[]>} An array of Sentence objects.
+     */
+    private _prepareSentences;
+    /**
+     * Create a chunk from a list of sentences.
+     *
+     * @param {Sentence[]} sentences - The sentences to create a chunk from.
+     * @returns {Promise<SentenceChunk>} A promise that resolves to a SentenceChunk object.
+     */
+    private _createChunk;
+    /**
+     * Split text into overlapping chunks based on sentences while respecting token limits.
+     *
+     * @param {string} text - The text to split into chunks.
+     * @returns {Promise<SentenceChunk[]>} A promise that resolves to an array of SentenceChunk objects.
+     */
+    chunk(text: string): Promise<SentenceChunk[]>;
+    /**
+     * Binary search to find the leftmost position where value should be inserted to maintain order.
+     *
+     * @param {number[]} arr - The array to search.
+     * @param {number} value - The value to search for.
+     * @param {number} [lo] - The starting index of the search.
+     * @returns {number} The index of the leftmost position where value should be inserted.
+     */
+    private _bisectLeft;
+    /**
+     * Return a string representation of the SentenceChunker.
+     *
+     * @returns {string} A string representation of the SentenceChunker.
+     */
+    toString(): string;
+}
+
 type JOB_STATUS = "completed" | "failed" | "delayed" | "active" | "waiting" | "paused" | "stuck";
 declare const JOB_STATUS_ENUM: {
     completed: string;
@@ -549,12 +1407,10 @@ declare const ExuluJobs: {
 };
 
 declare const ExuluChunkers: {
-    chonkie: {
-        sentence: typeof SentenceChunker;
-        recursive: {
-            function: typeof RecursiveChunker;
-            rules: typeof RecursiveRules;
-        };
+    sentence: typeof SentenceChunker;
+    recursive: {
+        function: typeof RecursiveChunker;
+        rules: typeof RecursiveRules;
     };
 };
 declare const ExuluDatabase: {