@exulu/backend 0.3.4 → 0.3.6

package/dist/index.js

@@ -1847,9 +1847,25 @@ var rateLimiter = async (key, windowSeconds, limit, points) => {
   }
 };
 
-// src/registry/route-validators/index.ts
-import "express";
-import { getToken } from "next-auth/jwt";
+// src/auth/get-token.ts
+import { jwtVerify, importJWK } from "jose";
+var getToken = async (authHeader) => {
+  const token = authHeader.split(" ")[1];
+  if (!token) {
+    throw new Error("No token provided");
+  }
+  if (!process.env.NEXTAUTH_SECRET) {
+    throw new Error("No NEXTAUTH_SECRET provided");
+  }
+  try {
+    const secret = process.env.NEXTAUTH_SECRET;
+    const jwk = await importJWK({ k: secret, alg: "HS256", kty: "oct" });
+    const { payload } = await jwtVerify(token, jwk);
+    return payload;
+  } catch (error) {
+    throw new Error("Invalid token");
+  }
+};
 
 // src/auth/auth.ts
 import bcrypt2 from "bcryptjs";
@@ -1987,7 +2003,8 @@ var requestValidators = {
     let authtoken = null;
     if (typeof apikey !== "string") {
       const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken({ req, secret });
+      authtoken = await getToken(req.headers["authorization"] ?? "");
+      console.log("[EXULU] authtoken", authtoken);
     }
     return await authentication({
       authtoken,
@@ -2512,7 +2529,6 @@ import { expressMiddleware } from "@as-integrations/express5";
 // src/registry/uppy.ts
 import "express";
 import bodyParser from "body-parser";
-import { getToken as getToken2 } from "next-auth/jwt";
 var createUppyRoutes = async (app) => {
   const {
     S3Client,
@@ -2575,11 +2591,11 @@ var createUppyRoutes = async (app) => {
   }
   app.use(bodyParser.urlencoded({ extended: true }), bodyParser.json());
   app.get("/s3/list", async (req, res, next) => {
+    req.accepts;
     const apikey = req.headers["exulu-api-key"] || null;
     let authtoken = null;
     if (typeof apikey !== "string") {
-      const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken2({ req, secret });
+      authtoken = await getToken(req.headers.authorization ?? "");
     }
     const { db: db2 } = await postgresClient();
     const authenticationResult = await authentication({
@@ -2629,8 +2645,7 @@ var createUppyRoutes = async (app) => {
     const { db: db2 } = await postgresClient();
     let authtoken = null;
     if (typeof apikey !== "string" && typeof internalkey !== "string") {
-      const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken2({ req, secret });
+      authtoken = await getToken(req.headers.authorization ?? "");
     }
     const authenticationResult = await authentication({
       authtoken,
@@ -2703,8 +2718,7 @@ var createUppyRoutes = async (app) => {
     const { db: db2 } = await postgresClient();
     let authtoken = null;
     if (typeof apikey !== "string") {
-      const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken2({ req, secret });
+      authtoken = await getToken(req.headers.authorization ?? "");
     }
     const authenticationResult = await authentication({
       authtoken,
@@ -2752,8 +2766,7 @@ var createUppyRoutes = async (app) => {
     const { db: db2 } = await postgresClient();
     let authtoken = null;
     if (typeof apikey !== "string") {
-      const secret = process.env.NEXTAUTH_SECRET;
-      authtoken = await getToken2({ req, secret });
+      authtoken = await getToken(req.headers.authorization ?? "");
     }
     const authenticationResult = await authentication({
       authtoken,
@@ -4188,8 +4201,1208 @@ var ExuluApp = class {
   };
 };
 
-// src/index.ts
-import { RecursiveChunker, SentenceChunker, RecursiveRules } from "chonkie";
+// src/chunking/types/base.ts
+var Chunk = class _Chunk {
+  /** The text of the chunk. */
+  text;
+  /** The starting index of the chunk in the original text. */
+  startIndex;
+  /** The ending index of the chunk in the original text. */
+  endIndex;
+  /** The number of tokens in the chunk. */
+  tokenCount;
+  /** Optional embedding for the chunk. */
+  embedding;
+  /**
+   * Constructs a new Chunk object.
+   * 
+   * @param {ChunkData} data - The data to construct the Chunk from.
+   */
+  constructor(data) {
+    this.text = data.text;
+    this.startIndex = data.startIndex;
+    this.endIndex = data.endIndex;
+    this.tokenCount = data.tokenCount;
+    this.embedding = data.embedding;
+    if (this.startIndex > this.endIndex) {
+      throw new Error("Start index must be less than or equal to end index.");
+    }
+    if (this.tokenCount < 0) {
+      throw new Error("Token count must be a non-negative integer.");
+    }
+  }
+  /** Return a string representation of the Chunk. 
+   * 
+   * @returns {string} The text of the chunk.
+   */
+  toString() {
+    return this.text;
+  }
+  /** Return a detailed string representation of the Chunk. 
+   * 
+   * @returns {string} The detailed string representation of the Chunk.
+   */
+  toRepresentation() {
+    let repr = `Chunk(text='${this.text}', tokenCount=${this.tokenCount}, startIndex=${this.startIndex}, endIndex=${this.endIndex}`;
+    repr += ")";
+    return repr;
+  }
+  /** Return a slice of the chunk's text. 
+   * 
+   * @param {number} [start] - The starting index of the slice.
+   * @param {number} [end] - The ending index of the slice.
+   * @returns {string} The slice of the chunk's text.
+   */
+  slice(start, end) {
+    return this.text.slice(start, end);
+  }
+  /** Return the Chunk as a dictionary-like object. 
+   * 
+   * @returns {ChunkData} The dictionary-like object.
+   */
+  toDict() {
+    return {
+      text: this.text,
+      startIndex: this.startIndex,
+      endIndex: this.endIndex,
+      tokenCount: this.tokenCount,
+      embedding: this.embedding
+    };
+  }
+  /** Create a Chunk object from a dictionary-like object. 
+   * 
+   * @param {ChunkData} data - The dictionary-like object.
+   * @returns {Chunk} The Chunk object.
+   */
+  static fromDict(data) {
+    return new _Chunk({
+      text: data.text,
+      startIndex: data.startIndex,
+      endIndex: data.endIndex,
+      tokenCount: data.tokenCount,
+      embedding: data.embedding
+    });
+  }
+  /** Return a deep copy of the chunk. 
+   * 
+   * @returns {Chunk} The deep copy of the chunk.
+   */
+  copy() {
+    return _Chunk.fromDict(this.toDict());
+  }
+};
+
+// src/chunking/types/recursive.ts
+var RecursiveLevel = class _RecursiveLevel {
+  /** Custom delimiters for chunking */
+  delimiters;
+  /** Whether to use whitespace as a delimiter */
+  whitespace;
+  /** Whether to include the delimiter in the previous or next chunk */
+  includeDelim;
+  /**
+   * Constructs a new RecursiveLevel object.
+   * 
+   * @param {RecursiveLevelData} data - The data to construct the RecursiveLevel from.
+   */
+  constructor(data = {}) {
+    this.delimiters = data.delimiters;
+    this.whitespace = data.whitespace ?? false;
+    this.includeDelim = data.includeDelim ?? "prev";
+    this.validate();
+  }
+  /**
+   * Validates the RecursiveLevel object.
+   * 
+   * @private
+   */
+  validate() {
+    if (this.delimiters !== void 0 && this.whitespace) {
+      throw new Error("Cannot use whitespace as a delimiter and also specify custom delimiters.");
+    }
+    if (this.delimiters !== void 0) {
+      if (typeof this.delimiters === "string" && this.delimiters.length === 0) {
+        throw new Error("Custom delimiters cannot be an empty string.");
+      }
+      if (Array.isArray(this.delimiters)) {
+        if (this.delimiters.some((delim) => typeof delim !== "string" || delim.length === 0)) {
+          throw new Error("Custom delimiters cannot be an empty string.");
+        }
+        if (this.delimiters.includes(" ")) {
+          throw new Error("Custom delimiters cannot be whitespace only. Set whitespace to true instead.");
+        }
+      }
+    }
+  }
+  /** Return a string representation of the RecursiveLevel
+   * 
+   * @returns {string} The string representation of the RecursiveLevel.
+   */
+  toString() {
+    return `RecursiveLevel(delimiters=${this.delimiters}, whitespace=${this.whitespace}, includeDelim=${this.includeDelim})`;
+  }
+  /** Return the RecursiveLevel as a dictionary-like object
+   * 
+   * @returns {RecursiveLevelData} The dictionary-like object.
+   */
+  toDict() {
+    return {
+      delimiters: this.delimiters,
+      whitespace: this.whitespace,
+      includeDelim: this.includeDelim
+    };
+  }
+  /** Create RecursiveLevel object from a dictionary
+   * 
+   * @param {RecursiveLevelData} data - The dictionary-like object.
+   * @returns {RecursiveLevel} The RecursiveLevel object.
+   */
+  static fromDict(data) {
+    return new _RecursiveLevel(data);
+  }
+  /** Create RecursiveLevel object from a recipe
+   * 
+   * @param {string} name - The name of the recipe.
+   * @param {string} lang - The language of the recipe.
+   * @returns {Promise<RecursiveLevel>} The RecursiveLevel object.
+   */
+  static async fromRecipe(name, lang = "en") {
+    throw new Error("Not implemented");
+  }
+};
+var RecursiveRules = class _RecursiveRules {
+  /** List of recursive levels */
+  levels;
+  constructor(data = {}) {
+    if (data.levels === void 0) {
+      const paragraphs = new RecursiveLevel({ delimiters: ["\n\n", "\r\n", "\n", "\r"] });
+      const sentences = new RecursiveLevel({ delimiters: [". ", "! ", "? "] });
+      const pauses = new RecursiveLevel({
+        delimiters: [
+          "{",
+          "}",
+          '"',
+          "[",
+          "]",
+          "<",
+          ">",
+          "(",
+          ")",
+          ":",
+          ";",
+          ",",
+          "\u2014",
+          "|",
+          "~",
+          "-",
+          "...",
+          "`",
+          "'"
+        ]
+      });
+      const word = new RecursiveLevel({ whitespace: true });
+      const token = new RecursiveLevel();
+      this.levels = [paragraphs, sentences, pauses, word, token];
+    } else {
+      this.levels = data.levels.map((level) => new RecursiveLevel(level));
+    }
+  }
+  /** Return a string representation of the RecursiveRules
+   * 
+   * @returns {string} The string representation of the RecursiveRules.
+   */
+  toString() {
+    return `RecursiveRules(levels=${this.levels})`;
+  }
+  /** Return the number of levels
+   * 
+   * @returns {number} The number of levels.
+   */
+  get length() {
+    return this.levels.length;
+  }
+  /** Get a level by index
+   * 
+   * @param {number} index - The index of the level.
+   * @returns {RecursiveLevel | undefined} The level.
+   */
+  getLevel(index) {
+    return this.levels[index];
+  }
+  /** Return an iterator over the levels
+   * 
+   * @returns {Iterator<RecursiveLevel>} The iterator over the levels.
+   */
+  [Symbol.iterator]() {
+    return this.levels[Symbol.iterator]();
+  }
+  /** Create a RecursiveRules object from a dictionary
+   * 
+   * @param {RecursiveRulesData} data - The dictionary-like object.
+   * @returns {RecursiveRules} The RecursiveRules object.
+   */
+  static fromDict(data) {
+    return new _RecursiveRules(data);
+  }
+  /** Return the RecursiveRules as a dictionary-like object
+   * 
+   * @returns {RecursiveRulesData} The dictionary-like object.
+   */
+  toDict() {
+    return {
+      levels: this.levels.map((level) => level.toDict())
+    };
+  }
+  /** Create a RecursiveRules object from a recipe
+   * 
+   * @param {string} name - The name of the recipe.
+   * @param {string} lang - The language of the recipe.
+   * @param {string} path - The path to the recipe.
+   * @returns {Promise<RecursiveRules>} The RecursiveRules object.
+   */
+  static async fromRecipe(name = "default", lang = "en", path3) {
+    throw new Error("Not implemented");
+  }
+};
+var RecursiveChunk = class _RecursiveChunk extends Chunk {
+  /** The level of recursion for the chunk */
+  level;
+  constructor(data) {
+    super(data);
+    this.level = data.level;
+  }
+  /** Return a string representation of the RecursiveChunk
+   * 
+   * @returns {string} The string representation of the RecursiveChunk.
+   */
+  toString() {
+    return `RecursiveChunk(text=${this.text}, startIndex=${this.startIndex}, endIndex=${this.endIndex}, tokenCount=${this.tokenCount}, level=${this.level})`;
+  }
+  /** Return the RecursiveChunk as a dictionary-like object
+   * 
+   * @returns {RecursiveChunkData} The dictionary-like object.
+   */
+  toDict() {
+    const baseDict = super.toDict();
+    return {
+      ...baseDict,
+      level: this.level
+    };
+  }
+  /** Create a RecursiveChunk object from a dictionary
+   * 
+   * @param {RecursiveChunkData} data - The dictionary-like object.
+   * @returns {RecursiveChunk} The RecursiveChunk object.
+   */
+  static fromDict(data) {
+    return new _RecursiveChunk(data);
+  }
+};
+
+// src/chunking/tokenizer.ts
+import { Tiktoken } from "tiktoken/lite";
+import { load } from "tiktoken/load";
+import registry from "tiktoken/registry.json";
+import models from "tiktoken/model_to_encoding.json";
+var ExuluTokenizer = class {
+  constructor() {
+  }
+  encoder = null;
+  async create(modelName) {
+    if (this.encoder) {
+      return this.encoder;
+    }
+    const time = performance.now();
+    console.log("[EXULU] Loading tokenizer.", modelName);
+    const model = await load(registry[models[modelName]]);
+    console.log("[EXULU] Loaded tokenizer.", modelName, performance.now() - time);
+    console.log("[EXULU] Model.", model.bpe_ranks);
+    console.log("[EXULU] Model.", model.special_tokens);
+    console.log("[EXULU] Model.", model.pat_str);
+    const encoder = new Tiktoken(
+      model.bpe_ranks,
+      model.special_tokens,
+      model.pat_str
+    );
+    console.log("[EXULU] Encoder.", encoder);
+    this.encoder = encoder;
+    return encoder;
+  }
+  async decode(tokens) {
+    if (!this.encoder) {
+      throw new Error("Tokenizer not initialized");
+    }
+    const text = this.encoder.decode(tokens);
+    return new TextDecoder().decode(text);
+  }
+  async decodeBatch(tokenSequences) {
+    if (!this.encoder) {
+      throw new Error("Tokenizer not initialized");
+    }
+    const promises2 = tokenSequences.map((tokens) => this.decode(tokens));
+    return await Promise.all(promises2);
+  }
+  encode(text) {
+    if (!this.encoder) {
+      throw new Error("Tokenizer not initialized");
+    }
+    const time = performance.now();
+    console.log("[EXULU] Encoding text.", text);
+    const tokens = this.encoder.encode(text);
+    console.log("[EXULU] Encoded text.", text, performance.now() - time);
+    return tokens;
+  }
+  async countTokensBatch(texts) {
+    if (!this.encoder) {
+      throw new Error("Tokenizer not initialized");
+    }
+    const promises2 = texts.map((text) => this.countTokens(text));
+    return await Promise.all(promises2);
+  }
+  countTokens(text) {
+    if (!this.encoder) {
+      throw new Error("Tokenizer not initialized");
+    }
+    console.log("[EXULU] Counting tokens.", text);
+    const tokens = this.encoder.encode(text);
+    const count = tokens.length;
+    console.log("[EXULU] Token count.", count);
+    return count;
+  }
+  async free() {
+    console.log("[EXULU] Freeing tokenizer.");
+    if (this.encoder) {
+      this.encoder.free();
+    }
+  }
+};
+
+// src/chunking/base.ts
+var BaseChunker = class {
+  tokenizer;
+  _useConcurrency = true;
+  // Determines if batch processing uses Promise.all
+  constructor(tokenizer) {
+    this.tokenizer = tokenizer;
+  }
+  /**
+   * Returns a string representation of the chunker instance.
+   *
+   * @returns {string} The class name and constructor signature.
+   */
+  toString() {
+    return `${this.constructor.name}()`;
+  }
+  async call(textOrTexts, showProgress = false) {
+    if (typeof textOrTexts === "string") {
+      return this.chunk(textOrTexts);
+    } else if (Array.isArray(textOrTexts)) {
+      return this.chunkBatch(textOrTexts, showProgress);
+    } else {
+      throw new Error("Input must be a string or an array of strings.");
+    }
+  }
+  /**
+   * Process a batch of texts sequentially (one after another).
+   *
+   * @protected
+   * @param {string[]} texts - The texts to chunk.
+   * @param {boolean} [showProgress=false] - Whether to display progress in the console.
+   * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+   */
+  async _sequential_batch_processing(texts, showProgress = false) {
+    const results = [];
+    const total = texts.length;
+    for (let i = 0; i < total; i++) {
+      if (showProgress && total > 1) {
+        const progress = Math.round((i + 1) / total * 100);
+        process.stdout.write(`Sequential processing: Document ${i + 1}/${total} (${progress}%)\r`);
+      }
+      results.push(await this.chunk(texts[i]));
+    }
+    if (showProgress && total > 1) {
+      process.stdout.write("\n");
+    }
+    return results;
+  }
+  /**
+   * Process a batch of texts concurrently using Promise.all.
+   *
+   * @protected
+   * @param {string[]} texts - The texts to chunk.
+   * @param {boolean} [showProgress=false] - Whether to display progress in the console.
+   * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+   */
+  async _concurrent_batch_processing(texts, showProgress = false) {
+    const total = texts.length;
+    let completedCount = 0;
+    const updateProgress = () => {
+      if (showProgress && total > 1) {
+        completedCount++;
+        const progress = Math.round(completedCount / total * 100);
+        process.stdout.write(`Concurrent processing: Document ${completedCount}/${total} (${progress}%)\r`);
+      }
+    };
+    const chunkPromises = texts.map(
+      (text) => this.chunk(text).then((result) => {
+        updateProgress();
+        return result;
+      })
+    );
+    const results = await Promise.all(chunkPromises);
+    if (showProgress && total > 1 && completedCount > 0) {
+      process.stdout.write("\n");
+    }
+    return results;
+  }
+  /**
+   * Chunk a batch of texts, using either concurrent or sequential processing.
+   *
+   * If only one text is provided, processes it directly without batch overhead.
+   *
+   * @param {string[]} texts - The texts to chunk.
+   * @param {boolean} [showProgress=true] - Whether to display progress in the console.
+   * @returns {Promise<Chunk[][]>} An array of chunked results for each input text.
+   */
+  async chunkBatch(texts, showProgress = true) {
+    if (texts.length === 0) {
+      return [];
+    }
+    if (texts.length === 1) {
+      return [await this.chunk(texts[0])];
+    }
+    if (this._useConcurrency) {
+      return this._concurrent_batch_processing(texts, showProgress);
+    } else {
+      return this._sequential_batch_processing(texts, showProgress);
+    }
+  }
+};
+
+// src/chunking/recursive.ts
+var RecursiveChunker = class _RecursiveChunker extends BaseChunker {
+  chunkSize;
+  minCharactersPerChunk;
+  rules;
+  sep;
+  _CHARS_PER_TOKEN = 6.5;
+  /**
+   * Private constructor. Use `RecursiveChunker.create()` to instantiate.
+   */
+  constructor(tokenizer, chunkSize, rules, minCharactersPerChunk) {
+    super(tokenizer);
+    if (chunkSize <= 0) {
+      throw new Error("chunkSize must be greater than 0");
+    }
+    if (minCharactersPerChunk <= 0) {
+      throw new Error("minCharactersPerChunk must be greater than 0");
+    }
+    if (!(rules instanceof RecursiveRules)) {
+      throw new Error("rules must be a RecursiveRules object");
+    }
+    this.chunkSize = chunkSize;
+    this.minCharactersPerChunk = minCharactersPerChunk;
+    this.rules = rules;
+    this.sep = "\u2704";
+  }
+  /**
+   * Creates and initializes a directly callable RecursiveChunker instance.
+   *
+   * This static factory method constructs a RecursiveChunker with the provided options and returns a callable function object.
+   * The returned instance can be used as both a function (to chunk text(s)) and as an object (with all RecursiveChunker methods and properties).
+   *
+   * @param {RecursiveChunkerOptions} [options] - Configuration options for the chunker. All options are optional:
+   *   @param {string|Tokenizer} [options.tokenizer="Xenova/gpt2"] - Tokenizer to use for text processing. Can be a string identifier (e.g., "Xenova/gpt2") or a Tokenizer instance. If a string is provided, Tokenizer.create() is called internally.
+   *   @param {number} [options.chunkSize=512] - Maximum number of tokens per chunk. Must be > 0.
+   *   @param {RecursiveRules} [options.rules=new RecursiveRules()] - Rules for recursive chunking. See {@link RecursiveRules} for customization.
+   *   @param {number} [options.minCharactersPerChunk=24] - Minimum number of characters per chunk. Must be > 0.
+   *
+   * @returns {Promise<CallableRecursiveChunker>} Promise resolving to a callable RecursiveChunker instance.
+   *
+   * @throws {Error} If any option is invalid (e.g., chunkSize <= 0).
+   *
+   * @see CallableRecursiveChunker for the callable interface and available properties/methods.
+   *
+   * @example <caption>Basic usage with default options</caption>
+   * const chunker = await RecursiveChunker.create();
+   * const chunks = await chunker("Some text to chunk");
+   *
+   * @example <caption>Custom options and batch chunking</caption>
+   * const chunker = await RecursiveChunker.create({ chunkSize: 256 });
+   * const batchChunks = await chunker(["Text 1", "Text 2"]);
+   *
+   * @example <caption>Accessing properties and methods</caption>
+   * const chunker = await RecursiveChunker.create();
+   * console.log(chunker.chunkSize); // 512
+   * console.log(chunker.rules); // RecursiveRules instance
+   * const chunks = await chunker.chunk("Some text"); // Use as object method
+   *
+   * @note
+   * The returned instance is both callable (like a function) and has all properties/methods of RecursiveChunker.
+   * You can use it as a drop-in replacement for a function or a class instance.
+   *
+   * @note
+   * For advanced customization, pass a custom RecursiveRules object to the rules option.
+   * See {@link RecursiveRules} and {@link RecursiveLevel} for rule structure.
+   */
+  static async create(options = {}) {
+    const {
+      tokenizer = "gpt-3.5-turbo",
+      chunkSize = 512,
+      rules = new RecursiveRules(),
+      minCharactersPerChunk = 24
+    } = options;
+    const tokenizerInstance = await new ExuluTokenizer();
+    await tokenizerInstance.create(tokenizer);
+    const plainInstance = new _RecursiveChunker(
+      tokenizerInstance,
+      chunkSize,
+      rules,
+      minCharactersPerChunk
+    );
+    const callableFn = function(textOrTexts, showProgress) {
+      if (typeof textOrTexts === "string") {
+        return plainInstance.call(textOrTexts, showProgress);
+      } else {
+        return plainInstance.call(textOrTexts, showProgress);
+      }
+    };
+    Object.setPrototypeOf(callableFn, _RecursiveChunker.prototype);
+    Object.assign(callableFn, plainInstance);
+    return callableFn;
+  }
+  /**
+   * Estimates the number of tokens in a given text.
+   * 
+   * This method uses a character-to-token ratio (default: 6.5 characters per token) for quick estimation.
+   * If the estimated token count exceeds the chunk size, it performs an actual token count.
+   * 
+   * @param {string} text - The text to estimate token count for
+   * @returns {Promise<number>} A promise that resolves to the estimated number of tokens
+   * @private
+   */
+  async _estimateTokenCount(text) {
+    const estimate = Math.max(1, Math.floor(text.length / this._CHARS_PER_TOKEN));
+    if (estimate > this.chunkSize) {
+      return this.chunkSize + 1;
+    }
+    return this.tokenizer.countTokens(text);
+  }
+  /**
+   * Split the text into chunks based on the provided recursive level rules.
+   * 
+   * This method handles three different splitting strategies:
+   * 1. Whitespace-based splitting: Splits text on spaces
+   * 2. Delimiter-based splitting: Splits text on specified delimiters with options to include delimiters
+   * 3. Token-based splitting: Splits text into chunks of maximum token size
+   * 
+   * @param {string} text - The text to be split into chunks
+   * @param {RecursiveLevel} recursiveLevel - The rules defining how to split the text
+   * @returns {Promise<string[]>} A promise that resolves to an array of text chunks
+   * @private
+   */
+  async _splitText(text, recursiveLevel) {
+    if (recursiveLevel.whitespace) {
+      return text.split(" ");
+    } else if (recursiveLevel.delimiters) {
+      let t = text;
+      if (recursiveLevel.includeDelim === "prev") {
+        for (const delimiter of Array.isArray(recursiveLevel.delimiters) ? recursiveLevel.delimiters : [recursiveLevel.delimiters]) {
+          t = t.replace(delimiter, delimiter + this.sep);
+        }
+      } else if (recursiveLevel.includeDelim === "next") {
+        for (const delimiter of Array.isArray(recursiveLevel.delimiters) ? recursiveLevel.delimiters : [recursiveLevel.delimiters]) {
+          t = t.replace(delimiter, this.sep + delimiter);
+        }
+      } else {
+        for (const delimiter of Array.isArray(recursiveLevel.delimiters) ? recursiveLevel.delimiters : [recursiveLevel.delimiters]) {
+          t = t.replace(delimiter, this.sep);
+        }
+      }
+      const splits = t.split(this.sep).filter((split) => split !== "");
+      let current = "";
+      const merged = [];
+      for (const split of splits) {
+        if (split.length < this.minCharactersPerChunk) {
+          current += split;
+        } else if (current) {
+          current += split;
+          merged.push(current);
+          current = "";
+        } else {
+          merged.push(split);
+        }
+        if (current.length >= this.minCharactersPerChunk) {
+          merged.push(current);
+          current = "";
+        }
+      }
+      if (current) {
+        merged.push(current);
+      }
+      return merged;
+    } else {
+      const encoded = await this.tokenizer.encode(text);
+      const tokenSplits = [];
+      for (let i = 0; i < encoded.length; i += this.chunkSize) {
+        tokenSplits.push(encoded.slice(i, i + this.chunkSize));
+      }
+      return await this.tokenizer.decodeBatch(tokenSplits);
+    }
+  }
+  /**
+   * Create a RecursiveChunk object with indices based on the current offset.
+   * 
+   * This method constructs a RecursiveChunk object that contains metadata about the chunk,
+   * including the text content, its start and end indices, token count, and the level of recursion.
+   * 
+   * @param {string} text - The text content of the chunk
+   * @param {number} tokenCount - The number of tokens in the chunk
+   */
+  _makeChunks(text, tokenCount, level, startOffset) {
+    return new RecursiveChunk({
+      text,
+      startIndex: startOffset,
+      endIndex: startOffset + text.length,
+      tokenCount,
+      level
+    });
+  }
+  /**
+   * Merge short splits.
+   */
+  _mergeSplits(splits, tokenCounts, combineWhitespace = false) {
+    if (!splits.length || !tokenCounts.length) {
+      return [[], []];
+    }
+    if (splits.length !== tokenCounts.length) {
+      throw new Error(
+        `Number of splits ${splits.length} does not match number of token counts ${tokenCounts.length}`
+      );
+    }
+    if (tokenCounts.every((count) => count > this.chunkSize)) {
+      return [splits, tokenCounts];
+    }
+    const merged = [];
+    const cumulativeTokenCounts = [];
+    let sum = 0;
+    if (combineWhitespace) {
+      cumulativeTokenCounts.push(0);
+      for (const count of tokenCounts) {
+        sum += count + 1;
+        cumulativeTokenCounts.push(sum);
+      }
+    } else {
+      cumulativeTokenCounts.push(0);
+      for (const count of tokenCounts) {
+        sum += count;
+        cumulativeTokenCounts.push(sum);
+      }
+    }
+    let currentIndex = 0;
+    const combinedTokenCounts = [];
+    while (currentIndex < splits.length) {
+      const currentTokenCount = cumulativeTokenCounts[currentIndex] ?? 0;
+      const requiredTokenCount = currentTokenCount + this.chunkSize;
+      let index = this._bisectLeft(
+        cumulativeTokenCounts,
+        requiredTokenCount,
+        currentIndex
+      ) - 1;
+      index = Math.min(index, splits.length);
+      if (index === currentIndex) {
+        index += 1;
+      }
+      if (combineWhitespace) {
+        merged.push(splits.slice(currentIndex, index).join(" "));
+      } else {
+        merged.push(splits.slice(currentIndex, index).join(""));
+      }
+      combinedTokenCounts.push(
+        (cumulativeTokenCounts[Math.min(index, splits.length)] ?? 0) - currentTokenCount
+      );
+      currentIndex = index;
+    }
+    return [merged, combinedTokenCounts];
+  }
+  /**
+   * Binary search to find the leftmost position where value should be inserted to maintain order.
+   * 
+   * @param {number[]} arr - The array to search
+   * @param {number} value - The value to insert
+   * @param {number} [lo=0] - The starting index for the search
+   * @returns {number} The index where the value should be inserted
+   * @private
+   */
+  _bisectLeft(arr, value, lo = 0) {
+    let hi = arr.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (arr[mid] < value) {
+        lo = mid + 1;
+      } else {
+        hi = mid;
+      }
+    }
+    return lo;
+  }
+  /**
+   * Recursive helper for core chunking.
+   */
+  async _recursiveChunk(text, level = 0, startOffset = 0) {
+    if (!text) {
+      return [];
+    }
+    console.log("[EXULU] Rule.", this.rules.length);
+    console.log("[EXULU] Level.", level);
+    if (level >= this.rules.length) {
+      const tokenCount = await this._estimateTokenCount(text);
+      return [
+        this._makeChunks(
+          text,
+          tokenCount,
+          level,
+          startOffset
+        )
+      ];
+    }
+    const currRule = this.rules.getLevel(level);
+    if (!currRule) {
+      throw new Error(`No rule found at level ${level}`);
+    }
+    const splits = await this._splitText(text, currRule);
+    const tokenCounts = await Promise.all(splits.map((split) => this._estimateTokenCount(split)));
+    let merged;
+    let combinedTokenCounts;
+    if (currRule.delimiters === void 0 && !currRule.whitespace) {
+      [merged, combinedTokenCounts] = [splits, tokenCounts];
+    } else if (currRule.delimiters === void 0 && currRule.whitespace) {
+      [merged, combinedTokenCounts] = this._mergeSplits(
+        splits,
+        tokenCounts,
+        true
+      );
+      merged = merged.slice(0, 1).concat(
+        merged.slice(1).map((text2) => " " + text2)
+      );
+    } else {
+      [merged, combinedTokenCounts] = this._mergeSplits(
+        splits,
+        tokenCounts,
+        false
+      );
+    }
+    const chunks = [];
+    let currentOffset = startOffset;
+    for (let i = 0; i < merged.length; i++) {
+      const split = merged[i];
+      const tokenCount = combinedTokenCounts[i];
+      if (tokenCount && tokenCount > this.chunkSize) {
+        chunks.push(...await this._recursiveChunk(split ?? "", level + 1, currentOffset));
+      } else {
+        chunks.push(
+          this._makeChunks(split ?? "", tokenCount ?? 0, level, currentOffset)
+        );
+      }
+      currentOffset += split?.length ?? 0;
+    }
+    return chunks;
+  }
+  /**
+   * Recursively chunk text.
+   * 
+   * This method is the main entry point for chunking text using the RecursiveChunker.
+   * It takes a single text string and returns an array of RecursiveChunk objects.
+   * 
+   * @param {string} text - The text to be chunked
+   * @returns {Promise<RecursiveChunk[]>} A promise that resolves to an array of RecursiveChunk objects
+   */
+  async chunk(text) {
+    console.log("[EXULU] Chunking text.", text);
+    const result = await this._recursiveChunk(text, 0, 0);
+    await this.tokenizer.free();
+    return result;
+  }
+  /**
+   * Return a string representation of the RecursiveChunker.
+   * 
+   * This method provides a string representation of the RecursiveChunker instance,
+   * including its tokenizer, rules, chunk size, minimum characters per chunk, and return type.
+   * 
+   * @returns {string} A string representation of the RecursiveChunker
+   */
+  toString() {
+    return `RecursiveChunker(tokenizer=${this.tokenizer}, rules=${this.rules}, chunkSize=${this.chunkSize}, minCharactersPerChunk=${this.minCharactersPerChunk})`;
+  }
+};
+
+// src/chunking/types/sentence.ts
+var Sentence = class _Sentence {
+  /** The text of the sentence */
+  text;
+  /** The starting index of the sentence in the original text */
+  startIndex;
+  /** The ending index of the sentence in the original text */
+  endIndex;
+  /** The number of tokens in the sentence */
+  tokenCount;
+  constructor(data) {
+    this.text = data.text;
+    this.startIndex = data.startIndex;
+    this.endIndex = data.endIndex;
+    this.tokenCount = data.tokenCount;
+  }
+  /** Return a string representation of the Sentence */
+  toString() {
+    return `Sentence(text=${this.text}, startIndex=${this.startIndex}, endIndex=${this.endIndex}, tokenCount=${this.tokenCount})`;
+  }
+  /** Return the Sentence as a dictionary-like object */
+  toDict() {
+    return {
+      text: this.text,
+      startIndex: this.startIndex,
+      endIndex: this.endIndex,
+      tokenCount: this.tokenCount
+    };
+  }
+  /** Create a Sentence object from a dictionary-like object */
+  static fromDict(data) {
+    return new _Sentence(data);
+  }
+};
+var SentenceChunk = class _SentenceChunk extends Chunk {
+  /** List of sentences in the chunk */
+  sentences;
+  constructor(data) {
+    super(data);
+    this.sentences = data.sentences;
+    this.embedding = data.embedding ?? void 0;
+  }
+  /**
+   * Returns a detailed string representation of the SentenceChunk, including its text, start and end indices, token count, and a list of all contained sentences with their metadata.
+   *
+   * This method overrides the base {@link Chunk} toString method to provide a more informative output, which is especially useful for debugging and logging. Each sentence in the chunk is represented using its own toString method, and all sentences are included in the output.
+   *
+   * @returns {string} A string describing the SentenceChunk and all its sentences, e.g.,
+   *   SentenceChunk(text=..., startIndex=..., endIndex=..., tokenCount=..., sentences=[Sentence(...), ...])
+   */
+  toString() {
+    const sentencesStr = this.sentences.map((s) => s.toString()).join(", ");
+    return `SentenceChunk(text=${this.text}, startIndex=${this.startIndex}, endIndex=${this.endIndex}, tokenCount=${this.tokenCount}, sentences=[${sentencesStr}])`;
+  }
+  /**
+   * Returns the SentenceChunk as a dictionary-like object.
+   *
+   * This method extends the base {@link Chunk} toDict method to include the sentences in the chunk.
+   *
+   * @returns {SentenceChunkData} A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
+  /** Return the SentenceChunk as a dictionary-like object */
+  toDict() {
+    const baseDict = super.toDict();
+    return {
+      ...baseDict,
+      sentences: this.sentences.map((sentence) => sentence.toDict())
+    };
+  }
+  /**
+   * Creates a SentenceChunk object from a dictionary-like object.
+   *
+   * This method extends the base {@link Chunk} fromDict method to include the sentences in the chunk.
+   *
+   * @param {SentenceChunkData} data - A dictionary-like object containing the chunk's text, start and end indices, token count, and an array of sentence data.
+   * @returns {SentenceChunk} A new SentenceChunk object created from the provided dictionary-like object.
+   */
+  static fromDict(data) {
+    const sentences = data.sentences.map((sentence) => Sentence.fromDict(sentence));
+    return new _SentenceChunk({
+      text: data.text,
+      startIndex: data.startIndex,
+      endIndex: data.endIndex,
+      tokenCount: data.tokenCount,
+      sentences,
+      embedding: data.embedding ?? void 0
+    });
+  }
+};
+
+// src/chunking/sentence.ts
+var SentenceChunker = class _SentenceChunker extends BaseChunker {
+  chunkSize;
+  chunkOverlap;
+  minSentencesPerChunk;
+  minCharactersPerSentence;
+  approximate;
+  delim;
+  includeDelim;
+  sep;
+  /**
+   * Private constructor. Use `SentenceChunker.create()` to instantiate.
+   * 
+   * @param {Tokenizer} tokenizer - The tokenizer to use for token counting.
+   * @param {number} chunkSize - Maximum number of tokens per chunk.
+   * @param {number} chunkOverlap - Number of tokens to overlap between consecutive chunks.
+   * @param {number} minSentencesPerChunk - Minimum number of sentences per chunk.
+   * @param {number} minCharactersPerSentence - Minimum number of characters for a valid sentence.
+   * @param {boolean} approximate - Whether to use approximate token counting.
+   * @param {string[]} delim - List of sentence delimiters to use for splitting.
+   * @param {('prev' | 'next' | null)} includeDelim - Whether to include the delimiter with the previous sentence ('prev'), next sentence ('next'), or exclude it (null).
+   */
+  constructor(tokenizer, chunkSize, chunkOverlap, minSentencesPerChunk, minCharactersPerSentence, approximate, delim, includeDelim) {
+    super(tokenizer);
+    if (chunkSize <= 0) {
+      throw new Error("chunkSize must be greater than 0");
+    }
+    if (chunkOverlap < 0) {
+      throw new Error("chunkOverlap must be non-negative");
+    }
+    if (chunkOverlap >= chunkSize) {
+      throw new Error("chunkOverlap must be less than chunkSize");
+    }
+    if (minSentencesPerChunk <= 0) {
+      throw new Error("minSentencesPerChunk must be greater than 0");
+    }
+    if (minCharactersPerSentence <= 0) {
+      throw new Error("minCharactersPerSentence must be greater than 0");
+    }
+    if (!delim) {
+      throw new Error("delim must be a list of strings or a string");
+    }
+    if (includeDelim !== "prev" && includeDelim !== "next" && includeDelim !== null) {
+      throw new Error("includeDelim must be 'prev', 'next' or null");
+    }
+    if (approximate) {
+      console.warn("Approximate has been deprecated and will be removed from next version onwards!");
+    }
+    this.chunkSize = chunkSize;
+    this.chunkOverlap = chunkOverlap;
+    this.minSentencesPerChunk = minSentencesPerChunk;
+    this.minCharactersPerSentence = minCharactersPerSentence;
+    this.approximate = approximate;
+    this.delim = delim;
+    this.includeDelim = includeDelim;
+    this.sep = "\u2704";
+  }
+  /**
+   * Creates and initializes a SentenceChunker instance that is directly callable.
+   * 
+   * This method is a static factory function that returns a Promise resolving to a CallableSentenceChunker instance.
+   * The returned instance is a callable function that can be used to chunk text strings or arrays of text strings.
+   * 
+   * @param {SentenceChunkerOptions} [options] - Options for configuring the SentenceChunker.
+   * @returns {Promise<CallableSentenceChunker>} A promise that resolves to a callable SentenceChunker instance.
+   * 
+   * @example
+   * const chunker = await SentenceChunker.create();
+   * const chunks = await chunker("This is a sample text.");
+   * const batchChunks = await chunker(["Text 1", "Text 2"]);
+   * 
+   * @see SentenceChunkerOptions
+   */
+  static async create(options = {}) {
+    const {
+      tokenizer = "gpt-3.5-turbo",
+      chunkSize = 512,
+      chunkOverlap = 0,
+      minSentencesPerChunk = 1,
+      minCharactersPerSentence = 12,
+      approximate = false,
+      delim = [". ", "! ", "? ", "\n"],
+      includeDelim = "prev"
+    } = options;
+    const tokenizerInstance = await new ExuluTokenizer();
+    await tokenizerInstance.create(tokenizer);
+    const plainInstance = new _SentenceChunker(
+      tokenizerInstance,
+      chunkSize,
+      chunkOverlap,
+      minSentencesPerChunk,
+      minCharactersPerSentence,
+      approximate,
+      delim,
+      includeDelim
+    );
+    const callableFn = function(textOrTexts, showProgress) {
+      if (typeof textOrTexts === "string") {
+        return plainInstance.call(textOrTexts, showProgress);
+      } else {
+        return plainInstance.call(textOrTexts, showProgress);
+      }
+    };
+    Object.setPrototypeOf(callableFn, _SentenceChunker.prototype);
+    Object.assign(callableFn, plainInstance);
+    return callableFn;
+  }
+  // NOTE: The replace + split method is not the best/most efficient way in general to be doing this. It works well in python because python implements .replace and .split in C while the re library is much slower in python. 
+  // NOTE: The new split -> join -> split is so weird, but it works. I don't quite like it however.
+  // TODO: Implement a more efficient method for splitting text into sentences.
+  /**
+   * Fast sentence splitting while maintaining accuracy.
+   * 
+   * @param {string} text - The text to split into sentences.
+   * @returns {string[]} An array of sentences.
+   */
+  _splitText(text) {
+    let t = text;
+    for (const c of this.delim) {
+      if (this.includeDelim === "prev") {
+        t = t.split(c).join(c + this.sep);
+      } else if (this.includeDelim === "next") {
+        t = t.split(c).join(this.sep + c);
+      } else {
+        t = t.split(c).join(this.sep);
+      }
+    }
+    const splits = t.split(this.sep);
+    const sentences = [];
+    let current = "";
+    for (const s of splits) {
+      if (!current) {
+        current = s;
+      } else {
+        if (current.length >= this.minCharactersPerSentence) {
+          sentences.push(current);
+          current = s;
+        } else {
+          current += s;
+        }
+      }
+    }
+    if (current) {
+      sentences.push(current);
+    }
+    return sentences;
+  }
+  /**
+   * Split text into sentences and calculate token counts for each sentence.
+   * 
+   * @param {string} text - The text to split into sentences.
+   * @returns {Promise<Sentence[]>} An array of Sentence objects.
+   */
+  async _prepareSentences(text) {
+    const sentenceTexts = this._splitText(text);
+    if (!sentenceTexts.length) {
+      return [];
+    }
+    const positions = [];
+    let currentPos = 0;
+    for (const sent of sentenceTexts) {
+      positions.push(currentPos);
+      currentPos += sent.length;
+    }
+    const tokenCounts = await this.tokenizer.countTokensBatch(sentenceTexts);
+    return sentenceTexts.map((sent, i) => new Sentence({
+      text: sent,
+      startIndex: positions[i],
+      endIndex: positions[i] + sent.length,
+      tokenCount: tokenCounts[i]
+    }));
+  }
+  /**
+   * Create a chunk from a list of sentences.
+   * 
+   * @param {Sentence[]} sentences - The sentences to create a chunk from.
+   * @returns {Promise<SentenceChunk>} A promise that resolves to a SentenceChunk object.
+   */
+  async _createChunk(sentences) {
+    const chunkText = sentences.map((sentence) => sentence.text).join("");
+    const tokenCount = await this.tokenizer.countTokens(chunkText);
+    return new SentenceChunk({
+      text: chunkText,
+      startIndex: sentences[0].startIndex,
+      endIndex: sentences[sentences.length - 1].endIndex,
+      tokenCount,
+      sentences
+    });
+  }
+  /**
+   * Split text into overlapping chunks based on sentences while respecting token limits.
+   * 
+   * @param {string} text - The text to split into chunks.
+   * @returns {Promise<SentenceChunk[]>} A promise that resolves to an array of SentenceChunk objects.
+   */
+  async chunk(text) {
+    if (!text.trim()) {
+      return [];
+    }
+    const sentences = await this._prepareSentences(text);
+    if (!sentences.length) {
+      return [];
+    }
+    const tokenSums = [];
+    let sum = 0;
+    for (const sentence of sentences) {
+      tokenSums.push(sum);
+      sum += sentence.tokenCount;
+    }
+    tokenSums.push(sum);
+    const chunks = [];
+    let pos = 0;
+    while (pos < sentences.length) {
+      const targetTokens = tokenSums[pos] + this.chunkSize;
+      let splitIdx = this._bisectLeft(tokenSums, targetTokens, pos) - 1;
+      splitIdx = Math.min(splitIdx, sentences.length);
+      splitIdx = Math.max(splitIdx, pos + 1);
+      if (splitIdx - pos < this.minSentencesPerChunk) {
+        if (pos + this.minSentencesPerChunk <= sentences.length) {
+          splitIdx = pos + this.minSentencesPerChunk;
+        } else {
+          console.warn(
+            `Minimum sentences per chunk as ${this.minSentencesPerChunk} could not be met for all chunks. Last chunk of the text will have only ${sentences.length - pos} sentences. Consider increasing the chunk_size or decreasing the min_sentences_per_chunk.`
+          );
+          splitIdx = sentences.length;
+        }
+      }
+      const chunkSentences = sentences.slice(pos, splitIdx);
+      chunks.push(await this._createChunk(chunkSentences));
+      if (this.chunkOverlap > 0 && splitIdx < sentences.length) {
+        let overlapTokens = 0;
+        let overlapIdx = splitIdx - 1;
+        while (overlapIdx > pos && overlapTokens < this.chunkOverlap) {
+          const sent = sentences[overlapIdx];
+          const nextTokens = overlapTokens + sent.tokenCount + 1;
+          if (nextTokens > this.chunkOverlap) {
+            break;
+          }
+          overlapTokens = nextTokens;
+          overlapIdx--;
+        }
+        pos = overlapIdx + 1;
+      } else {
+        pos = splitIdx;
+      }
+    }
+    await this.tokenizer.free();
+    return chunks;
+  }
+  /**
+   * Binary search to find the leftmost position where value should be inserted to maintain order.
+   * 
+   * @param {number[]} arr - The array to search.
+   * @param {number} value - The value to search for.
+   * @param {number} [lo] - The starting index of the search.
+   * @returns {number} The index of the leftmost position where value should be inserted.
+   */
+  _bisectLeft(arr, value, lo = 0) {
+    let hi = arr.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (arr[mid] < value) {
+        lo = mid + 1;
+      } else {
+        hi = mid;
+      }
+    }
+    return lo;
+  }
+  /**
+   * Return a string representation of the SentenceChunker.
+   * 
+   * @returns {string} A string representation of the SentenceChunker.
+   */
+  toString() {
+    return `SentenceChunker(tokenizer=${this.tokenizer}, chunkSize=${this.chunkSize}, chunkOverlap=${this.chunkOverlap}, minSentencesPerChunk=${this.minSentencesPerChunk}, minCharactersPerSentence=${this.minCharactersPerSentence}, approximate=${this.approximate}, delim=${this.delim}, includeDelim=${this.includeDelim})`;
+  }
+};
 
 // src/cli/index.tsx
 import { useState as useState2 } from "react";
@@ -4389,12 +5602,10 @@ var ExuluJobs = {
   }
 };
 var ExuluChunkers = {
-  chonkie: {
-    sentence: SentenceChunker,
-    recursive: {
-      function: RecursiveChunker,
-      rules: RecursiveRules
-    }
+  sentence: SentenceChunker,
+  recursive: {
+    function: RecursiveChunker,
+    rules: RecursiveRules
   }
 };
 var ExuluDatabase = {