@exulu/backend 1.46.1 → 1.48.1

package/dist/index.d.cts

@@ -1,145 +1,15 @@
 import * as _opentelemetry_sdk_node from '@opentelemetry/sdk-node';
+import * as knex from 'knex';
+import { LanguageModel, UIMessage, Tool, streamText } from 'ai';
+import { z } from 'zod';
 import * as bullmq from 'bullmq';
 import { Queue } from 'bullmq';
-import { RedisClientType } from 'redis';
-import { z } from 'zod';
-import { UIMessage, LanguageModel, Tool, streamText } from 'ai';
 import { Express, Request } from 'express';
 import { transport } from 'winston';
-import { Knex } from 'knex';
+import { RedisClientType } from 'redis';
 import { Tiktoken } from 'tiktoken/lite';
 import models from 'tiktoken/model_to_encoding.json';
 
-declare function redisClient(): Promise<{
-    client: RedisClientType | null;
-}>;
-
-type STATISTICS_TYPE = "CONTEXT_RETRIEVE" | "SOURCE_UPDATE" | "EMBEDDER_UPSERT" | "EMBEDDER_GENERATE" | "EMBEDDER_DELETE" | "WORKFLOW_RUN" | "CONTEXT_UPSERT" | "TOOL_CALL" | "AGENT_RUN";
-declare const STATISTICS_TYPE_ENUM: {
-    CONTEXT_RETRIEVE: string;
-    SOURCE_UPDATE: string;
-    EMBEDDER_UPSERT: string;
-    EMBEDDER_GENERATE: string;
-    EMBEDDER_DELETE: string;
-    WORKFLOW_RUN: string;
-    CONTEXT_UPSERT: string;
-    TOOL_CALL: string;
-    AGENT_RUN: string;
-};
-
-type ExuluFieldTypes = "text" | "longText" | "shortText" | "number" | "boolean" | "code" | "json" | "enum" | "markdown" | "file" | "date" | "uuid";
-
-interface Item {
-    id?: string;
-    name?: string;
-    description?: string;
-    createdAt?: string;
-    updatedAt?: string;
-    external_id?: string;
-    source?: string;
-    tags?: string[];
-    textlength?: number;
-    last_processed_at?: string;
-    chunks?: {
-        id: string;
-        index: number;
-        content: string;
-        source: string;
-        createdAt: string;
-        updatedAt: string;
-    }[];
-    [key: string]: any;
-}
-
-type ExuluConfig = {
-    telemetry?: {
-        enabled: boolean;
-    };
-    logger?: {
-        winston: {
-            transports: transport[];
-        };
-    };
-    workers: {
-        enabled: boolean;
-        logger?: {
-            winston: {
-                transports: transport[];
-            };
-        };
-        telemetry?: {
-            enabled: boolean;
-        };
-    };
-    MCP: {
-        enabled: boolean;
-    };
-    fileUploads?: {
-        s3region: string;
-        s3key: string;
-        s3secret: string;
-        s3Bucket: string;
-        s3endpoint?: string;
-        s3prefix?: string;
-    };
-    privacy?: {
-        systemPromptPersonalization?: boolean;
-    };
-};
-declare class ExuluApp {
-    private _agents;
-    private _config?;
-    private _evals;
-    private _queues;
-    private _rerankers;
-    private _contexts?;
-    private _tools;
-    private _expressApp;
-    constructor();
-    create: ({ contexts, agents, config, tools, evals, rerankers, }: {
-        contexts?: Record<string, ExuluContext>;
-        config: ExuluConfig;
-        agents?: ExuluAgent[];
-        rerankers?: ExuluReranker[];
-        evals?: ExuluEval[];
-        tools?: ExuluTool[];
-    }) => Promise<ExuluApp>;
-    express: {
-        init: () => Promise<Express>;
-    };
-    get expressApp(): Express;
-    tool(id: string): ExuluTool | undefined;
-    tools(): ExuluTool[];
-    context(id: string): ExuluContext | undefined;
-    agent(id: string): ExuluAgent | undefined;
-    get contexts(): ExuluContext[];
-    get agents(): ExuluAgent[];
-    embeddings: {
-        generate: {
-            one: ({ context: contextId, item: itemId, }: {
-                context: string;
-                item: string;
-            }) => Promise<{
-                id: string;
-                job?: string;
-                chunks?: number;
-            }>;
-            all: ({ context: contextId }: {
-                context: string;
-            }) => Promise<{
-                jobs: string[];
-                items: number;
-            }>;
-        };
-    };
-    bullmq: {
-        workers: {
-            create: (queues?: string[] | undefined) => Promise<bullmq.Worker<any, any, string>[]>;
-        };
-    };
-    private server;
-}
-
 type User = {
     id: number;
     firstname?: string;
@@ -161,125 +31,41 @@ type User = {
     };
 };
 
-declare const VectorMethodEnum: {
-    readonly cosineDistance: "cosineDistance";
-    readonly hybridSearch: "hybridSearch";
-    readonly tsvector: "tsvector";
-};
-type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
-
-/**
- * Base operator type with comparison operations
- */
-type BaseOperator<T> = {
-    /** Equals */
-    eq?: T;
-    /** Not equals */
-    ne?: T;
-    /** In array */
-    in?: T[];
-    /** AND conditions */
-    and?: BaseOperator<T>[];
-    /** OR conditions */
-    or?: BaseOperator<T>[];
-};
-/**
- * String field filter operators
- */
-type StringOperator = BaseOperator<string> & {
-    /** Contains (case-insensitive substring match) */
-    contains?: string;
-};
-/**
- * Number field filter operators
- */
-type NumberOperator = BaseOperator<number> & {
-    /** Less than or equal to */
-    lte?: number;
-    /** Greater than or equal to */
-    gte?: number;
-};
-/**
- * Date field filter operators
- */
-type DateOperator = BaseOperator<Date | string> & {
-    /** Less than or equal to */
-    lte?: Date | string;
-    /** Greater than or equal to */
-    gte?: Date | string;
-};
-/**
- * Boolean field filter operators
- */
-type BooleanOperator = BaseOperator<boolean>;
-/**
- * JSON field filter operators
- */
-type JsonOperator = BaseOperator<any> & {
-    /** Contains (PostgreSQL @> operator for JSON containment) */
-    contains?: any;
-};
-/**
- * Filter operator type based on field type
- */
-type FilterOperator = BaseOperator<any> | StringOperator | NumberOperator | DateOperator | BooleanOperator | JsonOperator;
-/**
- * Single filter object - a record of field names to their filter operators
- */
-type Filter = Record<string, FilterOperator>;
-/**
- * Type for the filters parameter used throughout the codebase
- * Filters is an array of filter objects, where each object contains field names mapped to their operators
- *
- * @example
- * ```typescript
- * const filters: Filters = [
- *   { name: { contains: "test" } },
- *   { age: { gte: 18, lte: 65 } },
- *   { status: { in: ["active", "pending"] } }
- * ];
- * ```
- */
-type SearchFilters = Filter[];
-type VectorSearchChunkResult = {
-    chunk_content: string;
-    chunk_index: number;
-    chunk_id: string;
-    chunk_source: string;
-    chunk_metadata: Record<string, string>;
-    chunk_created_at: string;
-    chunk_updated_at: string;
-    item_id: string;
-    item_external_id: string;
-    item_name: string;
-    item_updated_at: string;
-    item_created_at: string;
-    chunk_cosine_distance?: number;
-    chunk_fts_rank?: number;
-    chunk_hybrid_score?: number;
-    context?: {
+type ExuluProviderConfig = {
+    name: string;
+    instructions: string;
+    model: {
+        create: ({ apiKey }: {
+            apiKey?: string | undefined;
+        }) => LanguageModel;
+    };
+    custom?: {
         name: string;
-        id: string;
+        description: string;
+    }[];
+    memory?: {
+        lastMessages: number;
+        vector: boolean;
+        semanticRecall: {
+            topK: number;
+            messageRange: number;
+        };
     };
 };
 
-interface Agent {
+interface ExuluAgent {
     id: string;
     modelName?: string;
     providerName?: string;
-    backend: string;
+    provider: string;
+    source: "code" | "database";
     memory?: string;
     welcomemessage?: string;
     type: "agent";
     name: string;
     image?: string;
     providerapikey?: string;
-    workflows?: {
-        enabled: boolean;
-        queue?: {
-            name: string;
-        };
-    };
+    workflows?: ExuluProviderWorkflowConfig;
     firewall?: {
         enabled: boolean;
         scanners?: {
@@ -293,6 +79,7 @@ interface Agent {
     active?: boolean;
     description?: string;
     instructions?: string;
+    feedback?: boolean;
     slug?: string;
     tools?: {
         id: string;
@@ -316,7 +103,7 @@ interface Agent {
         video: videoTypes$1[];
     };
     rights_mode?: 'private' | 'users' | 'roles' | 'public';
-    created_by?: string;
+    created_by?: string | number;
     RBAC?: {
         type?: string;
         users?: Array<{
@@ -332,23 +119,10 @@ interface Agent {
     updatedAt?: string;
 }
 type imageTypes$1 = '.png' | '.jpg' | '.jpeg' | '.gif' | '.webp';
-type fileTypes$1 = '.pdf' | '.docx' | '.xlsx' | '.xls' | '.csv' | '.pptx' | '.ppt';
+type fileTypes$1 = '.pdf' | '.docx' | '.xlsx' | '.xls' | '.csv' | '.pptx' | '.ppt' | '.txt' | '.md' | '.json';
 type audioTypes$1 = '.mp3' | '.wav' | '.m4a' | '.mp4' | '.mpeg';
 type videoTypes$1 = '.mp4' | '.m4a' | '.mp3' | '.mpeg' | '.wav';
 
-interface TestCase {
-    id: string;
-    name: string;
-    description?: string;
-    inputs: UIMessage[];
-    expected_output: string;
-    expected_tools?: string[];
-    expected_knowledge_sources?: string[];
-    expected_agent_tools?: string[];
-    createdAt: string;
-    updatedAt: string;
-}
-
 interface RateLimiterRule {
     name?: string;
     rate_limit: {
@@ -356,84 +130,42 @@ interface RateLimiterRule {
         limit: number;
     };
 }
-type ExuluAgentConfig = {
-    name: string;
-    instructions: string;
-    model: {
-        create: ({ apiKey }: {
-            apiKey?: string | undefined;
-        }) => LanguageModel;
-    };
-    outputSchema?: z.ZodType;
-    custom?: {
-        name: string;
-        description: string;
-    }[];
-    memory?: {
-        lastMessages: number;
-        vector: boolean;
-        semanticRecall: {
-            topK: number;
-            messageRange: number;
-        };
-    };
-};
-type imageTypes = '.png' | '.jpg' | '.jpeg' | '.gif' | '.webp';
-type fileTypes = '.pdf' | '.docx' | '.doc' | '.xlsx' | '.xls' | '.csv' | '.pptx' | '.ppt' | '.txt' | '.md' | '.json' | '.srt' | '.html';
-type audioTypes = '.mp3' | '.wav' | '.m4a' | '.mp4' | '.mpeg';
-type videoTypes = '.mp4' | '.m4a' | '.mp3' | '.mpeg' | '.wav';
-type allFileTypes = imageTypes | fileTypes | audioTypes | videoTypes;
-interface ExuluAgentParams {
-    id: string;
-    name: string;
-    type: "agent";
-    description: string;
-    config?: ExuluAgentConfig | undefined;
-    queue?: ExuluQueueConfig;
-    maxContextLength?: number;
-    authenticationInformation?: string;
-    provider: string;
-    workflows?: ExuluAgentWorkflowConfig;
-    capabilities?: {
-        text: boolean;
-        images: imageTypes[];
-        files: fileTypes[];
-        audio: audioTypes[];
-        video: videoTypes[];
-    };
-    outputSchema?: z.ZodType;
-    rateLimit?: RateLimiterRule;
-}
-interface ExuluAgentToolConfig {
-    id: string;
-    type: string;
-    config: {
-        name: string;
-        variable: string | boolean | number;
-        type: "boolean" | "string" | "number" | "variable";
-        value?: any;
-        default?: string | boolean | number;
+
+interface Item {
+    id?: string;
+    name?: string;
+    description?: string;
+    createdAt?: string;
+    updatedAt?: string;
+    external_id?: string;
+    source?: string;
+    tags?: string[];
+    textlength?: number;
+    last_processed_at?: string;
+    chunks?: {
+        id: string;
+        index: number;
+        content: string;
+        source: string;
+        createdAt: string;
+        updatedAt: string;
     }[];
+    [key: string]: any;
 }
-declare class ExuluReranker {
+
+interface TestCase {
     id: string;
     name: string;
-    description: string;
-    execute: (params: {
-        query: string;
-        chunks: VectorSearchChunkResult[];
-    }) => Promise<VectorSearchChunkResult[]>;
-    constructor({ id, name, description, execute }: {
-        id: string;
-        name: string;
-        description: string;
-        execute: (params: {
-            query: string;
-            chunks: VectorSearchChunkResult[];
-        }) => Promise<VectorSearchChunkResult[]>;
-    });
-    run(query: string, chunks: VectorSearchChunkResult[]): Promise<VectorSearchChunkResult[]>;
+    description?: string;
+    inputs: UIMessage[];
+    expected_output: string;
+    expected_tools?: string[];
+    expected_knowledge_sources?: string[];
+    expected_agent_tools?: string[];
+    createdAt: string;
+    updatedAt: string;
 }
+
 type ExuluQueueConfig = {
     queue: Queue;
     ratelimit: number;
@@ -444,29 +176,19 @@ type ExuluQueueConfig = {
     };
     retries?: number;
     backoff?: {
-        type: 'exponential' | 'linear';
+        type: "exponential" | "linear";
         delay: number;
     };
 };
-type ExuluEvalTokenMetadata = {
-    totalTokens?: number;
-    reasoningTokens?: number;
-    inputTokens?: number;
-    outputTokens?: number;
-    cachedInputTokens?: number;
-};
-type ExuluEvalMetadata = {
-    tokens?: ExuluEvalTokenMetadata;
-    duration?: number;
-};
+
 interface ExuluEvalParams {
     id: string;
     name: string;
     description: string;
     llm: boolean;
     execute: (params: {
-        agent: Agent;
-        backend: ExuluAgent;
+        agent: ExuluAgent;
+        provider: ExuluProvider;
         messages: UIMessage[];
         testCase: TestCase;
         config?: Record<string, any>;
@@ -489,94 +211,167 @@ declare class ExuluEval {
     }[];
     queue?: Promise<ExuluQueueConfig>;
     constructor({ id, name, description, execute, config, queue, llm }: ExuluEvalParams);
-    run(agent: Agent, backend: ExuluAgent, testCase: TestCase, messages: UIMessage[], config?: Record<string, any>): Promise<number>;
+    run(agent: ExuluAgent, provider: ExuluProvider, testCase: TestCase, messages: UIMessage[], config?: Record<string, any>): Promise<number>;
 }
-type ExuluAgentWorkflowConfig = {
-    enabled: boolean;
-    queue?: Promise<ExuluQueueConfig>;
+
+/**
+ * Base operator type with comparison operations
+ */
+type BaseOperator<T> = {
+    /** Equals */
+    eq?: T;
+    /** Not equals */
+    ne?: T;
+    /** In array */
+    in?: T[];
+    /** AND conditions */
+    and?: BaseOperator<T>[];
+    /** OR conditions */
+    or?: BaseOperator<T>[];
+    /** Contains (case-insensitive substring match) */
+    contains?: string;
 };
-declare class ExuluAgent {
-    id: string;
+/**
+ * String field filter operators
+ */
+type StringOperator = BaseOperator<string> & {
+    /** Contains (case-insensitive substring match) */
+    contains?: string;
+};
+/**
+ * Number field filter operators
+ */
+type NumberOperator = BaseOperator<number> & {
+    /** Less than or equal to */
+    lte?: number;
+    /** Greater than or equal to */
+    gte?: number;
+};
+/**
+ * Date field filter operators
+ */
+type DateOperator = BaseOperator<Date | string> & {
+    /** Less than or equal to */
+    lte?: Date | string;
+    /** Greater than or equal to */
+    gte?: Date | string;
+};
+/**
+ * Boolean field filter operators
+ */
+type BooleanOperator = BaseOperator<boolean>;
+/**
+ * JSON field filter operators
+ */
+type JsonOperator = BaseOperator<any> & {
+    /** Contains (PostgreSQL @> operator for JSON containment) */
+    contains?: any;
+};
+/**
+ * Filter operator type based on field type
+ */
+type FilterOperator = BaseOperator<any> | StringOperator | NumberOperator | DateOperator | BooleanOperator | JsonOperator;
+/**
+ * Single filter object - a record of field names to their filter operators
+ */
+type Filter = Record<string, FilterOperator>;
+/**
+ * Type for the filters parameter used throughout the codebase
+ * Filters is an array of filter objects, where each object contains field names mapped to their operators
+ *
+ * @example
+ * ```typescript
+ * const filters: Filters = [
+ *   { name: { contains: "test" } },
+ *   { age: { gte: 18, lte: 65 } },
+ *   { status: { in: ["active", "pending"] } }
+ * ];
+ * ```
+ */
+type SearchFilters = Filter[];
+
+type ExuluFieldTypes = "text" | "longText" | "shortText" | "number" | "boolean" | "code" | "json" | "enum" | "markdown" | "file" | "date" | "uuid";
+
+type allFileTypes = imageTypes | fileTypes | audioTypes | videoTypes;
+type audioTypes = ".mp3" | ".wav" | ".m4a" | ".mp4" | ".mpeg";
+type videoTypes = ".mp4" | ".m4a" | ".mp3" | ".mpeg" | ".wav";
+type imageTypes = ".png" | ".jpg" | ".jpeg" | ".gif" | ".webp";
+type fileTypes = ".pdf" | ".docx" | ".doc" | ".xlsx" | ".xls" | ".csv" | ".pptx" | ".ppt" | ".txt" | ".md" | ".json" | ".srt" | ".html";
+
+declare class ExuluStorage {
+    private config;
+    constructor({ config }: {
+        config: ExuluConfig;
+    });
+    getPresignedUrl: (key: string) => Promise<string>;
+    uploadFile: (file: Buffer | Uint8Array, fileName: string, type: string, user?: number, metadata?: Record<string, string>, customBucket?: string) => Promise<string>;
+}
+
+type ExuluContextProcessor = {
     name: string;
-    provider: string;
     description: string;
-    slug: string;
-    type: "agent";
-    streaming: boolean;
-    authenticationInformation?: string;
-    maxContextLength?: number;
-    workflows?: ExuluAgentWorkflowConfig;
-    queue?: ExuluQueueConfig;
-    rateLimit?: RateLimiterRule;
-    config?: ExuluAgentConfig | undefined;
-    model?: {
-        create: ({ apiKey }: {
-            apiKey?: string | undefined;
-        }) => LanguageModel;
-    };
-    capabilities: {
-        text: boolean;
-        images: string[];
-        files: string[];
-        audio: string[];
-        video: string[];
+    filter: ({ item, user, role, utils, exuluConfig, }: {
+        item: Item;
+        user?: number;
+        role?: string;
+        utils: {
+            storage: ExuluStorage;
+        };
+        exuluConfig: ExuluConfig;
+    }) => Promise<Item | undefined | null>;
+    execute: ({ item, user, role, utils, exuluConfig, }: {
+        item: Item;
+        user?: number;
+        role?: string;
+        utils: {
+            storage: ExuluStorage;
+        };
+        exuluConfig: ExuluConfig;
+    }) => Promise<Item>;
+    config?: {
+        queue?: Promise<ExuluQueueConfig>;
+        timeoutInSeconds?: number;
+        trigger: "manual" | "onUpdate" | "onInsert" | "always";
+        generateEmbeddings?: boolean;
     };
-    constructor({ id, name, description, config, rateLimit, capabilities, type, maxContextLength, provider, queue, authenticationInformation, workflows }: ExuluAgentParams);
-    get providerName(): string;
-    get modelName(): string;
-    tool: (instance: string, agents: ExuluAgent[], contexts: ExuluContext[], rerankers: ExuluReranker[]) => Promise<ExuluTool | null>;
-    generateSync: ({ prompt, req, user, session, inputMessages, currentTools, allExuluTools, statistics, toolConfigs, providerapikey, contexts, rerankers, exuluConfig, outputSchema, agentInstance, instructions }: {
-        prompt?: string;
-        user?: User;
-        req?: Request;
-        session?: string;
-        agentInstance?: Agent;
-        inputMessages?: UIMessage[];
-        currentTools?: ExuluTool[];
-        allExuluTools?: ExuluTool[];
-        statistics?: ExuluStatisticParams;
-        toolConfigs?: ExuluAgentToolConfig[];
-        providerapikey?: string | undefined;
-        contexts?: ExuluContext[] | undefined;
-        rerankers?: ExuluReranker[] | undefined;
-        exuluConfig?: ExuluConfig;
-        instructions?: string;
-        outputSchema?: z.ZodType;
-    }) => Promise<string | any>;
-    /**
-     * Convert file parts in messages to OpenAI Responses API compatible format.
-     * The OpenAI Responses API doesn't support inline file parts with type 'file'.
-     * This function converts:
-     * - Document files (PDF, DOCX, etc.) -> text parts with extracted content using officeparser
-     * - Image files -> image parts (which ARE supported by Responses API)
-     */
-    private processFilePartsInMessages;
-    generateStream: ({ user, session, agentInstance, message, previousMessages, currentTools, approvedTools, allExuluTools, toolConfigs, providerapikey, contexts, rerankers, exuluConfig, instructions, req, }: {
-        user?: User;
-        session?: string;
-        agentInstance?: Agent;
-        message?: UIMessage;
-        previousMessages?: UIMessage[];
-        currentTools?: ExuluTool[];
-        approvedTools?: string[];
-        allExuluTools?: ExuluTool[];
-        toolConfigs?: ExuluAgentToolConfig[];
-        providerapikey?: string | undefined;
-        contexts?: ExuluContext[] | undefined;
-        rerankers?: ExuluReranker[] | undefined;
-        exuluConfig?: ExuluConfig;
-        instructions?: string;
-        req?: Request;
-    }) => Promise<{
-        stream: ReturnType<typeof streamText>;
-        originalMessages: UIMessage[];
-        previousMessages: UIMessage[];
-    }>;
-}
-type VectorOperationResponse = Promise<{
-    count: number;
-    results: any;
-    errors?: string[];
+};
+
+type STATISTICS_TYPE = "CONTEXT_RETRIEVE" | "SOURCE_UPDATE" | "EMBEDDER_UPSERT" | "EMBEDDER_GENERATE" | "EMBEDDER_DELETE" | "WORKFLOW_RUN" | "CONTEXT_UPSERT" | "TOOL_CALL" | "AGENT_RUN";
+declare const STATISTICS_TYPE_ENUM: {
+    CONTEXT_RETRIEVE: string;
+    SOURCE_UPDATE: string;
+    EMBEDDER_UPSERT: string;
+    EMBEDDER_GENERATE: string;
+    EMBEDDER_DELETE: string;
+    WORKFLOW_RUN: string;
+    CONTEXT_UPSERT: string;
+    TOOL_CALL: string;
+    AGENT_RUN: string;
+};
+
+type ExuluStatisticParams = Omit<ExuluStatistic, "total" | "name" | "type">;
+type ExuluStatistic = {
+    name: string;
+    label: string;
+    type: STATISTICS_TYPE;
+    trigger: STATISTICS_LABELS;
+    total: number;
+};
+type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user" | "processor";
+
+type ExuluEmbedderConfig = {
+    name: string;
+    description: string;
+    default?: string;
+};
+type VectorGenerationResponse = Promise<{
+    id: string;
+    chunks: {
+        content: string;
+        index: number;
+        metadata: Record<string, string>;
+        vector: number[];
+    }[];
 }>;
 type VectorGenerateOperation = (inputs: ChunkerResponse, settings: Record<string, string>) => VectorGenerationResponse;
 type ChunkerOperation = (item: Item & {
@@ -593,20 +388,6 @@ type ChunkerResponse = {
         index: number;
     }[];
 };
-type VectorGenerationResponse = Promise<{
-    id: string;
-    chunks: {
-        content: string;
-        index: number;
-        metadata: Record<string, string>;
-        vector: number[];
-    }[];
-}>;
-type ExuluEmbedderConfig = {
-    name: string;
-    description: string;
-    default?: string;
-};
 declare class ExuluEmbedder {
     id: string;
     name: string;
@@ -618,7 +399,7 @@ declare class ExuluEmbedder {
     config?: ExuluEmbedderConfig[];
     maxChunkSize: number;
     _chunker: ChunkerOperation;
-    constructor({ id, name, description, generateEmbeddings, queue, vectorDimensions, maxChunkSize, chunker, config }: {
+    constructor({ id, name, description, generateEmbeddings, queue, vectorDimensions, maxChunkSize, chunker, config, }: {
         id: string;
         name: string;
         description: string;
@@ -636,80 +417,16 @@ declare class ExuluEmbedder {
     generateFromQuery(context: string, query: string, statistics?: ExuluStatisticParams, user?: number, role?: string): VectorGenerationResponse;
     generateFromDocument(context: string, input: Item, config: ExuluConfig, statistics?: ExuluStatisticParams, user?: number, role?: string): VectorGenerationResponse;
 }
-declare class ExuluTool {
-    id: string;
-    name: string;
-    description: string;
-    category: string;
-    inputSchema?: z.ZodType;
-    type: "context" | "function" | "agent" | "web_search";
-    tool: Tool;
-    config: {
-        name: string;
-        description: string;
-        type: "boolean" | "string" | "number" | "variable";
-        default?: string | boolean | number | "variable";
-    }[];
-    constructor({ id, name, description, category, inputSchema, type, execute, config }: {
-        id: string;
-        name: string;
-        description: string;
-        category?: string;
-        inputSchema?: z.ZodType;
-        type: "context" | "function" | "agent" | "web_search";
-        config: {
-            name: string;
-            description: string;
-            type: "boolean" | "string" | "number" | "variable";
-            default?: string | boolean | number | "variable";
-        }[];
-        execute: (inputs: any) => Promise<{
-            result?: string;
-            job?: string;
-            items?: Item[];
-        }> | AsyncGenerator<{
-            result?: string;
-            job?: string;
-            items?: Item[];
-        }>;
-    });
-    execute: ({ agent, config, user, inputs, project, items }: {
-        agent: string;
-        config: ExuluConfig;
-        user?: User;
-        inputs: any;
-        project?: string;
-        items?: string[];
-    }) => Promise<any>;
-}
-type ExuluContextProcessor = {
-    name: string;
-    description: string;
-    filter: ({ item, user, role, utils, exuluConfig }: {
-        item: Item;
-        user?: number;
-        role?: string;
-        utils: {
-            storage: ExuluStorage;
-        };
-        exuluConfig: ExuluConfig;
-    }) => Promise<Item | undefined | null>;
-    execute: ({ item, user, role, utils, exuluConfig }: {
-        item: Item;
-        user?: number;
-        role?: string;
-        utils: {
-            storage: ExuluStorage;
-        };
-        exuluConfig: ExuluConfig;
-    }) => Promise<Item>;
-    config?: {
-        queue?: Promise<ExuluQueueConfig>;
-        timeoutInSeconds?: number;
-        trigger: "manual" | "onUpdate" | "onInsert" | "always";
-        generateEmbeddings?: boolean;
-    };
+
+type ExuluRightsMode = "private" | "users" | "roles" | "public";
+
+declare const VectorMethodEnum: {
+    readonly cosineDistance: "cosineDistance";
+    readonly hybridSearch: "hybridSearch";
+    readonly tsvector: "tsvector";
 };
+type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
+
 type ExuluContextFieldDefinition = {
     name: string;
     type: ExuluFieldTypes;
@@ -722,15 +439,6 @@ type ExuluContextFieldDefinition = {
     enumValues?: string[];
     allowedFileTypes?: allFileTypes[];
 };
-type ExuluRightsMode = "private" | "users" | "roles" | "public";
-declare class ExuluStorage {
-    private config;
-    constructor({ config }: {
-        config: ExuluConfig;
-    });
-    getPresignedUrl: (key: string) => Promise<string>;
-    uploadFile: (file: Buffer | Uint8Array, fileName: string, type: string, user?: number, metadata?: Record<string, string>, customBucket?: string) => Promise<string>;
-}
 type ExuluContextSource = {
     id: string;
     name: string;
@@ -740,7 +448,7 @@ type ExuluContextSource = {
         queue?: Promise<ExuluQueueConfig>;
         retries?: number;
         backoff?: {
-            type: 'exponential' | 'linear';
+            type: "exponential" | "linear";
             delay: number;
         };
         params?: {
@@ -801,10 +509,10 @@ declare class ExuluContext {
             before?: number;
             after?: number;
         };
-        language?: "german" | "english";
+        languages?: ("german" | "english")[];
     };
     sources: ExuluContextSource[];
-    constructor({ id, name, description, embedder, processor, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
+    constructor({ id, name, description, embedder, processor, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources, }: {
         id: string;
         name: string;
         fields: ExuluContextFieldDefinition[];
@@ -821,7 +529,7 @@ declare class ExuluContext {
             calculateVectors?: "manual" | "onUpdate" | "onInsert" | "always";
             defaultRightsMode?: ExuluRightsMode;
             enableAsTool?: boolean;
-            language?: "german" | "english";
+            languages?: ("german" | "english")[];
             maxRetrievalResults?: number;
             expand?: {
                 before?: number;
@@ -870,79 +578,374 @@ declare class ExuluContext {
             id: string;
             embedder: string;
         };
-        chunks: VectorSearchChunkResult[];
-    }>;
-    deleteAll: () => Promise<VectorOperationResponse>;
-    executeSource: (source: ExuluContextSource, inputs: any, exuluConfig: ExuluConfig) => Promise<Item[]>;
-    tableExists: () => Promise<boolean>;
-    chunksTableExists: () => Promise<boolean>;
-    createAndUpsertEmbeddings: (item: Item, config: ExuluConfig, user?: number, statistics?: ExuluStatisticParams, role?: string, job?: string) => Promise<{
-        id: string;
-        chunks?: number;
-        job?: string;
-    }>;
-    createItem: (item: Item, config: ExuluConfig, user?: number, role?: string, upsert?: boolean, generateEmbeddingsOverwrite?: boolean | undefined) => Promise<{
-        item: Item;
-        job?: string;
-    }>;
-    updateItem: (item: Item, config: ExuluConfig, user?: number, role?: string, generateEmbeddingsOverwrite?: boolean | undefined) => Promise<{
-        item: Item;
-        job?: string;
-    }>;
-    deleteItem: (item: Item, user?: number, role?: string) => Promise<{
-        id: string;
-        job?: string;
-    }>;
-    getItem: ({ item }: {
-        item: Item;
-    }) => Promise<Item>;
-    getItems: ({ filters, fields }: {
-        filters?: any[];
-        fields?: string[];
-    }) => Promise<Item[]>;
+        chunks: VectorSearchChunkResult[];
+    }>;
+    deleteAll: () => Promise<{
+        count: number;
+        results: any;
+        errors?: string[];
+    }>;
+    executeSource: (source: ExuluContextSource, inputs: any, exuluConfig: ExuluConfig) => Promise<Item[]>;
+    tableExists: () => Promise<boolean>;
+    chunksTableExists: () => Promise<boolean>;
+    createAndUpsertEmbeddings: (item: Item, config: ExuluConfig, user?: number, statistics?: ExuluStatisticParams, role?: string, job?: string) => Promise<{
+        id: string;
+        chunks?: number;
+        job?: string;
+    }>;
+    createItem: (item: Item, config: ExuluConfig, user?: number, role?: string, upsert?: boolean, generateEmbeddingsOverwrite?: boolean) => Promise<{
+        item: Item;
+        job?: string;
+    }>;
+    updateItem: (item: Item, config: ExuluConfig, user?: number, role?: string, generateEmbeddingsOverwrite?: boolean) => Promise<{
+        item: Item;
+        job?: string;
+    }>;
+    deleteItem: (item: Item, user?: number, role?: string) => Promise<{
+        id: string;
+        job?: string;
+    }>;
+    getItem: ({ item }: {
+        item: Item;
+    }) => Promise<Item>;
+    getItems: ({ filters, fields, }: {
+        filters?: any[];
+        fields?: string[];
+    }) => Promise<Item[]>;
+    embeddings: {
+        generate: {
+            one: ({ item, user, role, trigger, config, }: {
+                item: Item;
+                user?: number;
+                role?: string;
+                trigger: STATISTICS_LABELS;
+                config: ExuluConfig;
+            }) => Promise<{
+                id: string;
+                job?: string;
+                chunks?: number;
+            }>;
+            all: (config: ExuluConfig, userId?: number, roleId?: string, limit?: number) => Promise<{
+                jobs: string[];
+                items: number;
+            }>;
+        };
+    };
+    createItemsTable: () => Promise<void>;
+    createChunksTable: () => Promise<void>;
+    tool: () => ExuluTool | null;
+}
+
+type VectorSearchChunkResult = {
+    chunk_content: string;
+    chunk_index: number;
+    chunk_id: string;
+    chunk_source: string;
+    chunk_metadata: Record<string, string>;
+    chunk_created_at: string;
+    chunk_updated_at: string;
+    item_id: string;
+    item_external_id: string;
+    item_name: string;
+    item_updated_at: string;
+    item_created_at: string;
+    chunk_cosine_distance?: number;
+    chunk_fts_rank?: number;
+    chunk_hybrid_score?: number;
+    context?: {
+        name: string;
+        id: string;
+    };
+};
+
+declare class ExuluReranker {
+    id: string;
+    name: string;
+    description: string;
+    execute: (params: {
+        query: string;
+        chunks: VectorSearchChunkResult[];
+    }) => Promise<VectorSearchChunkResult[]>;
+    constructor({ id, name, description, execute, }: {
+        id: string;
+        name: string;
+        description: string;
+        execute: (params: {
+            query: string;
+            chunks: VectorSearchChunkResult[];
+        }) => Promise<VectorSearchChunkResult[]>;
+    });
+    run(query: string, chunks: VectorSearchChunkResult[]): Promise<VectorSearchChunkResult[]>;
+}
+
+type ExuluConfig = {
+    telemetry?: {
+        enabled: boolean;
+    };
+    logger?: {
+        winston: {
+            transports: transport[];
+        };
+    };
+    workers: {
+        enabled: boolean;
+        logger?: {
+            winston: {
+                transports: transport[];
+            };
+        };
+        telemetry?: {
+            enabled: boolean;
+        };
+    };
+    MCP: {
+        enabled: boolean;
+    };
+    fileUploads?: {
+        s3region: string;
+        s3key: string;
+        s3secret: string;
+        s3Bucket: string;
+        s3endpoint?: string;
+        s3prefix?: string;
+    };
+    privacy?: {
+        systemPromptPersonalization?: boolean;
+    };
+};
+declare class ExuluApp {
+    private _providers;
+    private _agents;
+    private _config?;
+    private _evals;
+    private _queues;
+    private _rerankers;
+    private _contexts?;
+    private _tools;
+    private _expressApp;
+    constructor();
+    create: ({ contexts, providers, config, agents, tools, evals, rerankers, }: {
+        contexts?: Record<string, ExuluContext>;
+        config: ExuluConfig;
+        agents?: ExuluAgent[];
+        providers?: ExuluProvider[];
+        rerankers?: ExuluReranker[];
+        evals?: ExuluEval[];
+        tools?: ExuluTool[];
+    }) => Promise<ExuluApp>;
+    express: {
+        init: () => Promise<Express>;
+    };
+    get expressApp(): Express;
+    tool(id: string): ExuluTool | undefined;
+    tools(): ExuluTool[];
+    agent(id: string, include?: {
+        source: {
+            code: boolean;
+            database: boolean;
+        };
+    }): Promise<ExuluAgent | undefined>;
+    agents(include?: {
+        source: {
+            code: boolean;
+            database: boolean;
+        };
+    }): Promise<ExuluAgent[]>;
+    context(id: string): ExuluContext | undefined;
+    provider(id: string): ExuluProvider | undefined;
+    get contexts(): ExuluContext[];
+    get providers(): ExuluProvider[];
     embeddings: {
         generate: {
-            one: ({ item, user, role, trigger, config }: {
-                item: Item;
-                user?: number;
-                role?: string;
-                trigger: STATISTICS_LABELS;
-                config: ExuluConfig;
+            one: ({ context: contextId, item: itemId }: {
+                context: string;
+                item: string;
             }) => Promise<{
                 id: string;
                 job?: string;
                 chunks?: number;
             }>;
-            all: (config: ExuluConfig, userId?: number, roleId?: string, limit?: number) => Promise<{
+            all: ({ context: contextId }: {
+                context: string;
+            }) => Promise<{
                 jobs: string[];
                 items: number;
             }>;
         };
     };
-    createItemsTable: () => Promise<void>;
-    createChunksTable: () => Promise<void>;
-    tool: () => ExuluTool | null;
+    bullmq: {
+        workers: {
+            create: (queues?: string[]) => Promise<bullmq.Worker<any, any, string>[]>;
+        };
+    };
+    private server;
 }
-type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user" | "processor";
-type ExuluStatistic = {
+
+declare class ExuluTool {
+    id: string;
     name: string;
-    label: string;
-    type: STATISTICS_TYPE;
-    trigger: STATISTICS_LABELS;
-    total: number;
+    description: string;
+    category: string;
+    inputSchema?: z.ZodType;
+    type: "context" | "function" | "agent" | "web_search";
+    tool: Tool;
+    config: {
+        name: string;
+        description: string;
+        type: "boolean" | "string" | "number" | "variable";
+        default?: string | boolean | number;
+    }[];
+    constructor({ id, name, description, category, inputSchema, type, execute, config, }: {
+        id: string;
+        name: string;
+        description: string;
+        category?: string;
+        inputSchema?: z.ZodType;
+        type: "context" | "function" | "agent" | "web_search";
+        config: {
+            name: string;
+            description: string;
+            type: "boolean" | "string" | "number" | "variable";
+            default?: string | boolean | number;
+        }[];
+        execute: (inputs: any) => Promise<{
+            result?: string;
+            job?: string;
+            items?: Item[];
+        }> | AsyncGenerator<{
+            result?: string;
+            job?: string;
+            items?: Item[];
+        }>;
+    });
+    execute: ({ agent: agentId, config, user, inputs, project, items, }: {
+        agent: string;
+        config: ExuluConfig;
+        user?: User;
+        inputs: any;
+        project?: string;
+        items?: string[];
+    }) => Promise<any>;
+}
+
+type ExuluAgentToolConfig = {
+    id: string;
+    type: string;
+    config: {
+        name: string;
+        variable: string | boolean | number;
+        type: "boolean" | "string" | "number" | "variable";
+        value?: any;
+        default?: string | boolean | number;
+    }[];
 };
-type ExuluStatisticParams = Omit<ExuluStatistic, "total" | "name" | "type">;
 
-declare const authentication: ({ apikey, authtoken, internalkey, db, }: {
-    authtoken?: any;
-    apikey?: string;
-    internalkey?: string;
-    db: Knex;
-}) => Promise<{
-    error: boolean;
-    message?: string;
-    code?: number;
-    user?: User;
+type ExuluProviderWorkflowConfig = {
+    enabled: boolean;
+    queue?: Promise<ExuluQueueConfig>;
+};
+interface ExuluProviderParams {
+    id: string;
+    name: string;
+    type: "agent";
+    description: string;
+    config?: ExuluProviderConfig | undefined;
+    queue?: ExuluQueueConfig;
+    maxContextLength?: number;
+    authenticationInformation?: string;
+    provider: string;
+    workflows?: ExuluProviderWorkflowConfig;
+    capabilities?: {
+        text: boolean;
+        images: imageTypes$1[];
+        files: fileTypes$1[];
+        audio: audioTypes$1[];
+        video: videoTypes$1[];
+    };
+    outputSchema?: z.ZodType;
+    rateLimit?: RateLimiterRule;
+}
+declare class ExuluProvider {
+    id: string;
+    name: string;
+    provider: string;
+    description: string;
+    slug: string;
+    type: "agent";
+    streaming: boolean;
+    authenticationInformation?: string;
+    maxContextLength?: number;
+    workflows?: ExuluProviderWorkflowConfig;
+    queue?: ExuluQueueConfig;
+    rateLimit?: RateLimiterRule;
+    config?: ExuluProviderConfig | undefined;
+    model?: {
+        create: ({ apiKey }: {
+            apiKey?: string | undefined;
+        }) => LanguageModel;
+    };
+    capabilities: {
+        text: boolean;
+        images: string[];
+        files: string[];
+        audio: string[];
+        video: string[];
+    };
+    constructor({ id, name, description, config, rateLimit, capabilities, type, maxContextLength, provider, queue, authenticationInformation, workflows, }: ExuluProviderParams);
+    get providerName(): string;
+    get modelName(): string;
+    tool: (instance: string, providers: ExuluProvider[], contexts: ExuluContext[], rerankers: ExuluReranker[]) => Promise<ExuluTool | null>;
+    generateSync: ({ prompt, req, user, session, inputMessages, currentTools, allExuluTools, statistics, toolConfigs, providerapikey, contexts, rerankers, exuluConfig, outputSchema, agent, instructions, }: {
+        prompt?: string;
+        user?: User;
+        req?: Request;
+        session?: string;
+        agent?: ExuluAgent;
+        inputMessages?: UIMessage[];
+        currentTools?: ExuluTool[];
+        allExuluTools?: ExuluTool[];
+        statistics?: ExuluStatisticParams;
+        toolConfigs?: ExuluAgentToolConfig[];
+        providerapikey?: string | undefined;
+        contexts?: ExuluContext[] | undefined;
+        rerankers?: ExuluReranker[] | undefined;
+        exuluConfig?: ExuluConfig;
+        instructions?: string;
+        outputSchema?: z.ZodType;
+    }) => Promise<any>;
+    /**
+     * Convert file parts in messages to OpenAI Responses API compatible format.
+     * The OpenAI Responses API doesn't support inline file parts with type 'file'.
+     * This function converts:
+     * - Document files (PDF, DOCX, etc.) -> text parts with extracted content using officeparser
+     * - Image files -> image parts (which ARE supported by Responses API)
+     */
+    private processFilePartsInMessages;
+    generateStream: ({ user, session, agent, message, previousMessages, currentTools, approvedTools, allExuluTools, toolConfigs, providerapikey, contexts, rerankers, exuluConfig, instructions, req, }: {
+        user?: User;
+        session?: string;
+        agent?: ExuluAgent;
+        message?: UIMessage;
+        previousMessages?: UIMessage[];
+        currentTools?: ExuluTool[];
+        approvedTools?: string[];
+        allExuluTools?: ExuluTool[];
+        toolConfigs?: ExuluAgentToolConfig[];
+        providerapikey?: string | undefined;
+        contexts?: ExuluContext[] | undefined;
+        rerankers?: ExuluReranker[] | undefined;
+        exuluConfig?: ExuluConfig;
+        instructions?: string;
+        req?: Request;
+    }) => Promise<{
+        stream: ReturnType<typeof streamText>;
+        originalMessages: UIMessage[];
+        previousMessages: UIMessage[];
+    }>;
+}
+
+declare function redisClient(): Promise<{
+    client: RedisClientType | null;
 }>;
 
 declare class ExuluQueues {
@@ -983,29 +986,6 @@ declare class ExuluQueues {
 }
 declare const queues: ExuluQueues;
 
-/**
- * Create metadata for console logging that will be used by S3 transport
- * to group logs by ID and store them in S3 with the ID as prefix
- *
- * @param id - The ID to group logs by (e.g., job ID, request ID, etc.)
- * @param additionalMetadata - Any additional metadata to include with the log
- * @returns Metadata object to pass as the last argument to console methods
- *
- * @example
- * ```typescript
- * import { logMetadata } from './registry/log-metadata';
- *
- * const jobId = 'job-123';
- * console.log('Starting job', logMetadata(jobId));
- * console.log('Processing...', logMetadata(jobId, { step: 'validation' }));
- * console.error('Job failed', logMetadata(jobId, { error: 'timeout' }));
- * ```
- */
-declare function logMetadata(id: string, additionalMetadata?: Record<string, any>): {
-    __logMetadata: boolean;
-    id: string;
-};
-
 /**
  * Represents the data structure for a chunk object.
  *
@@ -1092,7 +1072,7 @@ declare class Chunk {
  *
  * @enum {string}
  */
-type IncludeDelim = 'prev' | 'next';
+type IncludeDelim = "prev" | "next";
 /** Interface for RecursiveLevel data
  *
  * @interface RecursiveLevelData
@@ -1147,13 +1127,6 @@ declare class RecursiveLevel {
      * @returns {RecursiveLevel} The RecursiveLevel object.
      */
     static fromDict(data: RecursiveLevelData): RecursiveLevel;
-    /** Create RecursiveLevel object from a recipe
-     *
-     * @param {string} name - The name of the recipe.
-     * @param {string} lang - The language of the recipe.
-     * @returns {Promise<RecursiveLevel>} The RecursiveLevel object.
-     */
-    static fromRecipe(name: string, lang?: string): Promise<RecursiveLevel>;
 }
 /** Interface for RecursiveRules data
  *
@@ -1204,14 +1177,6 @@ declare class RecursiveRules {
      * @returns {RecursiveRulesData} The dictionary-like object.
      */
     toDict(): RecursiveRulesData;
-    /** Create a RecursiveRules object from a recipe
-     *
-     * @param {string} name - The name of the recipe.
-     * @param {string} lang - The language of the recipe.
-     * @param {string} path - The path to the recipe.
-     * @returns {Promise<RecursiveRules>} The RecursiveRules object.
-     */
-    static fromRecipe(name?: string, lang?: string, path?: string): Promise<RecursiveRules>;
 }
 /** Interface for RecursiveChunk data
  *
@@ -1372,15 +1337,19 @@ declare abstract class BaseChunker {
  *
  * @interface RecursiveChunkerOptions
  * @property {string | Tokenizer} [tokenizer] - The tokenizer to use for text processing. Can be a string identifier (default: "Xenova/gpt2") or a Tokenizer instance.
- * @property {number} [chunkSize] - The maximum number of tokens per chunk. Must be greater than 0. Default: 512.
+ * @property {number} [chunkSize] - The maximum number Thof tokens per chunk. Must be greater than 0. Default: 512.
  * @property {RecursiveRules} [rules] - The rules that define how text should be recursively chunked. Default: new RecursiveRules().
  * @property {number} [minCharactersPerChunk] - The minimum number of characters that should be in each chunk. Must be greater than 0. Default: 24.
+ * @property {string} [prefix] - Optional prefix to prepend to each chunk. The token count of the prefix will be subtracted from the chunk size.
+ * @property {boolean} [retainHeaders] - Whether to retain headers in each chunk for context. Headers (HTML h1-h6 or Markdown #-######) will be prepended to chunks. Default: false.
  */
 interface RecursiveChunkerOptions {
     tokenizer?: TokenizerModelName;
     chunkSize?: number;
     rules?: RecursiveRules;
     minCharactersPerChunk?: number;
+    prefix?: string;
+    retainHeaders?: boolean;
 }
 /**
  * Represents a RecursiveChunker instance that is also directly callable as a function.
@@ -1414,7 +1383,7 @@ interface RecursiveChunkerOptions {
  */
 type CallableRecursiveChunker = RecursiveChunker & {
     (text: string, showProgress?: boolean): Promise<RecursiveChunk[]>;
-    (texts: string[], showProgress?: boolean): Promise<(RecursiveChunk[])[]>;
+    (texts: string[], showProgress?: boolean): Promise<RecursiveChunk[][]>;
 };
 /**
  * Recursively chunk text using a set of rules.
@@ -1424,10 +1393,13 @@ type CallableRecursiveChunker = RecursiveChunker & {
  * delimiters, whitespace, and token-based chunking.
  *
  * @extends BaseChunker
- * @property {number} chunkSize - The maximum number of tokens per chunk.
+ * @property {number} chunkSize - The maximum number of tokens per chunk (adjusted for prefix if provided).
  * @property {number} minCharactersPerChunk - The minimum number of characters per chunk.
  * @property {RecursiveRules} rules - The rules that define how text should be recursively chunked.
  * @property {string} sep - The separator string used for internal splitting (usually "✄").
+ * @property {string} [prefix] - Optional prefix prepended to each chunk.
+ * @property {number} prefixTokenCount - The number of tokens in the prefix (0 if no prefix).
+ * @property {boolean} retainHeaders - Whether to retain headers in each chunk for context.
  *
  * @method chunk - Recursively chunk a single text into chunks or strings.
  * @method chunkBatch - Recursively chunk a batch of texts.
@@ -1450,7 +1422,12 @@ declare class RecursiveChunker extends BaseChunker {
     readonly minCharactersPerChunk: number;
     readonly rules: RecursiveRules;
     readonly sep: string;
+    readonly prefix?: string;
+    readonly prefixTokenCount: number;
+    readonly retainHeaders: boolean;
     private readonly _CHARS_PER_TOKEN;
+    private _headers;
+    private _headerHierarchy;
     /**
      * Private constructor. Use `RecursiveChunker.create()` to instantiate.
      */
@@ -1466,6 +1443,7 @@ declare class RecursiveChunker extends BaseChunker {
      *   @param {number} [options.chunkSize=512] - Maximum number of tokens per chunk. Must be > 0.
      *   @param {RecursiveRules} [options.rules=new RecursiveRules()] - Rules for recursive chunking. See {@link RecursiveRules} for customization.
      *   @param {number} [options.minCharactersPerChunk=24] - Minimum number of characters per chunk. Must be > 0.
+     *   @param {string} [options.prefix] - Optional prefix to prepend to each chunk. The token count of the prefix will be subtracted from the chunk size.
      *
      * @returns {Promise<CallableRecursiveChunker>} Promise resolving to a callable RecursiveChunker instance.
      *
@@ -1485,6 +1463,10 @@ declare class RecursiveChunker extends BaseChunker {
      * const chunker = await RecursiveChunker.create();
      * const chunks = await chunker.chunk("Some text"); // Use as object method
      *
+     * @example <caption>Using a prefix</caption>
+     * const chunker = await RecursiveChunker.create({ chunkSize: 512, prefix: "Context: " });
+     * const chunks = await chunker("Some text to chunk");
+     *
      * @note
      * The returned instance is both callable (like a function) and has all properties/methods of RecursiveChunker.
      * You can use it as a drop-in replacement for a function or a class instance.
@@ -1494,6 +1476,21 @@ declare class RecursiveChunker extends BaseChunker {
      * See {@link RecursiveRules} and {@link RecursiveLevel} for rule structure.
      */
     static create(options?: RecursiveChunkerOptions): Promise<CallableRecursiveChunker>;
+    /**
+     * Extract all headers from the text (both HTML and Markdown).
+     *
+     * @param {string} text - The text to extract headers from
+     * @returns {HeaderInfo[]} Array of header information
+     * @private
+     */
+    private _extractHeaders;
+    /**
+     * Get the current header context as a formatted string.
+     *
+     * @returns {string} The formatted header context
+     * @private
+     */
+    private _getHeaderContext;
     /**
      * Estimates the number of tokens in a given text.
      *
@@ -1552,6 +1549,7 @@ declare class RecursiveChunker extends BaseChunker {
      *
      * This method is the main entry point for chunking text using the RecursiveChunker.
      * It takes a single text string and returns an array of RecursiveChunk objects.
+     * If a prefix was provided during instantiation, it will be prepended to each chunk.
      *
      * @param {string} text - The text to be chunked
      * @returns {Promise<RecursiveChunk[]>} A promise that resolves to an array of RecursiveChunk objects
@@ -1561,7 +1559,7 @@ declare class RecursiveChunker extends BaseChunker {
      * Return a string representation of the RecursiveChunker.
      *
      * This method provides a string representation of the RecursiveChunker instance,
-     * including its tokenizer, rules, chunk size, minimum characters per chunk, and return type.
+     * including its tokenizer, rules, chunk size, minimum characters per chunk, and prefix (if any).
      *
      * @returns {string} A string representation of the RecursiveChunker
      */
@@ -1861,6 +1859,66 @@ declare class SentenceChunker extends BaseChunker {
     toString(): string;
 }
 
+declare class MarkdownChunker {
+    private readonly _CHARS_PER_TOKEN;
+    constructor();
+    /**
+     * Converts markdown tables to LLM-friendly numbered list format.
+     * Example:
+     * | First name | Last name | Age |
+     * | -------- | -------- | -------- |
+     * | John     | Doe      | 30      |
+     * | Jane     | Smith    | 25      |
+     *
+     * Becomes:
+     * 1. First name: John, Last name: Doe, Age: 30
+     * 2. First name: Jane, Last name: Smith, Age: 25
+     *
+     * Handles malformed tables gracefully by validating structure and skipping invalid tables.
+     */
+    convertTablesToText(text: string): string;
+    /**
+     * Find the nearest logical breakpoint working backwards from the end of the text.
+     * Logical breakpoints are prioritized as follows:
+     * 1. Before markdown headers (##) or HTML headers (<h1>-<h6>)
+     * 2. Paragraph breaks (double newlines)
+     * 3. Single newlines after list items or content
+     * 4. End of sentences (. ! ? followed by space)
+     * 5. Any whitespace
+     *
+     * Only considers breakpoints in the last 50% of the text to avoid creating very small chunks.
+     * Returns the position of the breakpoint, or null if none found
+     */
+    private findLogicalBreakpoint;
+    private headers;
+    chunk(text: string, chunkSize: number, prefix?: string): Promise<{
+        text: string;
+        page: number;
+    }[]>;
+}
+
+type DocumentProcessorConfig = {
+    vlm?: {
+        model: LanguageModel;
+        concurrency: number;
+    };
+    docling?: boolean;
+};
+type ProcessedPage = {
+    page: number;
+    content: string;
+    headings?: any;
+    image?: string;
+    vlm_corrected_text?: string;
+    vlm_validated?: boolean;
+};
+type ProcessedDocument = ProcessedPage[];
+declare function documentProcessor({ file, name, config }: {
+    file: string | Buffer;
+    name: string;
+    config?: DocumentProcessorConfig;
+}): Promise<ProcessedDocument | undefined>;
+
 type JOB_STATUS = "completed" | "failed" | "delayed" | "active" | "waiting" | "paused" | "stuck";
 declare const JOB_STATUS_ENUM: {
     completed: string;
@@ -1874,70 +1932,69 @@ declare const JOB_STATUS_ENUM: {
 
 declare const ExuluJobs: {
     redis: typeof redisClient;
-    jobs: {
-        validate: (job: bullmq.Job) => bullmq.Job<any, any, string>;
-    };
 };
-declare const ExuluDefaultAgents: {
+declare const ExuluDefaultProviders: {
     anthropic: {
-        opus4: ExuluAgent;
-        sonnet4: ExuluAgent;
-        sonnet45: ExuluAgent;
+        opus4: ExuluProvider;
+        sonnet4: ExuluProvider;
+        sonnet45: ExuluProvider;
     };
     cerebras: {
-        gptOss120b: ExuluAgent;
-        llama38b: ExuluAgent;
-        llama3370b: ExuluAgent;
+        gptOss120b: ExuluProvider;
+        llama38b: ExuluProvider;
+        llama3370b: ExuluProvider;
     };
     google: {
-        vertexGemini25Flash: ExuluAgent;
-        vertexGemini25Pro: ExuluAgent;
-        vertexGemini3Pro: ExuluAgent;
+        vertexGemini25Flash: ExuluProvider;
+        vertexGemini25Pro: ExuluProvider;
+        vertexGemini3Pro: ExuluProvider;
     };
     openai: {
-        gpt5Mini: ExuluAgent;
-        gpt5: ExuluAgent;
-        gpt5pro: ExuluAgent;
-        gpt5Codex: ExuluAgent;
-        gpt5Nano: ExuluAgent;
-        gpt41: ExuluAgent;
-        gpt41Mini: ExuluAgent;
-        gpt4o: ExuluAgent;
-        gpt4oMini: ExuluAgent;
+        gpt5Mini: ExuluProvider;
+        gpt5: ExuluProvider;
+        gpt5pro: ExuluProvider;
+        gpt5Codex: ExuluProvider;
+        gpt5Nano: ExuluProvider;
+        gpt41: ExuluProvider;
+        gpt41Mini: ExuluProvider;
+        gpt4o: ExuluProvider;
+        gpt4oMini: ExuluProvider;
     };
 };
 declare const ExuluVariables: {
     get: (name: string) => Promise<string>;
 };
-declare const ExuluUtils: {
-    batch: ({ fn, size, inputs, delay, retries }: {
-        fn: (data: any) => Promise<any>;
-        size: number;
-        inputs: any[];
-        delay: number;
-        retries: {
-            max: number;
-            delays: number[];
-        };
-    }) => Promise<any[]>;
-    retry: ({ fn, retries, delays }: {
-        fn: () => Promise<any>;
-        retries?: number;
-        delays?: number[];
-    }) => Promise<any>;
+declare const ExuluAuthentication: {
+    authenticate: ({ apikey, authtoken, internalkey, db, }: {
+        authtoken?: any;
+        apikey?: string;
+        internalkey?: string;
+        db: knex.Knex;
+    }) => Promise<{
+        error: boolean;
+        message?: string;
+        code?: number;
+        user?: User;
+    }>;
+};
+declare const ExuluDocumentProcessor: {
+    process: typeof documentProcessor;
 };
 declare const ExuluOtel: {
-    create: ({ SIGNOZ_ACCESS_TOKEN, SIGNOZ_TRACES_URL, SIGNOZ_LOGS_URL }: {
+    create: ({ SIGNOZ_ACCESS_TOKEN, SIGNOZ_TRACES_URL, SIGNOZ_LOGS_URL, }: {
         SIGNOZ_ACCESS_TOKEN: string;
         SIGNOZ_TRACES_URL: string;
         SIGNOZ_LOGS_URL: string;
     }) => _opentelemetry_sdk_node.NodeSDK;
 };
 
-declare const db: {
+declare const ExuluDatabase: {
     init: ({ contexts }: {
         contexts: ExuluContext[];
     }) => Promise<void>;
+    update: ({ contexts }: {
+        contexts: ExuluContext[];
+    }) => Promise<void>;
     api: {
         key: {
             generate: (name: string, email: string) => Promise<{
@@ -1948,10 +2005,11 @@ declare const db: {
 };
 declare const ExuluChunkers: {
     sentence: typeof SentenceChunker;
+    markdown: typeof MarkdownChunker;
     recursive: {
         function: typeof RecursiveChunker;
         rules: typeof RecursiveRules;
     };
 };
 
-export { type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, ExuluAgent, ExuluApp, authentication as ExuluAuthentication, ExuluChunkers, ExuluContext, ExuluDefaultAgents, ExuluEmbedder, ExuluEval, type ExuluEvalMetadata, type ExuluEvalTokenMetadata, type Item as ExuluItem, ExuluJobs, ExuluOtel, type ExuluQueueConfig, queues as ExuluQueues, ExuluReranker, ExuluStorage, ExuluTool, ExuluUtils, ExuluVariables, db, logMetadata };
+export { type JOB_STATUS as EXULU_JOB_STATUS, JOB_STATUS_ENUM as EXULU_JOB_STATUS_ENUM, type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, ExuluApp, ExuluAuthentication, ExuluChunkers, ExuluContext, ExuluDatabase, ExuluDefaultProviders, ExuluDocumentProcessor, ExuluEmbedder, ExuluEval, type Item as ExuluItem, ExuluJobs, ExuluOtel, queues as ExuluQueues, ExuluReranker, ExuluTool, ExuluVariables };