@exulu/backend 1.39.3 → 1.41.0

package/dist/index.d.cts

@@ -40,12 +40,12 @@ interface Item {
     source?: string;
     tags?: string[];
     textlength?: number;
+    last_processed_at?: string;
     chunks?: {
         id: string;
         index: number;
         content: string;
         source: string;
-        embedding_size: number;
         createdAt: string;
         updatedAt: string;
     }[];
@@ -160,6 +160,31 @@ type User = {
     };
 };
 
+declare const VectorMethodEnum: {
+    readonly cosineDistance: "cosineDistance";
+    readonly hybridSearch: "hybridSearch";
+    readonly tsvector: "tsvector";
+};
+type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
+
+type VectorSearchChunkResult = {
+    chunk_content: string;
+    chunk_index: number;
+    chunk_id: string;
+    chunk_source: string;
+    chunk_metadata: Record<string, string>;
+    chunk_created_at: string;
+    chunk_updated_at: string;
+    item_id: string;
+    item_external_id: string;
+    item_name: string;
+    item_updated_at: string;
+    item_created_at: string;
+    chunk_cosine_distance?: number;
+    chunk_fts_rank?: number;
+    chunk_hybrid_score?: number;
+};
+
 interface Agent {
     id: string;
     modelName?: string;
@@ -236,13 +261,6 @@ interface TestCase {
     updatedAt: string;
 }
 
-declare const VectorMethodEnum: {
-    readonly cosineDistance: "cosineDistance";
-    readonly hybridSearch: "hybridSearch";
-    readonly tsvector: "tsvector";
-};
-type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
-
 interface RateLimiterRule {
     name?: string;
     rate_limit: {
@@ -311,7 +329,10 @@ type ExuluQueueConfig = {
     queue: Queue;
     ratelimit: number;
     timeoutInSeconds?: number;
-    concurrency: number;
+    concurrency: {
+        worker: number;
+        queue: number;
+    };
     retries?: number;
     backoff?: {
         type: 'exponential' | 'linear';
@@ -532,24 +553,18 @@ declare class ExuluTool {
         }>;
     });
 }
-type ExuluContextFieldProcessor = {
+type ExuluContextProcessor = {
+    name: string;
     description: string;
     execute: ({ item, user, role, utils, exuluConfig }: {
-        item: Item & {
-            field: string;
-        };
+        item: Item;
         user?: number;
         role?: string;
         utils: {
             storage: ExuluStorage;
-            items: {
-                update: ExuluContext['updateItem'];
-                create: ExuluContext['createItem'];
-                delete: ExuluContext['deleteItem'];
-            };
         };
         exuluConfig: ExuluConfig;
-    }) => Promise<string>;
+    }) => Promise<Item>;
     config?: {
         queue?: Promise<ExuluQueueConfig>;
         timeoutInSeconds?: number;
@@ -567,7 +582,6 @@ type ExuluContextFieldDefinition = {
     index?: boolean;
     enumValues?: string[];
     allowedFileTypes?: allFileTypes[];
-    processor?: ExuluContextFieldProcessor;
 };
 type ExuluRightsMode = "private" | "users" | "roles" | "public";
 declare class ExuluStorage {
@@ -576,7 +590,7 @@ declare class ExuluStorage {
         config: ExuluConfig;
     });
     getPresignedUrl: (key: string) => Promise<string>;
-    uploadFile: (file: Buffer | Uint8Array, key: string, type: string, user?: number, metadata?: Record<string, string>) => Promise<string>;
+    uploadFile: (file: Buffer | Uint8Array, fileName: string, type: string, user?: number, metadata?: Record<string, string>, customBucket?: string) => Promise<string>;
 }
 type ExuluContextSource = {
     id: string;
@@ -606,18 +620,43 @@ declare class ExuluContext {
     name: string;
     active: boolean;
     fields: ExuluContextFieldDefinition[];
+    processor?: ExuluContextProcessor;
     rateLimit?: RateLimiterRule;
     description: string;
     embedder?: ExuluEmbedder;
     queryRewriter?: (query: string) => Promise<string>;
-    resultReranker?: (results: any[]) => Promise<any[]>;
+    resultReranker?: (results: {
+        chunk_content: string;
+        chunk_index: number;
+        chunk_id: string;
+        chunk_source: string;
+        chunk_metadata: Record<string, string>;
+        chunk_created_at: string;
+        chunk_updated_at: string;
+        item_id: string;
+        item_external_id: string;
+        item_name: string;
+    }[]) => Promise<{
+        chunk_content: string;
+        chunk_index: number;
+        chunk_id: string;
+        chunk_source: string;
+        chunk_metadata: Record<string, string>;
+        chunk_created_at: string;
+        chunk_updated_at: string;
+        item_id: string;
+        item_external_id: string;
+        item_name: string;
+    }[]>;
     configuration: {
         calculateVectors?: "manual" | "onUpdate" | "onInsert" | "always";
+        maxRetrievalResults?: number;
         defaultRightsMode?: ExuluRightsMode;
+        enableAsTool?: boolean;
         language?: "german" | "english";
     };
     sources: ExuluContextSource[];
-    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
+    constructor({ id, name, description, embedder, processor, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
         id: string;
         name: string;
         fields: ExuluContextFieldDefinition[];
@@ -626,19 +665,20 @@ declare class ExuluContext {
         sources: ExuluContextSource[];
         category?: string;
         active: boolean;
+        processor?: ExuluContextProcessor;
         rateLimit?: RateLimiterRule;
         queryRewriter?: (query: string) => Promise<string>;
         resultReranker?: (results: any[]) => Promise<any[]>;
         configuration?: {
             calculateVectors?: "manual" | "onUpdate" | "onInsert" | "always";
             defaultRightsMode?: ExuluRightsMode;
+            enableAsTool?: boolean;
             language?: "german" | "english";
+            maxRetrievalResults?: number;
         };
     });
-    processField: (trigger: STATISTICS_LABELS, item: Item & {
-        field: string;
-    }, exuluConfig: ExuluConfig, user?: number, role?: string) => Promise<{
-        result: string;
+    processField: (trigger: STATISTICS_LABELS, item: Item, exuluConfig: ExuluConfig, user?: number, role?: string) => Promise<{
+        result: Item | undefined;
         job?: string;
     }>;
     search: (options: {
@@ -660,7 +700,7 @@ declare class ExuluContext {
             id: string;
             embedder: string;
         };
-        items: any[];
+        chunks: VectorSearchChunkResult[];
     }>;
     deleteAll: () => Promise<VectorOperationResponse>;
     executeSource: (source: ExuluContextSource, inputs: any, exuluConfig: ExuluConfig) => Promise<Item[]>;
@@ -711,9 +751,9 @@ declare class ExuluContext {
     };
     createItemsTable: () => Promise<void>;
     createChunksTable: () => Promise<void>;
-    tool: () => ExuluTool;
+    tool: () => ExuluTool | null;
 }
-type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user";
+type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user" | "processor";
 type ExuluStatistic = {
     name: string;
     label: string;
@@ -739,21 +779,35 @@ declare class ExuluQueues {
     queues: {
         queue: Queue;
         ratelimit: number;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
+        timeoutInSeconds: number;
     }[];
     constructor();
     list: Map<string, {
         name: string;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
         ratelimit: number;
+        timeoutInSeconds: number;
         use: () => Promise<ExuluQueueConfig>;
     }>;
     queue(name: string): {
         queue: Queue;
         ratelimit: number;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
     } | undefined;
-    register: (name: string, concurrency?: number, ratelimit?: number) => {
+    register: (name: string, concurrency: {
+        worker: number;
+        queue: number;
+    }, ratelimit?: number, timeoutInSeconds?: number) => {
         use: () => Promise<ExuluQueueConfig>;
     };
 }

package/dist/index.d.ts

@@ -40,12 +40,12 @@ interface Item {
     source?: string;
     tags?: string[];
     textlength?: number;
+    last_processed_at?: string;
     chunks?: {
         id: string;
         index: number;
         content: string;
         source: string;
-        embedding_size: number;
         createdAt: string;
         updatedAt: string;
     }[];
@@ -160,6 +160,31 @@ type User = {
     };
 };
 
+declare const VectorMethodEnum: {
+    readonly cosineDistance: "cosineDistance";
+    readonly hybridSearch: "hybridSearch";
+    readonly tsvector: "tsvector";
+};
+type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
+
+type VectorSearchChunkResult = {
+    chunk_content: string;
+    chunk_index: number;
+    chunk_id: string;
+    chunk_source: string;
+    chunk_metadata: Record<string, string>;
+    chunk_created_at: string;
+    chunk_updated_at: string;
+    item_id: string;
+    item_external_id: string;
+    item_name: string;
+    item_updated_at: string;
+    item_created_at: string;
+    chunk_cosine_distance?: number;
+    chunk_fts_rank?: number;
+    chunk_hybrid_score?: number;
+};
+
 interface Agent {
     id: string;
     modelName?: string;
@@ -236,13 +261,6 @@ interface TestCase {
     updatedAt: string;
 }
 
-declare const VectorMethodEnum: {
-    readonly cosineDistance: "cosineDistance";
-    readonly hybridSearch: "hybridSearch";
-    readonly tsvector: "tsvector";
-};
-type VectorMethod = (typeof VectorMethodEnum)[keyof typeof VectorMethodEnum];
-
 interface RateLimiterRule {
     name?: string;
     rate_limit: {
@@ -311,7 +329,10 @@ type ExuluQueueConfig = {
     queue: Queue;
     ratelimit: number;
     timeoutInSeconds?: number;
-    concurrency: number;
+    concurrency: {
+        worker: number;
+        queue: number;
+    };
     retries?: number;
     backoff?: {
         type: 'exponential' | 'linear';
@@ -532,24 +553,18 @@ declare class ExuluTool {
         }>;
     });
 }
-type ExuluContextFieldProcessor = {
+type ExuluContextProcessor = {
+    name: string;
     description: string;
     execute: ({ item, user, role, utils, exuluConfig }: {
-        item: Item & {
-            field: string;
-        };
+        item: Item;
         user?: number;
         role?: string;
         utils: {
             storage: ExuluStorage;
-            items: {
-                update: ExuluContext['updateItem'];
-                create: ExuluContext['createItem'];
-                delete: ExuluContext['deleteItem'];
-            };
         };
         exuluConfig: ExuluConfig;
-    }) => Promise<string>;
+    }) => Promise<Item>;
     config?: {
         queue?: Promise<ExuluQueueConfig>;
         timeoutInSeconds?: number;
@@ -567,7 +582,6 @@ type ExuluContextFieldDefinition = {
     index?: boolean;
     enumValues?: string[];
     allowedFileTypes?: allFileTypes[];
-    processor?: ExuluContextFieldProcessor;
 };
 type ExuluRightsMode = "private" | "users" | "roles" | "public";
 declare class ExuluStorage {
@@ -576,7 +590,7 @@ declare class ExuluStorage {
         config: ExuluConfig;
     });
     getPresignedUrl: (key: string) => Promise<string>;
-    uploadFile: (file: Buffer | Uint8Array, key: string, type: string, user?: number, metadata?: Record<string, string>) => Promise<string>;
+    uploadFile: (file: Buffer | Uint8Array, fileName: string, type: string, user?: number, metadata?: Record<string, string>, customBucket?: string) => Promise<string>;
 }
 type ExuluContextSource = {
     id: string;
@@ -606,18 +620,43 @@ declare class ExuluContext {
     name: string;
     active: boolean;
     fields: ExuluContextFieldDefinition[];
+    processor?: ExuluContextProcessor;
     rateLimit?: RateLimiterRule;
     description: string;
     embedder?: ExuluEmbedder;
     queryRewriter?: (query: string) => Promise<string>;
-    resultReranker?: (results: any[]) => Promise<any[]>;
+    resultReranker?: (results: {
+        chunk_content: string;
+        chunk_index: number;
+        chunk_id: string;
+        chunk_source: string;
+        chunk_metadata: Record<string, string>;
+        chunk_created_at: string;
+        chunk_updated_at: string;
+        item_id: string;
+        item_external_id: string;
+        item_name: string;
+    }[]) => Promise<{
+        chunk_content: string;
+        chunk_index: number;
+        chunk_id: string;
+        chunk_source: string;
+        chunk_metadata: Record<string, string>;
+        chunk_created_at: string;
+        chunk_updated_at: string;
+        item_id: string;
+        item_external_id: string;
+        item_name: string;
+    }[]>;
     configuration: {
         calculateVectors?: "manual" | "onUpdate" | "onInsert" | "always";
+        maxRetrievalResults?: number;
         defaultRightsMode?: ExuluRightsMode;
+        enableAsTool?: boolean;
         language?: "german" | "english";
     };
     sources: ExuluContextSource[];
-    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
+    constructor({ id, name, description, embedder, processor, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
         id: string;
         name: string;
         fields: ExuluContextFieldDefinition[];
@@ -626,19 +665,20 @@ declare class ExuluContext {
         sources: ExuluContextSource[];
         category?: string;
         active: boolean;
+        processor?: ExuluContextProcessor;
         rateLimit?: RateLimiterRule;
         queryRewriter?: (query: string) => Promise<string>;
         resultReranker?: (results: any[]) => Promise<any[]>;
         configuration?: {
             calculateVectors?: "manual" | "onUpdate" | "onInsert" | "always";
             defaultRightsMode?: ExuluRightsMode;
+            enableAsTool?: boolean;
             language?: "german" | "english";
+            maxRetrievalResults?: number;
         };
     });
-    processField: (trigger: STATISTICS_LABELS, item: Item & {
-        field: string;
-    }, exuluConfig: ExuluConfig, user?: number, role?: string) => Promise<{
-        result: string;
+    processField: (trigger: STATISTICS_LABELS, item: Item, exuluConfig: ExuluConfig, user?: number, role?: string) => Promise<{
+        result: Item | undefined;
         job?: string;
     }>;
     search: (options: {
@@ -660,7 +700,7 @@ declare class ExuluContext {
             id: string;
             embedder: string;
         };
-        items: any[];
+        chunks: VectorSearchChunkResult[];
     }>;
     deleteAll: () => Promise<VectorOperationResponse>;
     executeSource: (source: ExuluContextSource, inputs: any, exuluConfig: ExuluConfig) => Promise<Item[]>;
@@ -711,9 +751,9 @@ declare class ExuluContext {
     };
     createItemsTable: () => Promise<void>;
     createChunksTable: () => Promise<void>;
-    tool: () => ExuluTool;
+    tool: () => ExuluTool | null;
 }
-type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user";
+type STATISTICS_LABELS = "tool" | "agent" | "flow" | "api" | "claude-code" | "user" | "processor";
 type ExuluStatistic = {
     name: string;
     label: string;
@@ -739,21 +779,35 @@ declare class ExuluQueues {
     queues: {
         queue: Queue;
         ratelimit: number;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
+        timeoutInSeconds: number;
     }[];
     constructor();
     list: Map<string, {
         name: string;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
         ratelimit: number;
+        timeoutInSeconds: number;
         use: () => Promise<ExuluQueueConfig>;
     }>;
     queue(name: string): {
         queue: Queue;
         ratelimit: number;
-        concurrency: number;
+        concurrency: {
+            worker: number;
+            queue: number;
+        };
     } | undefined;
-    register: (name: string, concurrency?: number, ratelimit?: number) => {
+    register: (name: string, concurrency: {
+        worker: number;
+        queue: number;
+    }, ratelimit?: number, timeoutInSeconds?: number) => {
         use: () => Promise<ExuluQueueConfig>;
     };
 }