@exulu/backend 1.31.1 → 1.32.0

package/CHANGELOG.md

@@ -1,6 +1,6 @@
-## [1.31.1](https://github.com/Qventu/exulu-backend/compare/v1.31.0...v1.31.1) (2025-11-10)
+# [1.32.0](https://github.com/Qventu/exulu-backend/compare/v1.31.2...v1.32.0) (2025-11-11)
 
 
-### Bug Fixes
+### Features
 
-* redis auth url ([bbf51d6](https://github.com/Qventu/exulu-backend/commit/bbf51d63be8dbbc88e8989633207db4b147e3da6))
+* implement scheduled data sources for contexts ([d936754](https://github.com/Qventu/exulu-backend/commit/d9367545fc3dc35b9c2d2ad0820fc7d2fddd4666))

package/dist/index.cjs

@@ -67,6 +67,7 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
+  console.log("[EXULU] redisServer:", redisServer);
   if (!redisServer.host || !redisServer.port) {
     return { client: null };
   }
@@ -2333,7 +2334,7 @@ function createMutations(table, agents, contexts, tools, config) {
       if (!item) {
         throw new Error("Item not found, or your user does not have access to it.");
       }
-      const { job, result } = await exists.process(
+      const { job, result } = await exists.processField(
         "api",
         context.user.id,
         context.user.role?.id,
@@ -5379,10 +5380,12 @@ var ExuluContext = class {
   resultReranker;
   // todo typings
   configuration;
-  constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration }) {
+  sources = [];
+  constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }) {
     this.id = id;
     this.name = name;
     this.fields = fields || [];
+    this.sources = sources || [];
     this.configuration = configuration || {
       calculateVectors: "manual",
       language: "english",
@@ -5395,7 +5398,7 @@ var ExuluContext = class {
     this.queryRewriter = queryRewriter;
     this.resultReranker = resultReranker;
   }
-  process = async (trigger, user, role, item, config) => {
+  processField = async (trigger, user, role, item, config) => {
     console.log("[EXULU] processing field", item.field, " in context", this.id);
     console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
     const field = this.fields.find((field2) => field2.name === item.field?.replace("_s3key", ""));
@@ -5457,6 +5460,9 @@ var ExuluContext = class {
       results: []
     };
   };
+  executeSource = async (source, inputs) => {
+    return await source.execute(inputs);
+  };
   tableExists = async () => {
     const { db: db3 } = await postgresClient();
     const tableName = getTableName(this.id);
@@ -6650,8 +6656,13 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
     throw new Error("No redis server configured in the environment, so cannot start worker.");
   }
   if (!redisConnection) {
-    redisConnection = new import_ioredis.default({
-      ...redisServer,
+    let url = "";
+    if (redisServer.username) {
+      url = `redis://${redisServer.username}:${redisServer.password}@${redisServer.host}:${redisServer.port}`;
+    } else {
+      url = `redis://${redisServer.host}:${redisServer.port}`;
+    }
+    redisConnection = new import_ioredis.default(url, {
       enableOfflineQueue: true,
       retryStrategy: function(times) {
         return Math.max(Math.min(Math.exp(times), 2e4), 1e3);
@@ -6966,6 +6977,49 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 metadata: {}
               };
             }
+            if (data.type === "source") {
+              console.log("[EXULU] running a source job.", logMetadata(bullmqJob.name));
+              if (!data.source) {
+                throw new Error(`No source id set for source job.`);
+              }
+              if (!data.context) {
+                throw new Error(`No context id set for source job.`);
+              }
+              const context = contexts.find((c) => c.id === data.context);
+              if (!context) {
+                throw new Error(`Context ${data.context} not found in the registry.`);
+              }
+              const source = context.sources.find((s) => s.id === data.source);
+              if (!source) {
+                throw new Error(`Source ${data.source} not found in the context ${context.id}.`);
+              }
+              const result = await source.execute(data.inputs);
+              let jobs = [];
+              let items = [];
+              for (const item of result) {
+                const { item: createdItem, job } = await context.createItem(item, config, data.user, data.role);
+                if (job) {
+                  jobs.push(job);
+                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata(bullmqJob.name, {
+                    item: createdItem,
+                    job
+                  }));
+                }
+                if (createdItem.id) {
+                  items.push(createdItem.id);
+                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata(bullmqJob.name, {
+                    item: createdItem
+                  }));
+                }
+              }
+              return {
+                result,
+                metadata: {
+                  jobs,
+                  items
+                }
+              };
+            }
             throw new Error(`Invalid job type: ${data.type} for job ${bullmqJob.name}.`);
           } catch (error) {
             console.error(`[EXULU] job failed.`, error instanceof Error ? error.message : String(error));
@@ -8872,6 +8926,41 @@ var ExuluApp = class {
         if (queues2) {
           filteredQueues = filteredQueues.filter((q) => queues2.includes(q.queue.name));
         }
+        const sources = Object.values(this._contexts ?? {}).flatMap((context) => ({
+          ...context.sources,
+          context: context.id
+        }));
+        if (sources.length > 0) {
+          console.log("[EXULU] Creating ContextSource schedulers for", sources.length, "sources.");
+          for (const source of sources) {
+            const queue = await source.config?.queue;
+            if (queue) {
+              if (!source.config.schedule) {
+                throw new Error("Schedule is required for source when configuring a queue: " + source.name);
+              }
+              console.log("[EXULU] Creating ContextSource scheduler for", source.name, "in queue", queue.queue?.name);
+              await queue.queue?.upsertJobScheduler(source.id, {
+                pattern: source.config?.schedule
+              }, {
+                // default job data
+                name: `${source.id}-job`,
+                data: {
+                  source: source.id,
+                  context: source.context,
+                  type: "source"
+                },
+                opts: {
+                  backoff: {
+                    type: source.config.backoff?.type || "exponential",
+                    delay: source.config.backoff?.delay || 2e3
+                  },
+                  attempts: source.config.retries || 3,
+                  removeOnFail: 200
+                }
+              });
+            }
+          }
+        }
         return await createWorkers(
           this._agents,
           filteredQueues,

package/dist/index.d.cts

@@ -544,6 +544,21 @@ declare class ExuluStorage {
     getPresignedUrl: (key: string) => Promise<string>;
     uploadFile: (user: number, file: Buffer | Uint8Array, key: string, type: string, metadata?: Record<string, string>) => Promise<string>;
 }
+type ExuluContextSource = {
+    id: string;
+    name: string;
+    description: string;
+    config: {
+        schedule: string;
+        queue: Promise<ExuluQueueConfig>;
+        retries?: number;
+        backoff?: {
+            type: 'exponential' | 'linear';
+            delay: number;
+        };
+    };
+    execute: (inputs: any) => Promise<Item[]>;
+};
 declare class ExuluContext {
     id: string;
     name: string;
@@ -559,12 +574,14 @@ declare class ExuluContext {
         defaultRightsMode?: ExuluRightsMode;
         language?: "german" | "english";
     };
-    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration }: {
+    sources: ExuluContextSource[];
+    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
         id: string;
         name: string;
         fields: ExuluContextFieldDefinition[];
         description: string;
         embedder?: ExuluEmbedder;
+        sources: ExuluContextSource[];
         category?: string;
         active: boolean;
         rateLimit?: RateLimiterRule;
@@ -576,13 +593,14 @@ declare class ExuluContext {
             language?: "german" | "english";
         };
     });
-    process: (trigger: STATISTICS_LABELS, user: number, role: string, item: Item & {
+    processField: (trigger: STATISTICS_LABELS, user: number, role: string, item: Item & {
         field: string;
     }, config: ExuluConfig) => Promise<{
         result: string;
         job?: string;
     }>;
     deleteAll: () => Promise<VectorOperationResponse>;
+    executeSource: (source: ExuluContextSource, inputs: any) => Promise<Item[]>;
     tableExists: () => Promise<boolean>;
     chunksTableExists: () => Promise<boolean>;
     createAndUpsertEmbeddings: (item: Item, config: ExuluConfig, user?: number, statistics?: ExuluStatisticParams, role?: string, job?: string) => Promise<{

package/dist/index.d.ts

@@ -544,6 +544,21 @@ declare class ExuluStorage {
     getPresignedUrl: (key: string) => Promise<string>;
     uploadFile: (user: number, file: Buffer | Uint8Array, key: string, type: string, metadata?: Record<string, string>) => Promise<string>;
 }
+type ExuluContextSource = {
+    id: string;
+    name: string;
+    description: string;
+    config: {
+        schedule: string;
+        queue: Promise<ExuluQueueConfig>;
+        retries?: number;
+        backoff?: {
+            type: 'exponential' | 'linear';
+            delay: number;
+        };
+    };
+    execute: (inputs: any) => Promise<Item[]>;
+};
 declare class ExuluContext {
     id: string;
     name: string;
@@ -559,12 +574,14 @@ declare class ExuluContext {
         defaultRightsMode?: ExuluRightsMode;
         language?: "german" | "english";
     };
-    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration }: {
+    sources: ExuluContextSource[];
+    constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }: {
         id: string;
         name: string;
         fields: ExuluContextFieldDefinition[];
         description: string;
         embedder?: ExuluEmbedder;
+        sources: ExuluContextSource[];
         category?: string;
         active: boolean;
         rateLimit?: RateLimiterRule;
@@ -576,13 +593,14 @@ declare class ExuluContext {
             language?: "german" | "english";
         };
     });
-    process: (trigger: STATISTICS_LABELS, user: number, role: string, item: Item & {
+    processField: (trigger: STATISTICS_LABELS, user: number, role: string, item: Item & {
         field: string;
     }, config: ExuluConfig) => Promise<{
         result: string;
         job?: string;
     }>;
     deleteAll: () => Promise<VectorOperationResponse>;
+    executeSource: (source: ExuluContextSource, inputs: any) => Promise<Item[]>;
     tableExists: () => Promise<boolean>;
     chunksTableExists: () => Promise<boolean>;
     createAndUpsertEmbeddings: (item: Item, config: ExuluConfig, user?: number, statistics?: ExuluStatisticParams, role?: string, job?: string) => Promise<{

package/dist/index.js

@@ -15,6 +15,7 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
+  console.log("[EXULU] redisServer:", redisServer);
   if (!redisServer.host || !redisServer.port) {
     return { client: null };
   }
@@ -2281,7 +2282,7 @@ function createMutations(table, agents, contexts, tools, config) {
       if (!item) {
         throw new Error("Item not found, or your user does not have access to it.");
       }
-      const { job, result } = await exists.process(
+      const { job, result } = await exists.processField(
         "api",
         context.user.id,
         context.user.role?.id,
@@ -5346,10 +5347,12 @@ var ExuluContext = class {
   resultReranker;
   // todo typings
   configuration;
-  constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration }) {
+  sources = [];
+  constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration, sources }) {
     this.id = id;
     this.name = name;
     this.fields = fields || [];
+    this.sources = sources || [];
     this.configuration = configuration || {
       calculateVectors: "manual",
       language: "english",
@@ -5362,7 +5365,7 @@ var ExuluContext = class {
     this.queryRewriter = queryRewriter;
     this.resultReranker = resultReranker;
   }
-  process = async (trigger, user, role, item, config) => {
+  processField = async (trigger, user, role, item, config) => {
     console.log("[EXULU] processing field", item.field, " in context", this.id);
     console.log("[EXULU] fields", this.fields.map((field2) => field2.name));
     const field = this.fields.find((field2) => field2.name === item.field?.replace("_s3key", ""));
@@ -5424,6 +5427,9 @@ var ExuluContext = class {
       results: []
     };
   };
+  executeSource = async (source, inputs) => {
+    return await source.execute(inputs);
+  };
   tableExists = async () => {
     const { db: db3 } = await postgresClient();
     const tableName = getTableName(this.id);
@@ -6617,8 +6623,13 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
     throw new Error("No redis server configured in the environment, so cannot start worker.");
   }
   if (!redisConnection) {
-    redisConnection = new IORedis({
-      ...redisServer,
+    let url = "";
+    if (redisServer.username) {
+      url = `redis://${redisServer.username}:${redisServer.password}@${redisServer.host}:${redisServer.port}`;
+    } else {
+      url = `redis://${redisServer.host}:${redisServer.port}`;
+    }
+    redisConnection = new IORedis(url, {
       enableOfflineQueue: true,
       retryStrategy: function(times) {
         return Math.max(Math.min(Math.exp(times), 2e4), 1e3);
@@ -6933,6 +6944,49 @@ var createWorkers = async (agents, queues2, config, contexts, evals, tools, trac
                 metadata: {}
               };
             }
+            if (data.type === "source") {
+              console.log("[EXULU] running a source job.", logMetadata(bullmqJob.name));
+              if (!data.source) {
+                throw new Error(`No source id set for source job.`);
+              }
+              if (!data.context) {
+                throw new Error(`No context id set for source job.`);
+              }
+              const context = contexts.find((c) => c.id === data.context);
+              if (!context) {
+                throw new Error(`Context ${data.context} not found in the registry.`);
+              }
+              const source = context.sources.find((s) => s.id === data.source);
+              if (!source) {
+                throw new Error(`Source ${data.source} not found in the context ${context.id}.`);
+              }
+              const result = await source.execute(data.inputs);
+              let jobs = [];
+              let items = [];
+              for (const item of result) {
+                const { item: createdItem, job } = await context.createItem(item, config, data.user, data.role);
+                if (job) {
+                  jobs.push(job);
+                  console.log(`[EXULU] Scheduled job through source update job for item ${createdItem.id} (Job ID: ${job})`, logMetadata(bullmqJob.name, {
+                    item: createdItem,
+                    job
+                  }));
+                }
+                if (createdItem.id) {
+                  items.push(createdItem.id);
+                  console.log(`[EXULU] created item through source update job ${createdItem.id}`, logMetadata(bullmqJob.name, {
+                    item: createdItem
+                  }));
+                }
+              }
+              return {
+                result,
+                metadata: {
+                  jobs,
+                  items
+                }
+              };
+            }
             throw new Error(`Invalid job type: ${data.type} for job ${bullmqJob.name}.`);
           } catch (error) {
             console.error(`[EXULU] job failed.`, error instanceof Error ? error.message : String(error));
@@ -8839,6 +8893,41 @@ var ExuluApp = class {
         if (queues2) {
           filteredQueues = filteredQueues.filter((q) => queues2.includes(q.queue.name));
         }
+        const sources = Object.values(this._contexts ?? {}).flatMap((context) => ({
+          ...context.sources,
+          context: context.id
+        }));
+        if (sources.length > 0) {
+          console.log("[EXULU] Creating ContextSource schedulers for", sources.length, "sources.");
+          for (const source of sources) {
+            const queue = await source.config?.queue;
+            if (queue) {
+              if (!source.config.schedule) {
+                throw new Error("Schedule is required for source when configuring a queue: " + source.name);
+              }
+              console.log("[EXULU] Creating ContextSource scheduler for", source.name, "in queue", queue.queue?.name);
+              await queue.queue?.upsertJobScheduler(source.id, {
+                pattern: source.config?.schedule
+              }, {
+                // default job data
+                name: `${source.id}-job`,
+                data: {
+                  source: source.id,
+                  context: source.context,
+                  type: "source"
+                },
+                opts: {
+                  backoff: {
+                    type: source.config.backoff?.type || "exponential",
+                    delay: source.config.backoff?.delay || 2e3
+                  },
+                  attempts: source.config.retries || 3,
+                  removeOnFail: 200
+                }
+              });
+            }
+          }
+        }
         return await createWorkers(
           this._agents,
           filteredQueues,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@exulu/backend",
   "author": "Qventu Bv.",
-  "version": "1.31.1",
+  "version": "1.32.0",
   "main": "./dist/index.js",
   "private": false,
   "publishConfig": {