@exulu/backend 0.1.9 → 0.2.1

package/dist/index.cjs

@@ -59,18 +59,19 @@ var redisServer = {
 var client = {};
 async function redisClient() {
   if (!redisServer.host || !redisServer.port) {
-    return {
-      client: null
-    };
+    return { client: null };
   }
   if (!client["exulu"]) {
-    const url = `redis://${redisServer.host}:${redisServer.port}`;
-    console.log(`[EXULU] connecting to redis.`);
-    client["exulu"] = (0, import_redis.createClient)({
-      // todo add password
-      url
-    });
-    await client["exulu"].connect();
+    try {
+      const url = `redis://${redisServer.host}:${redisServer.port}`;
+      client["exulu"] = (0, import_redis.createClient)({
+        url
+      });
+      await client["exulu"].connect();
+    } catch (error) {
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
+    }
   }
   return {
     client: client["exulu"]
@@ -113,25 +114,30 @@ var import_knex3 = require("pgvector/knex");
 var db = {};
 async function postgresClient() {
   if (!db["exulu"]) {
-    console.log("[EXULU] Initializing exulu database.");
-    console.log(process.env.POSTGRES_DB_HOST);
-    console.log(process.env.POSTGRES_DB_PORT);
-    console.log(process.env.POSTGRES_DB_USER);
-    console.log(process.env.POSTGRES_DB_PASSWORD);
-    console.log(process.env.POSTGRES_DB_SSL);
-    const knex = (0, import_knex.default)({
-      client: "pg",
-      connection: {
-        host: process.env.POSTGRES_DB_HOST,
-        port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
-        user: process.env.POSTGRES_DB_USER,
-        database: "exulu",
-        password: process.env.POSTGRES_DB_PASSWORD,
-        ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
-      }
-    });
-    await knex.schema.createExtensionIfNotExists("vector");
-    db["exulu"] = knex;
+    try {
+      console.log("[EXULU] Initializing exulu database.");
+      console.log(process.env.POSTGRES_DB_HOST);
+      console.log(process.env.POSTGRES_DB_PORT);
+      console.log(process.env.POSTGRES_DB_USER);
+      console.log(process.env.POSTGRES_DB_PASSWORD);
+      console.log(process.env.POSTGRES_DB_SSL);
+      const knex = (0, import_knex.default)({
+        client: "pg",
+        connection: {
+          host: process.env.POSTGRES_DB_HOST,
+          port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
+          user: process.env.POSTGRES_DB_USER,
+          database: "exulu",
+          password: process.env.POSTGRES_DB_PASSWORD,
+          ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+        }
+      });
+      await knex.schema.createExtensionIfNotExists("vector");
+      db["exulu"] = knex;
+    } catch (error) {
+      console.error("[EXULU] Error initializing exulu database.", error);
+      throw error;
+    }
   }
   return {
     db: db["exulu"]
@@ -547,8 +553,8 @@ var up = async function(knex) {
   }
 };
 var SALT_ROUNDS = 12;
-async function encryptApiKey(apiKey) {
-  const hash = await import_bcryptjs.default.hash(apiKey, SALT_ROUNDS);
+async function encryptApiKey(apikey) {
+  const hash = await import_bcryptjs.default.hash(apikey, SALT_ROUNDS);
   return hash;
 }
 var execute = async () => {
@@ -596,7 +602,7 @@ var execute = async () => {
       super_admin: true,
       createdAt: /* @__PURE__ */ new Date(),
       updatedAt: /* @__PURE__ */ new Date(),
-      type: "user",
+      type: "api",
       apikey: `${encryptedKey}${postFix}`,
       // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
       role: roleId
@@ -1424,30 +1430,38 @@ var import_express3 = require("express");
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -1546,10 +1560,10 @@ var authentication = async ({
         code: 401
       };
     }
-    const filtered = users.filter(({ apiKey, id }) => apiKey.includes(keyName));
+    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(keyName));
     for (const user of filtered) {
-      const lastSlashIndex = user.apiKey.lastIndexOf("/");
-      const compareValue = lastSlashIndex !== -1 ? user.apiKey.substring(0, lastSlashIndex) : user.apiKey;
+      const lastSlashIndex = user.apikey.lastIndexOf("/");
+      const compareValue = lastSlashIndex !== -1 ? user.apikey.substring(0, lastSlashIndex) : user.apikey;
       const isMatch = await import_bcryptjs2.default.compare(keyValue, compareValue);
       if (isMatch) {
         await db2.from("users").where({ id: user.id }).update({
@@ -1741,6 +1755,10 @@ var ExuluQueues = class {
     if (existing) {
       return existing;
     }
+    if (!redisServer.host?.length || !redisServer.port?.length) {
+      console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or workflow (look for ExuluQueues.use() ).`);
+      throw new Error(`[EXULU] no redis server configured.`);
+    }
     const newQueue = new import_bullmq5.Queue(`${name}`, { connection: redisServer });
     this.queues.push(newQueue);
     return newQueue;
@@ -2160,9 +2178,6 @@ var createUppyRoutes = async (app) => {
     return stsClient;
   }
   app.use(bodyParser.urlencoded({ extended: true }), bodyParser.json());
-  app.get("/", (req, res) => {
-    res.json("Exulu upload server.");
-  });
   app.get("/s3/list", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
     let authtoken = null;
@@ -2496,6 +2511,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+var import_utils = require("@apollo/utils.keyvaluecache");
 var Papa = require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2589,10 +2605,21 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     { route: "/items/export/:context", method: "GET", note: "Export items from context" },
     { route: "/graphql", method: "POST", note: "GraphQL endpoint" }
   );
-  await createRecurringJobs();
+  if (redisServer.host?.length && redisServer.port?.length) {
+    await createRecurringJobs();
+  } else {
+    console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
+  }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new import_server3.ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new import_server3.ApolloServer({
+    cache: new import_utils.InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     (0, import_cors.default)(),
@@ -2649,7 +2676,6 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
-  console.log("tools", tools);
   app.get("/tools", async (req, res) => {
     res.status(200).json(tools.map((tool) => ({
       id: tool.id,
@@ -2926,6 +2952,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2957,6 +2984,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3005,6 +3033,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3033,6 +3062,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3064,6 +3094,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3109,6 +3140,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3137,6 +3169,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3166,6 +3199,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3181,6 +3215,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3208,6 +3243,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3223,66 +3259,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3310,6 +3294,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         return;
       }
       if (agent.rateLimit) {
+        console.log("[EXULU] rate limiting agent.", agent.rateLimit);
         const limit = await rateLimiter(
           agent.rateLimit.name || agent.id,
           agent.rateLimit.rate_limit.time,
@@ -3382,6 +3367,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3447,7 +3433,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3547,11 +3537,18 @@ var bullmq = {
 var fs2 = __toESM(require("fs"), 1);
 var import_path = __toESM(require("path"), 1);
 var defaultLogsDir = import_path.default.join(process.cwd(), "logs");
-var redisConnection = new import_ioredis.default({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new import_ioredis.default({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);

package/dist/index.js

@@ -18,18 +18,19 @@ var redisServer = {
 var client = {};
 async function redisClient() {
   if (!redisServer.host || !redisServer.port) {
-    return {
-      client: null
-    };
+    return { client: null };
   }
   if (!client["exulu"]) {
-    const url = `redis://${redisServer.host}:${redisServer.port}`;
-    console.log(`[EXULU] connecting to redis.`);
-    client["exulu"] = createClient({
-      // todo add password
-      url
-    });
-    await client["exulu"].connect();
+    try {
+      const url = `redis://${redisServer.host}:${redisServer.port}`;
+      client["exulu"] = createClient({
+        url
+      });
+      await client["exulu"].connect();
+    } catch (error) {
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
+    }
   }
   return {
     client: client["exulu"]
@@ -72,25 +73,30 @@ import "pgvector/knex";
 var db = {};
 async function postgresClient() {
   if (!db["exulu"]) {
-    console.log("[EXULU] Initializing exulu database.");
-    console.log(process.env.POSTGRES_DB_HOST);
-    console.log(process.env.POSTGRES_DB_PORT);
-    console.log(process.env.POSTGRES_DB_USER);
-    console.log(process.env.POSTGRES_DB_PASSWORD);
-    console.log(process.env.POSTGRES_DB_SSL);
-    const knex = Knex({
-      client: "pg",
-      connection: {
-        host: process.env.POSTGRES_DB_HOST,
-        port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
-        user: process.env.POSTGRES_DB_USER,
-        database: "exulu",
-        password: process.env.POSTGRES_DB_PASSWORD,
-        ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
-      }
-    });
-    await knex.schema.createExtensionIfNotExists("vector");
-    db["exulu"] = knex;
+    try {
+      console.log("[EXULU] Initializing exulu database.");
+      console.log(process.env.POSTGRES_DB_HOST);
+      console.log(process.env.POSTGRES_DB_PORT);
+      console.log(process.env.POSTGRES_DB_USER);
+      console.log(process.env.POSTGRES_DB_PASSWORD);
+      console.log(process.env.POSTGRES_DB_SSL);
+      const knex = Knex({
+        client: "pg",
+        connection: {
+          host: process.env.POSTGRES_DB_HOST,
+          port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
+          user: process.env.POSTGRES_DB_USER,
+          database: "exulu",
+          password: process.env.POSTGRES_DB_PASSWORD,
+          ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+        }
+      });
+      await knex.schema.createExtensionIfNotExists("vector");
+      db["exulu"] = knex;
+    } catch (error) {
+      console.error("[EXULU] Error initializing exulu database.", error);
+      throw error;
+    }
   }
   return {
     db: db["exulu"]
@@ -506,8 +512,8 @@ var up = async function(knex) {
   }
 };
 var SALT_ROUNDS = 12;
-async function encryptApiKey(apiKey) {
-  const hash = await bcrypt.hash(apiKey, SALT_ROUNDS);
+async function encryptApiKey(apikey) {
+  const hash = await bcrypt.hash(apikey, SALT_ROUNDS);
   return hash;
 }
 var execute = async () => {
@@ -555,7 +561,7 @@ var execute = async () => {
       super_admin: true,
       createdAt: /* @__PURE__ */ new Date(),
       updatedAt: /* @__PURE__ */ new Date(),
-      type: "user",
+      type: "api",
       apikey: `${encryptedKey}${postFix}`,
       // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
       role: roleId
@@ -1383,30 +1389,38 @@ import "express";
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -1505,10 +1519,10 @@ var authentication = async ({
         code: 401
       };
     }
-    const filtered = users.filter(({ apiKey, id }) => apiKey.includes(keyName));
+    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(keyName));
     for (const user of filtered) {
-      const lastSlashIndex = user.apiKey.lastIndexOf("/");
-      const compareValue = lastSlashIndex !== -1 ? user.apiKey.substring(0, lastSlashIndex) : user.apiKey;
+      const lastSlashIndex = user.apikey.lastIndexOf("/");
+      const compareValue = lastSlashIndex !== -1 ? user.apikey.substring(0, lastSlashIndex) : user.apikey;
       const isMatch = await bcrypt2.compare(keyValue, compareValue);
       if (isMatch) {
         await db2.from("users").where({ id: user.id }).update({
@@ -1700,6 +1714,10 @@ var ExuluQueues = class {
     if (existing) {
       return existing;
     }
+    if (!redisServer.host?.length || !redisServer.port?.length) {
+      console.error(`[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or workflow (look for ExuluQueues.use() ).`);
+      throw new Error(`[EXULU] no redis server configured.`);
+    }
     const newQueue = new Queue3(`${name}`, { connection: redisServer });
     this.queues.push(newQueue);
     return newQueue;
@@ -2119,9 +2137,6 @@ var createUppyRoutes = async (app) => {
     return stsClient;
   }
   app.use(bodyParser.urlencoded({ extended: true }), bodyParser.json());
-  app.get("/", (req, res) => {
-    res.json("Exulu upload server.");
-  });
   app.get("/s3/list", async (req, res, next) => {
     const apikey = req.headers["exulu-api-key"] || null;
     let authtoken = null;
@@ -2455,6 +2470,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+import { InMemoryLRUCache } from "@apollo/utils.keyvaluecache";
 var Papa = __require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2548,10 +2564,21 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     { route: "/items/export/:context", method: "GET", note: "Export items from context" },
     { route: "/graphql", method: "POST", note: "GraphQL endpoint" }
   );
-  await createRecurringJobs();
+  if (redisServer.host?.length && redisServer.port?.length) {
+    await createRecurringJobs();
+  } else {
+    console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
+  }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new ApolloServer({
+    cache: new InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     cors(),
@@ -2608,7 +2635,6 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
-  console.log("tools", tools);
   app.get("/tools", async (req, res) => {
     res.status(200).json(tools.map((tool) => ({
       id: tool.id,
@@ -2885,6 +2911,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2916,6 +2943,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -2964,6 +2992,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -2992,6 +3021,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3023,6 +3053,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3068,6 +3099,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3096,6 +3128,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3125,6 +3158,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3140,6 +3174,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? zerialize(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3167,6 +3202,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3182,66 +3218,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3269,6 +3253,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
         return;
       }
       if (agent.rateLimit) {
+        console.log("[EXULU] rate limiting agent.", agent.rateLimit);
         const limit = await rateLimiter(
           agent.rateLimit.name || agent.id,
           agent.rateLimit.rate_limit.time,
@@ -3341,6 +3326,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3406,7 +3392,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3506,11 +3496,18 @@ var bullmq = {
 import * as fs2 from "fs";
 import path2 from "path";
 var defaultLogsDir = path2.join(process.cwd(), "logs");
-var redisConnection = new IORedis({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new IORedis({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@exulu/backend",
     "author": "Qventu Bv.",
-    "version": "0.1.9",
+    "version": "0.2.1",
     "main": "./dist/index.js",
     "private": false,
     "publishConfig": {