@exulu/backend 0.2.0 → 0.2.1

package/dist/index.cjs

@@ -58,25 +58,19 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisClient", redisServer);
-  if (!redisServer.host?.length || !redisServer.port?.length) {
-    console.error(`[EXULU] no redis server configured.`);
-    return {
-      client: null
-    };
+  if (!redisServer.host || !redisServer.port) {
+    return { client: null };
   }
   if (!client["exulu"]) {
     try {
       const url = `redis://${redisServer.host}:${redisServer.port}`;
-      console.log(`[EXULU] connecting to redis.`);
       client["exulu"] = (0, import_redis.createClient)({
-        // todo add password
         url
       });
       await client["exulu"].connect();
     } catch (error) {
-      console.error(`[EXULU] error connecting to redis.`, error);
-      throw error;
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
     }
   }
   return {
@@ -1436,30 +1430,38 @@ var import_express3 = require("express");
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -2509,6 +2511,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+var import_utils = require("@apollo/utils.keyvaluecache");
 var Papa = require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2608,8 +2611,15 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new import_server3.ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new import_server3.ApolloServer({
+    cache: new import_utils.InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     (0, import_cors.default)(),
@@ -2942,6 +2952,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2973,6 +2984,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3021,6 +3033,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3049,6 +3062,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3080,6 +3094,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3125,6 +3140,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3153,6 +3169,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3182,6 +3199,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3197,6 +3215,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3224,6 +3243,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3239,66 +3259,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3399,6 +3367,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3464,7 +3433,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3564,11 +3537,18 @@ var bullmq = {
 var fs2 = __toESM(require("fs"), 1);
 var import_path = __toESM(require("path"), 1);
 var defaultLogsDir = import_path.default.join(process.cwd(), "logs");
-var redisConnection = new import_ioredis.default({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new import_ioredis.default({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);

package/dist/index.js

@@ -17,25 +17,19 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisClient", redisServer);
-  if (!redisServer.host?.length || !redisServer.port?.length) {
-    console.error(`[EXULU] no redis server configured.`);
-    return {
-      client: null
-    };
+  if (!redisServer.host || !redisServer.port) {
+    return { client: null };
   }
   if (!client["exulu"]) {
     try {
       const url = `redis://${redisServer.host}:${redisServer.port}`;
-      console.log(`[EXULU] connecting to redis.`);
       client["exulu"] = createClient({
-        // todo add password
         url
       });
       await client["exulu"].connect();
     } catch (error) {
-      console.error(`[EXULU] error connecting to redis.`, error);
-      throw error;
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
     }
   }
   return {
@@ -1395,30 +1389,38 @@ import "express";
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -2468,6 +2470,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+import { InMemoryLRUCache } from "@apollo/utils.keyvaluecache";
 var Papa = __require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2567,8 +2570,15 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new ApolloServer({
+    cache: new InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     cors(),
@@ -2901,6 +2911,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2932,6 +2943,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -2980,6 +2992,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3008,6 +3021,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3039,6 +3053,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3084,6 +3099,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3112,6 +3128,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3141,6 +3158,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3156,6 +3174,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? zerialize(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3183,6 +3202,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3198,66 +3218,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3358,6 +3326,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3423,7 +3392,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3523,11 +3496,18 @@ var bullmq = {
 import * as fs2 from "fs";
 import path2 from "path";
 var defaultLogsDir = path2.join(process.cwd(), "logs");
-var redisConnection = new IORedis({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new IORedis({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@exulu/backend",
     "author": "Qventu Bv.",
-    "version": "0.2.0",
+    "version": "0.2.1",
     "main": "./dist/index.js",
     "private": false,
     "publishConfig": {