@exulu/backend 0.2.0 → 0.2.2

package/dist/index.cjs

@@ -58,25 +58,19 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisClient", redisServer);
-  if (!redisServer.host?.length || !redisServer.port?.length) {
-    console.error(`[EXULU] no redis server configured.`);
-    return {
-      client: null
-    };
+  if (!redisServer.host || !redisServer.port) {
+    return { client: null };
   }
   if (!client["exulu"]) {
     try {
       const url = `redis://${redisServer.host}:${redisServer.port}`;
-      console.log(`[EXULU] connecting to redis.`);
       client["exulu"] = (0, import_redis.createClient)({
-        // todo add password
         url
       });
       await client["exulu"].connect();
     } catch (error) {
-      console.error(`[EXULU] error connecting to redis.`, error);
-      throw error;
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
     }
   }
   return {
@@ -200,6 +194,10 @@ var usersSchema = {
       name: "apikey",
       type: "text"
     },
+    {
+      name: "last_used",
+      type: "date"
+    },
     {
       name: "role",
       type: "reference",
@@ -208,10 +206,6 @@ var usersSchema = {
         field: "id",
         onDelete: "CASCADE"
       }
-    },
-    {
-      name: "last_used",
-      type: "date"
     }
   ]
 };
@@ -410,7 +404,61 @@ var sanitizeName = (name) => {
 };
 
 // src/postgres/init-db.ts
+var import_bcryptjs2 = __toESM(require("bcryptjs"), 1);
+
+// src/auth/generate-key.ts
 var import_bcryptjs = __toESM(require("bcryptjs"), 1);
+var SALT_ROUNDS = 12;
+async function encryptApiKey(apikey) {
+  const hash = await import_bcryptjs.default.hash(apikey, SALT_ROUNDS);
+  return hash;
+}
+var generateApiKey = async (name, email) => {
+  const { db: db2 } = await postgresClient();
+  console.log("[EXULU] Inserting default user and admin role.");
+  const existingRole = await db2.from("roles").where({ name: "admin" }).first();
+  let roleId;
+  if (!existingRole) {
+    console.log("[EXULU] Creating default admin role.");
+    const role = await db2.from("roles").insert({
+      name: "admin",
+      is_admin: true,
+      agents: []
+    }).returning("id");
+    roleId = role[0].id;
+  } else {
+    roleId = existingRole.id;
+  }
+  const newKeyName = name;
+  const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
+  const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
+  const encryptedKey = await encryptApiKey(plainKey);
+  const existingApiUser = await db2.from("users").where({ email }).first();
+  if (!existingApiUser) {
+    console.log("[EXULU] Creating default api user.");
+    await db2.from("users").insert({
+      name,
+      email,
+      super_admin: true,
+      createdAt: /* @__PURE__ */ new Date(),
+      updatedAt: /* @__PURE__ */ new Date(),
+      type: "api",
+      apikey: `${encryptedKey}${postFix}`,
+      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
+      role: roleId
+    });
+    console.log("[EXULU] Default api user created. Key: ", `${plainKey}${postFix}`);
+  } else {
+    console.log("[EXULU] API user with that name already exists.");
+  }
+  console.log("[EXULU] Key generated, copy and use the plain key from here, you will not be able to access it again.");
+  console.log("[EXULU] Key: ", `${plainKey}${postFix}`);
+  return {
+    key: `${plainKey}${postFix}`
+  };
+};
+
+// src/postgres/init-db.ts
 var up = async function(knex) {
   if (!await knex.schema.hasTable("roles")) {
     await knex.schema.createTable("roles", (table) => {
@@ -515,6 +563,7 @@ var up = async function(knex) {
       table.timestamp("emailVerified", { useTz: true });
       table.text("image");
       for (const field of usersSchema.fields) {
+        console.log("[EXULU] field", field);
         const { type, name, references, default: defaultValue } = field;
         if (name === "id" || name === "name" || name === "email" || name === "emailVerified" || name === "image") {
           continue;
@@ -558,9 +607,9 @@ var up = async function(knex) {
     });
   }
 };
-var SALT_ROUNDS = 12;
-async function encryptApiKey(apikey) {
-  const hash = await import_bcryptjs.default.hash(apikey, SALT_ROUNDS);
+var SALT_ROUNDS2 = 12;
+async function encryptApiKey2(apikey) {
+  const hash = await import_bcryptjs2.default.hash(apikey, SALT_ROUNDS2);
   return hash;
 }
 var execute = async () => {
@@ -584,7 +633,7 @@ var execute = async () => {
   const newKeyName = "exulu_default_key";
   const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
   const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
-  const encryptedKey = await encryptApiKey(plainKey);
+  const encryptedKey = await encryptApiKey2(plainKey);
   const existingUser = await db2.from("users").where({ email: "admin@exulu.com" }).first();
   if (!existingUser) {
     console.log("[EXULU] Creating default admin user.");
@@ -599,23 +648,9 @@ var execute = async () => {
       role: roleId
     });
   }
-  const existingApiUser = await db2.from("users").where({ email: "api@exulu.com" }).first();
-  if (!existingApiUser) {
-    console.log("[EXULU] Creating default api user.");
-    await db2.from("users").insert({
-      name: "exulu",
-      email: "admin@exulu.com",
-      super_admin: true,
-      createdAt: /* @__PURE__ */ new Date(),
-      updatedAt: /* @__PURE__ */ new Date(),
-      type: "api",
-      apikey: `${encryptedKey}${postFix}`,
-      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
-      role: roleId
-    });
-  }
+  const { key } = await generateApiKey("exulu", "api@exulu.com");
   console.log("[EXULU] Database initialized.");
-  console.log("[EXULU] Default api key: ", `${encryptedKey}${postFix}`);
+  console.log("[EXULU] Default api key: ", `${key}`);
   return;
 };
 
@@ -1436,30 +1471,38 @@ var import_express3 = require("express");
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -1467,13 +1510,14 @@ var import_express = require("express");
 var import_jwt = require("next-auth/jwt");
 
 // src/auth/auth.ts
-var import_bcryptjs2 = __toESM(require("bcryptjs"), 1);
+var import_bcryptjs3 = __toESM(require("bcryptjs"), 1);
 var authentication = async ({
   apikey,
   authtoken,
   internalkey,
   db: db2
 }) => {
+  console.log("[EXULU] apikey", apikey);
   if (internalkey) {
     if (!process.env.INTERNAL_SECRET) {
       return {
@@ -1541,31 +1585,38 @@ var authentication = async ({
         code: 401
       };
     }
-    const keyParts = apikey.split("/");
-    const keyName = keyParts.pop();
-    const keyValue = keyParts[0];
-    if (!keyName) {
+    const request_key_parts = apikey.split("/");
+    const request_key_name = request_key_parts.pop();
+    const request_key_last_slash_index = apikey.lastIndexOf("/");
+    const request_key_compare_value = apikey.substring(0, request_key_last_slash_index);
+    if (!request_key_name) {
       return {
         error: true,
         message: "Provided api key does not include postfix with key name ({key}/{name}).",
         code: 401
       };
     }
-    if (!keyValue) {
+    if (!request_key_compare_value) {
       return {
         error: true,
         message: "Provided api key is not in the correct format.",
         code: 401
       };
     }
-    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(keyName));
+    console.log("[EXULU] users", users);
+    console.log("[EXULU] request_key_name", request_key_name);
+    console.log("[EXULU] request_key_compare_value", request_key_compare_value);
+    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(request_key_name));
+    console.log("[EXULU] filtered", filtered);
     for (const user of filtered) {
-      const lastSlashIndex = user.apikey.lastIndexOf("/");
-      const compareValue = lastSlashIndex !== -1 ? user.apikey.substring(0, lastSlashIndex) : user.apikey;
-      const isMatch = await import_bcryptjs2.default.compare(keyValue, compareValue);
+      const user_key_last_slash_index = user.apikey.lastIndexOf("/");
+      const user_key_compare_value = user.apikey.substring(0, user_key_last_slash_index);
+      console.log("[EXULU] user_key_compare_value", user_key_compare_value);
+      const isMatch = await import_bcryptjs3.default.compare(request_key_compare_value, user_key_compare_value);
+      console.log("[EXULU] isMatch", isMatch);
       if (isMatch) {
         await db2.from("users").where({ id: user.id }).update({
-          lastUsed: /* @__PURE__ */ new Date()
+          last_used: /* @__PURE__ */ new Date()
         }).returning("id");
         return {
           error: false,
@@ -1574,6 +1625,12 @@ var authentication = async ({
         };
       }
     }
+    console.log("[EXULU] No matching api key found.");
+    return {
+      error: true,
+      message: "No matching api key found.",
+      code: 401
+    };
   }
   return {
     error: true,
@@ -2509,6 +2566,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+var import_utils = require("@apollo/utils.keyvaluecache");
 var Papa = require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2608,8 +2666,15 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new import_server3.ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new import_server3.ApolloServer({
+    cache: new import_utils.InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     (0, import_cors.default)(),
@@ -2830,12 +2895,25 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
   });
   app.post("/items/:context", async (req, res) => {
     try {
+      console.log("[EXULU] post items");
       if (!req.params.context) {
         res.status(400).json({
           message: "Missing context in request."
         });
         return;
       }
+      if (!req.body) {
+        res.status(400).json({
+          message: "Missing body in request."
+        });
+        return;
+      }
+      if (!req.body.name) {
+        res.status(400).json({
+          message: "Missing in body of request."
+        });
+        return;
+      }
       const authenticationResult = await requestValidators.authenticate(req);
       if (!authenticationResult.user?.id) {
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
@@ -2942,6 +3020,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2973,6 +3052,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3021,6 +3101,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3049,6 +3130,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3080,6 +3162,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3125,6 +3208,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3153,6 +3237,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3182,6 +3267,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3197,6 +3283,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3224,6 +3311,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3239,66 +3327,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3399,6 +3435,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3464,7 +3501,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3564,11 +3605,18 @@ var bullmq = {
 var fs2 = __toESM(require("fs"), 1);
 var import_path = __toESM(require("path"), 1);
 var defaultLogsDir = import_path.default.join(process.cwd(), "logs");
-var redisConnection = new import_ioredis.default({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new import_ioredis.default({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);
@@ -3803,6 +3851,9 @@ var ExuluJobs = {
 var ExuluDatabase = {
   init: async () => {
     await execute();
+  },
+  generateApiKey: async (name, email) => {
+    return await generateApiKey(name, email);
   }
 };
 // Annotate the CommonJS export names for ESM import in node:

package/dist/index.d.cts

@@ -449,6 +449,9 @@ declare const ExuluJobs: {
 
 declare const ExuluDatabase: {
     init: () => Promise<void>;
+    generateApiKey: (name: string, email: string) => Promise<{
+        key: string;
+    }>;
 };
 
 export { type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, ExuluAgent, ExuluApp, authentication as ExuluAuthentication, ExuluContext, ExuluDatabase, ExuluEmbedder, ExuluJobs, queues as ExuluQueues, ExuluSource, ExuluTool, ExuluWorkflow, ExuluZodFileType };

package/dist/index.d.ts

@@ -449,6 +449,9 @@ declare const ExuluJobs: {
 
 declare const ExuluDatabase: {
     init: () => Promise<void>;
+    generateApiKey: (name: string, email: string) => Promise<{
+        key: string;
+    }>;
 };
 
 export { type STATISTICS_TYPE as EXULU_STATISTICS_TYPE, STATISTICS_TYPE_ENUM as EXULU_STATISTICS_TYPE_ENUM, ExuluAgent, ExuluApp, authentication as ExuluAuthentication, ExuluContext, ExuluDatabase, ExuluEmbedder, ExuluJobs, queues as ExuluQueues, ExuluSource, ExuluTool, ExuluWorkflow, ExuluZodFileType };

package/dist/index.js

@@ -17,25 +17,19 @@ var redisServer = {
 // src/redis/client.ts
 var client = {};
 async function redisClient() {
-  console.log("[EXULU] redisClient", redisServer);
-  if (!redisServer.host?.length || !redisServer.port?.length) {
-    console.error(`[EXULU] no redis server configured.`);
-    return {
-      client: null
-    };
+  if (!redisServer.host || !redisServer.port) {
+    return { client: null };
   }
   if (!client["exulu"]) {
     try {
       const url = `redis://${redisServer.host}:${redisServer.port}`;
-      console.log(`[EXULU] connecting to redis.`);
       client["exulu"] = createClient({
-        // todo add password
         url
       });
       await client["exulu"].connect();
     } catch (error) {
-      console.error(`[EXULU] error connecting to redis.`, error);
-      throw error;
+      console.error(`[EXULU] error connecting to redis:`, error);
+      return { client: null };
     }
   }
   return {
@@ -159,6 +153,10 @@ var usersSchema = {
       name: "apikey",
       type: "text"
     },
+    {
+      name: "last_used",
+      type: "date"
+    },
     {
       name: "role",
       type: "reference",
@@ -167,10 +165,6 @@ var usersSchema = {
         field: "id",
         onDelete: "CASCADE"
       }
-    },
-    {
-      name: "last_used",
-      type: "date"
     }
   ]
 };
@@ -369,7 +363,61 @@ var sanitizeName = (name) => {
 };
 
 // src/postgres/init-db.ts
+import bcrypt2 from "bcryptjs";
+
+// src/auth/generate-key.ts
 import bcrypt from "bcryptjs";
+var SALT_ROUNDS = 12;
+async function encryptApiKey(apikey) {
+  const hash = await bcrypt.hash(apikey, SALT_ROUNDS);
+  return hash;
+}
+var generateApiKey = async (name, email) => {
+  const { db: db2 } = await postgresClient();
+  console.log("[EXULU] Inserting default user and admin role.");
+  const existingRole = await db2.from("roles").where({ name: "admin" }).first();
+  let roleId;
+  if (!existingRole) {
+    console.log("[EXULU] Creating default admin role.");
+    const role = await db2.from("roles").insert({
+      name: "admin",
+      is_admin: true,
+      agents: []
+    }).returning("id");
+    roleId = role[0].id;
+  } else {
+    roleId = existingRole.id;
+  }
+  const newKeyName = name;
+  const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
+  const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
+  const encryptedKey = await encryptApiKey(plainKey);
+  const existingApiUser = await db2.from("users").where({ email }).first();
+  if (!existingApiUser) {
+    console.log("[EXULU] Creating default api user.");
+    await db2.from("users").insert({
+      name,
+      email,
+      super_admin: true,
+      createdAt: /* @__PURE__ */ new Date(),
+      updatedAt: /* @__PURE__ */ new Date(),
+      type: "api",
+      apikey: `${encryptedKey}${postFix}`,
+      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
+      role: roleId
+    });
+    console.log("[EXULU] Default api user created. Key: ", `${plainKey}${postFix}`);
+  } else {
+    console.log("[EXULU] API user with that name already exists.");
+  }
+  console.log("[EXULU] Key generated, copy and use the plain key from here, you will not be able to access it again.");
+  console.log("[EXULU] Key: ", `${plainKey}${postFix}`);
+  return {
+    key: `${plainKey}${postFix}`
+  };
+};
+
+// src/postgres/init-db.ts
 var up = async function(knex) {
   if (!await knex.schema.hasTable("roles")) {
     await knex.schema.createTable("roles", (table) => {
@@ -474,6 +522,7 @@ var up = async function(knex) {
       table.timestamp("emailVerified", { useTz: true });
       table.text("image");
       for (const field of usersSchema.fields) {
+        console.log("[EXULU] field", field);
         const { type, name, references, default: defaultValue } = field;
         if (name === "id" || name === "name" || name === "email" || name === "emailVerified" || name === "image") {
           continue;
@@ -517,9 +566,9 @@ var up = async function(knex) {
     });
   }
 };
-var SALT_ROUNDS = 12;
-async function encryptApiKey(apikey) {
-  const hash = await bcrypt.hash(apikey, SALT_ROUNDS);
+var SALT_ROUNDS2 = 12;
+async function encryptApiKey2(apikey) {
+  const hash = await bcrypt2.hash(apikey, SALT_ROUNDS2);
   return hash;
 }
 var execute = async () => {
@@ -543,7 +592,7 @@ var execute = async () => {
   const newKeyName = "exulu_default_key";
   const plainKey = `sk_${Math.random().toString(36).substring(2, 15)}_${Math.random().toString(36).substring(2, 15)}`;
   const postFix = `/${newKeyName.toLowerCase().trim().replaceAll(" ", "_")}`;
-  const encryptedKey = await encryptApiKey(plainKey);
+  const encryptedKey = await encryptApiKey2(plainKey);
   const existingUser = await db2.from("users").where({ email: "admin@exulu.com" }).first();
   if (!existingUser) {
     console.log("[EXULU] Creating default admin user.");
@@ -558,23 +607,9 @@ var execute = async () => {
       role: roleId
     });
   }
-  const existingApiUser = await db2.from("users").where({ email: "api@exulu.com" }).first();
-  if (!existingApiUser) {
-    console.log("[EXULU] Creating default api user.");
-    await db2.from("users").insert({
-      name: "exulu",
-      email: "admin@exulu.com",
-      super_admin: true,
-      createdAt: /* @__PURE__ */ new Date(),
-      updatedAt: /* @__PURE__ */ new Date(),
-      type: "api",
-      apikey: `${encryptedKey}${postFix}`,
-      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
-      role: roleId
-    });
-  }
+  const { key } = await generateApiKey("exulu", "api@exulu.com");
   console.log("[EXULU] Database initialized.");
-  console.log("[EXULU] Default api key: ", `${encryptedKey}${postFix}`);
+  console.log("[EXULU] Default api key: ", `${key}`);
   return;
 };
 
@@ -1395,30 +1430,38 @@ import "express";
 
 // src/registry/rate-limiter.ts
 var rateLimiter = async (key, windowSeconds, limit, points) => {
-  const { client: client2 } = await redisClient();
-  if (!client2) {
+  try {
+    const { client: client2 } = await redisClient();
+    if (!client2) {
+      console.warn("[EXULU] Rate limiting disabled - Redis not available");
+      return {
+        status: true,
+        retryAfter: null
+      };
+    }
+    const redisKey = `exulu/${key}`;
+    const current = await client2.incrBy(redisKey, points);
+    if (current === points) {
+      await client2.expire(redisKey, windowSeconds);
+    }
+    if (current > limit) {
+      const ttl = await client2.ttl(redisKey);
+      return {
+        status: false,
+        retryAfter: ttl
+      };
+    }
     return {
-      status: false,
-      retryAfter: 10
-      // 10 seconds
+      status: true,
+      retryAfter: null
     };
-  }
-  const redisKey = `exulu/${key}`;
-  const current = await client2.incrBy(redisKey, points);
-  if (current === points) {
-    await client2.expire(redisKey, windowSeconds);
-  }
-  if (current > limit) {
-    const ttl = await client2.ttl(redisKey);
+  } catch (error) {
+    console.error("[EXULU] Rate limiting error:", error);
     return {
-      status: false,
-      retryAfter: ttl
+      status: true,
+      retryAfter: null
     };
   }
-  return {
-    status: true,
-    retryAfter: null
-  };
 };
 
 // src/registry/route-validators/index.ts
@@ -1426,13 +1469,14 @@ import "express";
 import { getToken } from "next-auth/jwt";
 
 // src/auth/auth.ts
-import bcrypt2 from "bcryptjs";
+import bcrypt3 from "bcryptjs";
 var authentication = async ({
   apikey,
   authtoken,
   internalkey,
   db: db2
 }) => {
+  console.log("[EXULU] apikey", apikey);
   if (internalkey) {
     if (!process.env.INTERNAL_SECRET) {
       return {
@@ -1500,31 +1544,38 @@ var authentication = async ({
         code: 401
       };
     }
-    const keyParts = apikey.split("/");
-    const keyName = keyParts.pop();
-    const keyValue = keyParts[0];
-    if (!keyName) {
+    const request_key_parts = apikey.split("/");
+    const request_key_name = request_key_parts.pop();
+    const request_key_last_slash_index = apikey.lastIndexOf("/");
+    const request_key_compare_value = apikey.substring(0, request_key_last_slash_index);
+    if (!request_key_name) {
       return {
         error: true,
         message: "Provided api key does not include postfix with key name ({key}/{name}).",
         code: 401
       };
     }
-    if (!keyValue) {
+    if (!request_key_compare_value) {
       return {
         error: true,
         message: "Provided api key is not in the correct format.",
         code: 401
       };
     }
-    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(keyName));
+    console.log("[EXULU] users", users);
+    console.log("[EXULU] request_key_name", request_key_name);
+    console.log("[EXULU] request_key_compare_value", request_key_compare_value);
+    const filtered = users.filter(({ apikey: apikey2, id }) => apikey2.includes(request_key_name));
+    console.log("[EXULU] filtered", filtered);
     for (const user of filtered) {
-      const lastSlashIndex = user.apikey.lastIndexOf("/");
-      const compareValue = lastSlashIndex !== -1 ? user.apikey.substring(0, lastSlashIndex) : user.apikey;
-      const isMatch = await bcrypt2.compare(keyValue, compareValue);
+      const user_key_last_slash_index = user.apikey.lastIndexOf("/");
+      const user_key_compare_value = user.apikey.substring(0, user_key_last_slash_index);
+      console.log("[EXULU] user_key_compare_value", user_key_compare_value);
+      const isMatch = await bcrypt3.compare(request_key_compare_value, user_key_compare_value);
+      console.log("[EXULU] isMatch", isMatch);
       if (isMatch) {
         await db2.from("users").where({ id: user.id }).update({
-          lastUsed: /* @__PURE__ */ new Date()
+          last_used: /* @__PURE__ */ new Date()
         }).returning("id");
         return {
           error: false,
@@ -1533,6 +1584,12 @@ var authentication = async ({
         };
       }
     }
+    console.log("[EXULU] No matching api key found.");
+    return {
+      error: true,
+      message: "No matching api key found.",
+      code: 401
+    };
   }
   return {
     error: true,
@@ -2468,6 +2525,7 @@ var createUppyRoutes = async (app) => {
 };
 
 // src/registry/routes.ts
+import { InMemoryLRUCache } from "@apollo/utils.keyvaluecache";
 var Papa = __require("papaparse");
 var global_queues = {
   logs_cleaner: "logs-cleaner"
@@ -2567,8 +2625,15 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     console.log("===========================", "[EXULU] no redis server configured, not setting up recurring jobs.", "===========================");
   }
   const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
-  const server = new ApolloServer({ schema, introspection: true });
+  console.log("[EXULU] graphql server");
+  const server = new ApolloServer({
+    cache: new InMemoryLRUCache(),
+    schema,
+    introspection: true
+  });
+  console.log("[EXULU] starting graphql server");
   await server.start();
+  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     cors(),
@@ -2789,12 +2854,25 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
   });
   app.post("/items/:context", async (req, res) => {
     try {
+      console.log("[EXULU] post items");
       if (!req.params.context) {
         res.status(400).json({
           message: "Missing context in request."
         });
         return;
       }
+      if (!req.body) {
+        res.status(400).json({
+          message: "Missing body in request."
+        });
+        return;
+      }
+      if (!req.body.name) {
+        res.status(400).json({
+          message: "Missing in body of request."
+        });
+        return;
+      }
       const authenticationResult = await requestValidators.authenticate(req);
       if (!authenticationResult.user?.id) {
         res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
@@ -2901,6 +2979,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     method: "DELETE",
     note: `Delete specific embedding for a context.`
   });
+  console.log("[EXULU] delete embedding by id");
   app.delete(`items/:context/:id`, async (req, res) => {
     const id = req.params.id;
     if (!req.params.context) {
@@ -2932,6 +3011,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       message: "Embedding deleted."
     });
   });
+  console.log("[EXULU] statistics timeseries");
   app.post("/statistics/timeseries", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -2980,6 +3060,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] statistics totals");
   app.post("/statistics/totals", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3008,6 +3089,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] contexts statistics");
   app.get("/contexts/statistics", async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3039,6 +3121,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] context by id");
   app.get(`/contexts/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3084,6 +3167,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       // todo
     });
   });
+  console.log("[EXULU] items export by context");
   app.get(`/items/export/:context`, async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -3112,6 +3196,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
     const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
     res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
   });
+  console.log("[EXULU] contexts get list");
   app.get(`/contexts`, async (req, res) => {
     console.log("contexts!!");
     const authenticationResult = await requestValidators.authenticate(req);
@@ -3141,6 +3226,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }))
     })));
   });
+  console.log("[EXULU] workflows get list");
   app.get(`/workflows`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3156,6 +3242,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       inputSchema: workflow.inputSchema ? zerialize(workflow.inputSchema) : null
     })));
   });
+  console.log("[EXULU] workflow by id");
   app.get(`/workflows/:id`, async (req, res) => {
     const authenticationResult = await requestValidators.authenticate(req);
     if (!authenticationResult.user?.id) {
@@ -3183,6 +3270,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       workflow: void 0
     });
   });
+  console.log("[EXULU] contexts");
   contexts.forEach((context) => {
     const sources = context.sources.get();
     if (!Array.isArray(sources)) {
@@ -3198,66 +3286,14 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
             note: `Webhook updater for ${context.name}`
           });
           app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
-            const { context: id } = req.params;
-            if (!id) {
-              res.status(400).json({
-                message: "Missing context id in request."
-              });
-              return;
-            }
-            const context2 = contexts.find((context3) => context3.id === id);
-            if (!context2) {
-              res.status(400).json({
-                message: `Context for provided id: ${id} not found.`
-              });
-              return;
-            }
-            if (!context2.embedder.queue) {
-              res.status(500).json({ detail: "No queue set for embedder." });
-              return;
-            }
-            const authenticationResult = await requestValidators.authenticate(req);
-            if (!authenticationResult.user?.id) {
-              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
-              return;
-            }
-            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
-            if (requestValidationResult.error) {
-              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
-              return;
-            }
-            const documents = await updater.fn(req.body.configuration);
-            const batches = [];
-            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
-              batches.push(documents.slice(i, i + context2.embedder.batchSize));
-            }
-            let promises2 = [];
-            if (batches.length > 0) {
-              promises2 = batches.map((documents2) => {
-                return bullmqDecorator({
-                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
-                  type: "embedder",
-                  embedder: context2.embedder.id,
-                  updater: updater.id,
-                  context: context2.id,
-                  trigger: updater.type,
-                  source: source.id,
-                  inputs: req.body.inputs,
-                  ...updater.configuration && { configuration: req.body.configuration },
-                  documents: documents2,
-                  queue: context2.embedder.queue,
-                  user: authenticationResult.user.id
-                });
-              });
-            }
-            const jobs = await Promise.all(promises2);
-            res.status(200).json(jobs);
+            res.status(200).json([]);
             return;
           });
         }
       });
     });
   });
+  console.log("[EXULU] agents");
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
@@ -3358,6 +3394,7 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       }
     });
   });
+  console.log("[EXULU] workflows");
   workflows.forEach((workflow) => {
     routeLogs.push({
       route: workflow.slug,
@@ -3423,7 +3460,11 @@ var createExpressRoutes = async (app, agents, embedders, tools, workflows, conte
       return;
     });
   });
-  await createUppyRoutes(app);
+  if (process.env.COMPANION_S3_REGION && process.env.COMPANION_S3_KEY && process.env.COMPANION_S3_SECRET) {
+    await createUppyRoutes(app);
+  } else {
+    console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
+  }
   console.log("Routes:");
   console.table(routeLogs);
 };
@@ -3523,11 +3564,18 @@ var bullmq = {
 import * as fs2 from "fs";
 import path2 from "path";
 var defaultLogsDir = path2.join(process.cwd(), "logs");
-var redisConnection = new IORedis({
-  ...redisServer,
-  maxRetriesPerRequest: null
-});
+var redisConnection;
 var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  if (!redisServer.host || !redisServer.port) {
+    console.error("[EXULU] you are trying to start workers, but no redis server is configured in the environment.");
+    throw new Error("No redis server configured in the environment, so cannot start workers.");
+  }
+  if (!redisConnection) {
+    redisConnection = new IORedis({
+      ...redisServer,
+      maxRetriesPerRequest: null
+    });
+  }
   const logsDir = _logsDir || defaultLogsDir;
   const workers = queues2.map((queue) => {
     console.log(`[EXULU] creating worker for queue ${queue}.`);
@@ -3762,6 +3810,9 @@ var ExuluJobs = {
 var ExuluDatabase = {
   init: async () => {
     await execute();
+  },
+  generateApiKey: async (name, email) => {
+    return await generateApiKey(name, email);
   }
 };
 export {

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@exulu/backend",
     "author": "Qventu Bv.",
-    "version": "0.2.0",
+    "version": "0.2.2",
     "main": "./dist/index.js",
     "private": false,
     "publishConfig": {