@exulu/backend 0.1.6

package/dist/index.cjs

@@ -0,0 +1,3763 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  EXULU_STATISTICS_TYPE_ENUM: () => STATISTICS_TYPE_ENUM,
+  ExuluAgent: () => ExuluAgent,
+  ExuluApp: () => ExuluApp,
+  ExuluAuthentication: () => authentication,
+  ExuluContext: () => ExuluContext,
+  ExuluDatabase: () => ExuluDatabase,
+  ExuluEmbedder: () => ExuluEmbedder,
+  ExuluJobs: () => ExuluJobs,
+  ExuluQueues: () => queues,
+  ExuluSource: () => ExuluSource,
+  ExuluTool: () => ExuluTool,
+  ExuluWorkflow: () => ExuluWorkflow,
+  ExuluZodFileType: () => ExuluZodFileType
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/redis/client.ts
+var import_redis = require("redis");
+
+// src/bullmq/server.ts
+var redisServer = {
+  host: `${process.env.REDIS_HOST}`,
+  port: process.env.REDIS_PORT
+};
+
+// src/redis/client.ts
+var redisClient = null;
+(async () => {
+  if (!redisClient) {
+    const url = `redis://${redisServer.host}:${redisServer.port}`;
+    console.log(`[EXULU] connecting to redis.`);
+    redisClient = (0, import_redis.createClient)({
+      // todo add password
+      url
+    });
+    await redisClient.connect();
+  }
+})();
+
+// src/bullmq/validators.ts
+var validateJob = (job) => {
+  if (!job.data) {
+    throw new Error(`Missing job data for job ${job.id}.`);
+  }
+  if (!job.data.type) {
+    throw new Error(`Missing property "type" in data for job ${job.id}.`);
+  }
+  if (!job.data.function) {
+    throw new Error(`Missing property "function" in data for job ${job.id}.`);
+  }
+  if (!job.data.inputs) {
+    throw new Error(`Missing property "inputs" in data for job ${job.id}.`);
+  }
+  if (job.data.type !== "embedder" && job.data.type !== "workflow") {
+    throw new Error(`Property "type" in data for job ${job.id} must be of value "embedder" or "agent".`);
+  }
+  if (job.data.type === "workflow" && job.data.function !== "execute") {
+    throw new Error(`Property "function" in data for job ${job.id} must be of value "execute" when using type "workflow".`);
+  }
+  if (job.data.type === "embedder" && job.data.function !== "upsert" && job.data.function !== "delete" && job.data.function !== "retrieve") {
+    throw new Error(`Property "function" in data for job ${job.id} must be of value "upsert", "delete" or "retrieve" when using type "embedder".`);
+  }
+  if (!job.data.id) {
+    throw new Error(`Property "id" in data for job ${job.id} missing.`);
+  }
+  return job;
+};
+
+// src/postgres/client.ts
+var import_knex = __toESM(require("knex"), 1);
+var import_knex2 = require("knex");
+var import_knex3 = require("pgvector/knex");
+var db = {};
+async function postgresClient() {
+  if (!db["exulu"]) {
+    const knex = (0, import_knex.default)({
+      client: "pg",
+      connection: {
+        host: process.env.POSTGRES_DB_HOST,
+        port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
+        user: process.env.POSTGRES_DB_USER,
+        database: "exulu",
+        password: process.env.POSTGRES_DB_PASSWORD,
+        ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+      }
+    });
+    await knex.schema.createExtensionIfNotExists("vector");
+    db["exulu"] = knex;
+  }
+  return {
+    db: db["exulu"]
+  };
+}
+
+// src/postgres/core-schema.ts
+var usersSchema = {
+  name: {
+    plural: "users",
+    singular: "user"
+  },
+  fields: [
+    {
+      name: "firstname",
+      type: "text"
+    },
+    {
+      name: "lastname",
+      type: "text"
+    },
+    {
+      name: "email",
+      type: "text",
+      index: true
+    },
+    {
+      name: "temporary_token",
+      type: "text"
+    },
+    {
+      name: "type",
+      type: "text",
+      index: true
+    },
+    {
+      name: "profile_image",
+      type: "text"
+    },
+    {
+      name: "super_admin",
+      type: "boolean",
+      default: false
+    },
+    {
+      name: "status",
+      type: "text"
+    },
+    {
+      name: "emailVerified",
+      type: "text"
+    },
+    {
+      name: "apikey",
+      type: "text"
+    },
+    {
+      name: "role",
+      type: "reference",
+      references: {
+        table: "roles",
+        field: "id",
+        onDelete: "CASCADE"
+      }
+    },
+    {
+      name: "last_used",
+      type: "date"
+    }
+  ]
+};
+var rolesSchema = {
+  name: {
+    plural: "roles",
+    singular: "role"
+  },
+  fields: [
+    {
+      name: "name",
+      type: "text"
+    },
+    {
+      name: "is_admin",
+      type: "boolean",
+      default: false
+    },
+    {
+      name: "agents",
+      type: "json"
+    }
+  ]
+};
+var statisticsSchema = {
+  name: {
+    plural: "statistics",
+    singular: "statistic"
+  },
+  fields: [
+    {
+      name: "name",
+      type: "text"
+    },
+    {
+      name: "label",
+      type: "text"
+    },
+    {
+      name: "type",
+      type: "text"
+    },
+    {
+      name: "total",
+      type: "number"
+    },
+    {
+      name: "timeseries",
+      type: "json"
+    }
+  ]
+};
+var jobsSchema = {
+  name: {
+    plural: "jobs",
+    singular: "job"
+  },
+  fields: [
+    {
+      name: "redis",
+      type: "text"
+    },
+    {
+      name: "session",
+      type: "text"
+    },
+    {
+      name: "status",
+      type: "text"
+    },
+    {
+      name: "type",
+      type: "text"
+    },
+    {
+      name: "result",
+      type: "text"
+    },
+    {
+      name: "name",
+      type: "text"
+    },
+    {
+      name: "agent",
+      type: "text"
+    },
+    {
+      name: "user",
+      type: "text"
+    },
+    {
+      name: "item",
+      type: "text"
+    },
+    {
+      name: "inputs",
+      type: "json"
+    },
+    {
+      name: "finished_at",
+      type: "date"
+    },
+    {
+      name: "duration",
+      type: "number"
+    }
+  ]
+};
+var agentsSchema = {
+  name: {
+    plural: "agents",
+    singular: "agent"
+  },
+  fields: [
+    {
+      name: "name",
+      type: "text"
+    },
+    {
+      name: "description",
+      type: "text"
+    },
+    {
+      name: "extensions",
+      type: "json"
+    },
+    {
+      name: "backend",
+      type: "text"
+    },
+    {
+      name: "type",
+      type: "text"
+    },
+    {
+      name: "active",
+      type: "boolean",
+      default: false
+    },
+    {
+      name: "public",
+      type: "boolean",
+      default: false
+    },
+    {
+      name: "tools",
+      type: "json"
+    }
+  ]
+};
+
+// src/registry/utils/map-types.ts
+var mapType = (t, type, name, defaultValue) => {
+  if (type === "text") {
+    t.string(name, 255);
+    return;
+  }
+  if (type === "longText") {
+    t.text(name);
+    return;
+  }
+  if (type === "shortText") {
+    t.string(name, 100);
+    return;
+  }
+  if (type === "number") {
+    t.float(name);
+    return;
+  }
+  if (type === "boolean") {
+    t.boolean(name).defaultTo(defaultValue || false);
+    return;
+  }
+  if (type === "code") {
+    t.text(name);
+    return;
+  }
+  if (type === "json") {
+    t.jsonb(name);
+    return;
+  }
+  if (type === "date") {
+    t.date(name);
+    return;
+  }
+  if (type === "uuid") {
+    t.uuid(name);
+    return;
+  }
+  throw new Error("Invalid type: " + type);
+};
+
+// src/registry/utils/sanitize-name.ts
+var sanitizeName = (name) => {
+  return name.toLowerCase().replace(/ /g, "_");
+};
+
+// src/postgres/init-db.ts
+var up = async function(knex) {
+  if (!await knex.schema.hasTable("roles")) {
+    await knex.schema.createTable("roles", (table) => {
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.date("createdAt").defaultTo(knex.fn.now());
+      table.date("updatedAt").defaultTo(knex.fn.now());
+      for (const field of rolesSchema.fields) {
+        const { type, name, references, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        if (type === "reference") {
+          if (!references) {
+            throw new Error("Field with type reference must have a reference definition.");
+          }
+          table.uuid(name).references(references.field).inTable(references.table);
+          return;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("statistics")) {
+    await knex.schema.createTable("statistics", (table) => {
+      table.uuid("id").primary().defaultTo(knex.fn.uuid());
+      table.date("createdAt").defaultTo(knex.fn.now());
+      table.date("updatedAt").defaultTo(knex.fn.now());
+      for (const field of statisticsSchema.fields) {
+        const { type, name, references, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        if (type === "reference") {
+          if (!references) {
+            throw new Error("Field with type reference must have a reference definition.");
+          }
+          table.uuid(name).references(references.field).inTable(references.table);
+          return;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("jobs")) {
+    await knex.schema.createTable("jobs", (table) => {
+      table.increments("id").primary();
+      table.date("createdAt").defaultTo(knex.fn.now());
+      table.date("updatedAt").defaultTo(knex.fn.now());
+      for (const field of jobsSchema.fields) {
+        const { type, name, references, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        if (type === "reference") {
+          if (!references) {
+            throw new Error("Field with type reference must have a reference definition.");
+          }
+          table.uuid(name).references(references.field).inTable(references.table);
+          return;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("agents")) {
+    await knex.schema.createTable("agents", (table) => {
+      table.increments("id").primary();
+      table.date("createdAt").defaultTo(knex.fn.now());
+      table.date("updatedAt").defaultTo(knex.fn.now());
+      for (const field of agentsSchema.fields) {
+        const { type, name, references, default: defaultValue } = field;
+        if (!type || !name) {
+          continue;
+        }
+        if (type === "reference") {
+          if (!references) {
+            throw new Error("Field with type reference must have a reference definition.");
+          }
+          table.uuid(name).references(references.field).inTable(references.table);
+          return;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("verification_token")) {
+    await knex.schema.createTable("verification_token", (table) => {
+      table.text("identifier").notNullable();
+      table.timestamp("expires", { useTz: true }).notNullable();
+      table.text("token").notNullable();
+      table.primary(["identifier", "token"]);
+    });
+  }
+  if (!await knex.schema.hasTable("users")) {
+    await knex.schema.createTable("users", (table) => {
+      table.increments("id").primary();
+      table.date("createdAt").defaultTo(knex.fn.now());
+      table.date("updatedAt").defaultTo(knex.fn.now());
+      table.string("name", 255);
+      table.string("password", 255);
+      table.string("email", 255);
+      table.timestamp("emailVerified", { useTz: true });
+      table.text("image");
+      for (const field of usersSchema.fields) {
+        const { type, name, references, default: defaultValue } = field;
+        if (name === "id" || name === "name" || name === "email" || name === "emailVerified" || name === "image") {
+          continue;
+        }
+        if (!type || !name) {
+          continue;
+        }
+        if (type === "reference") {
+          if (!references) {
+            throw new Error("Field with type reference must have a reference definition.");
+          }
+          table.uuid(name).references(references.field).inTable(references.table);
+          return;
+        }
+        mapType(table, type, sanitizeName(name), defaultValue);
+      }
+    });
+  }
+  if (!await knex.schema.hasTable("accounts")) {
+    await knex.schema.createTable("accounts", (table) => {
+      table.increments("id").primary();
+      table.integer("userId").notNullable();
+      table.string("type", 255).notNullable();
+      table.string("provider", 255).notNullable();
+      table.string("providerAccountId", 255).notNullable();
+      table.text("refresh_token");
+      table.text("access_token");
+      table.bigInteger("expires_at");
+      table.text("id_token");
+      table.text("scope");
+      table.text("session_state");
+      table.text("token_type");
+    });
+  }
+  if (!await knex.schema.hasTable("sessions")) {
+    await knex.schema.createTable("sessions", (table) => {
+      table.increments("id").primary();
+      table.integer("userId").notNullable();
+      table.timestamp("expires", { useTz: true }).notNullable();
+      table.string("sessionToken", 255).notNullable();
+    });
+  }
+};
+var execute = async () => {
+  console.log("[EXULU] Initializing database.");
+  const { db: db2 } = await postgresClient();
+  await up(db2);
+  console.log("[EXULU] Inserting default user and admin role.");
+  const existingRole = await db2.from("roles").where({ name: "admin" }).first();
+  let roleId;
+  if (!existingRole) {
+    console.log("[EXULU] Creating default admin role.");
+    const role = await db2.from("roles").insert({
+      name: "admin",
+      is_admin: true,
+      agents: []
+    }).returning("id");
+    roleId = role[0].id;
+  } else {
+    roleId = existingRole.id;
+  }
+  const existingUser = await db2.from("users").where({ email: "admin@exulu.com" }).first();
+  if (!existingUser) {
+    console.log("[EXULU] Creating default admin user.");
+    await db2.from("users").insert({
+      name: "exulu",
+      email: "admin@exulu.com",
+      super_admin: true,
+      createdAt: /* @__PURE__ */ new Date(),
+      updatedAt: /* @__PURE__ */ new Date(),
+      // password: "admin", todo add this again when we implement password auth / encryption as alternative to OTP
+      role: roleId
+    });
+  }
+  console.log("[EXULU] Database initialized.");
+  return;
+};
+
+// src/registry/classes.ts
+var import_zod = require("zod");
+var import_bullmq2 = require("bullmq");
+var import_core = require("@mastra/core");
+var import_zod2 = require("zod");
+var fs = __toESM(require("fs"), 1);
+var path = __toESM(require("path"), 1);
+var import_bullmq3 = require("bullmq");
+var import_memory = require("@mastra/memory");
+var import_pg = require("@mastra/pg");
+
+// types/enums/statistics.ts
+var STATISTICS_TYPE_ENUM = {
+  CONTEXT_RETRIEVE: "context.retrieve",
+  SOURCE_UPDATE: "source.update",
+  EMBEDDER_UPSERT: "embedder.upsert",
+  EMBEDDER_GENERATE: "embedder.generate",
+  EMBEDDER_DELETE: "embedder.delete",
+  WORKFLOW_RUN: "workflow.run",
+  CONTEXT_UPSERT: "context.upsert",
+  TOOL_CALL: "tool.call",
+  AGENT_RUN: "agent.run"
+};
+
+// src/registry/classes.ts
+var import_knex4 = __toESM(require("pgvector/knex"), 1);
+
+// src/registry/decoraters/bullmq.ts
+var import_bullmq = require("bullmq");
+var import_uuid = require("uuid");
+var bullmqDecorator = async ({
+  label,
+  type,
+  workflow,
+  embedder,
+  inputs,
+  queue,
+  user,
+  agent,
+  session,
+  configuration,
+  updater,
+  context,
+  source,
+  documents,
+  trigger,
+  item
+}) => {
+  const redisId = (0, import_uuid.v4)();
+  const job = await queue.add(
+    `${embedder || workflow}`,
+    {
+      type: `${type}`,
+      ...embedder && { embedder },
+      ...workflow && { workflow },
+      ...configuration && { configuration },
+      ...updater && { updater },
+      ...context && { context },
+      ...source && { source },
+      ...documents && { documents },
+      ...trigger && { trigger },
+      ...item && { item },
+      agent,
+      user,
+      inputs,
+      session
+    },
+    {
+      jobId: redisId
+    }
+  );
+  const { db: db2 } = await postgresClient();
+  const now = /* @__PURE__ */ new Date();
+  console.log("[EXULU] scheduling new job", inputs);
+  const insertData = {
+    name: `${label}`,
+    redis: job.id,
+    status: "waiting",
+    type,
+    inputs,
+    agent,
+    item,
+    createdAt: now,
+    updatedAt: now,
+    user,
+    session,
+    ...embedder && { embedder },
+    ...workflow && { workflow },
+    ...configuration && { configuration },
+    ...updater && { updater },
+    ...context && { context },
+    ...source && { source },
+    ...documents && { documents: documents.map((doc2) => doc2.id) },
+    ...trigger && { trigger }
+  };
+  await db2("jobs").insert(insertData).onConflict("redis").merge({
+    ...insertData,
+    updatedAt: now
+    // Only updatedAt changes on updates
+  });
+  const doc = await db2.from("jobs").where({ redis: job.id }).first();
+  if (!doc?.id) {
+    throw new Error("Failed to get job ID after insert/update");
+  }
+  console.log("[EXULU] created job", doc?.id);
+  return {
+    ...job,
+    id: doc?.id,
+    redis: job.id
+  };
+};
+
+// src/registry/classes.ts
+function generateSlug(name) {
+  const normalized = name.normalize("NFKD").replace(/[\u0300-\u036f]/g, "");
+  const lowercase = normalized.toLowerCase();
+  const slug = lowercase.replace(/[\W_]+/g, "-").replace(/^-+|-+$/g, "");
+  return slug;
+}
+var ExuluZodFileType = ({
+  name,
+  label,
+  description,
+  allowedFileTypes
+}) => {
+  return import_zod2.z.object({
+    [`exulu_file_${name}`]: import_zod2.z.string().describe(JSON.stringify({
+      label,
+      isFile: true,
+      description,
+      allowedFileTypes
+    }))
+  });
+};
+var ExuluAgent = class {
+  id;
+  name;
+  description = "";
+  slug = "";
+  streaming = false;
+  type;
+  outputSchema;
+  rateLimit;
+  config;
+  memory;
+  tools;
+  capabilities;
+  constructor({ id, name, description, outputSchema, config, rateLimit, type, capabilities, tools }) {
+    this.id = id;
+    this.name = name;
+    this.type = type;
+    this.description = description;
+    this.outputSchema = outputSchema;
+    this.rateLimit = rateLimit;
+    this.tools = tools;
+    this.config = config;
+    this.capabilities = capabilities;
+    this.slug = `/agents/${generateSlug(this.name)}/run`;
+    if (config?.memory) {
+      const connectionString = `postgresql://${process.env.POSTGRES_DB_USER}:${process.env.POSTGRES_DB_PASSWORD}@${process.env.POSTGRES_DB_HOST}:${process.env.POSTGRES_DB_PORT}/exulu`;
+      this.memory = new import_memory.Memory({
+        storage: new import_pg.PostgresStore({
+          host: process.env.POSTGRES_DB_HOST || "",
+          port: parseInt(process.env.POSTGRES_DB_PORT || "5432"),
+          user: process.env.POSTGRES_DB_USER || "",
+          database: "exulu",
+          // putting it into an own database that is not managed by exulu
+          password: process.env.POSTGRES_DB_PASSWORD || "",
+          ssl: process.env.POSTGRES_DB_SSL === "true" ? { rejectUnauthorized: false } : false
+        }),
+        ...config?.memory.vector ? { vector: new import_pg.PgVector(connectionString) } : {},
+        options: {
+          lastMessages: config?.memory.lastMessages || 10,
+          semanticRecall: {
+            topK: config?.memory.semanticRecall.topK || 3,
+            messageRange: config?.memory.semanticRecall.messageRange || 2
+          }
+        }
+      });
+    }
+  }
+  chat = async (id) => {
+    const { db: db2 } = await postgresClient();
+    const agent = await db2.from("agents").select("*").where("id", "=", id).first();
+    if (!agent) {
+      throw new Error("Agent not found");
+    }
+    let tools = {};
+    agent.tools?.forEach(({ name }) => {
+      const tool = this.tools?.find((t) => t.name === name);
+      if (!tool) {
+        return;
+      }
+      return tool;
+    });
+    updateStatistic({
+      name: "count",
+      label: this.name,
+      type: STATISTICS_TYPE_ENUM.AGENT_RUN,
+      trigger: "agent"
+    });
+    return new import_core.Agent({
+      name: this.config.name,
+      instructions: this.config.instructions,
+      model: this.config.model,
+      tools,
+      memory: this.memory ? this.memory : void 0
+    });
+  };
+};
+var ExuluEmbedder = class {
+  id;
+  name;
+  slug = "";
+  queue;
+  generateEmbeddings;
+  vectorDimensions;
+  maxChunkSize;
+  chunker;
+  constructor({ id, name, description, generateEmbeddings, queue, vectorDimensions, maxChunkSize, chunker }) {
+    this.id = id;
+    this.name = name;
+    this.vectorDimensions = vectorDimensions;
+    this.maxChunkSize = maxChunkSize;
+    this.chunker = chunker;
+    this.slug = `/embedders/${generateSlug(this.name)}/run`;
+    this.queue = queue;
+    this.generateEmbeddings = generateEmbeddings;
+  }
+  async generateFromQuery(query, statistics) {
+    if (statistics) {
+    }
+    return await this.generateEmbeddings({
+      item: {
+        id: "placeholder"
+      },
+      chunks: [{
+        content: query,
+        index: 1
+      }]
+    });
+  }
+  async generateFromDocument(input, statistics) {
+    if (statistics) {
+    }
+    if (!this.chunker) {
+      throw new Error("Chunker not found for embedder " + this.name);
+    }
+    console.log("generating chunks");
+    if (!input.id) {
+      throw new Error("Item id is required for generating embeddings.");
+    }
+    const output = await this.chunker(input, this.maxChunkSize);
+    console.log("generating embeddings");
+    return await this.generateEmbeddings(output);
+  }
+};
+var ExuluWorkflow = class {
+  id;
+  name;
+  description = "";
+  enable_batch = false;
+  slug = "";
+  queue;
+  workflow;
+  inputSchema;
+  constructor({ id, name, description, workflow, queue, enable_batch, inputSchema }) {
+    this.id = id;
+    this.name = name;
+    this.description = description;
+    this.enable_batch = enable_batch;
+    this.slug = `/workflows/${generateSlug(this.name)}/run`;
+    this.queue = queue;
+    this.inputSchema = inputSchema;
+    this.workflow = workflow;
+  }
+};
+var ExuluLogger = class {
+  logPath;
+  job;
+  constructor(job, logsDir) {
+    this.job = job;
+    if (!fs.existsSync(logsDir)) {
+      fs.mkdirSync(logsDir, { recursive: true });
+    }
+    this.logPath = path.join(logsDir, `${job.id}_${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.txt`);
+  }
+  async write(message, level) {
+    const logMessage = message.endsWith("\n") ? message : message + "\n";
+    try {
+      await fs.promises.appendFile(this.logPath, `[EXULU][${level}] - ${(/* @__PURE__ */ new Date()).toISOString()}: ${logMessage}`);
+    } catch (error) {
+      console.error(`Error writing to log file ${this.job.id}:`, error);
+      throw error;
+    }
+  }
+};
+var ExuluTool = class {
+  id;
+  name;
+  description;
+  inputSchema;
+  outputSchema;
+  type;
+  _execute;
+  constructor({ id, name, description, inputSchema, outputSchema, type, execute: execute2 }) {
+    this.id = id;
+    this.name = name;
+    this.description = description;
+    this.inputSchema = inputSchema;
+    this.outputSchema = outputSchema;
+    this.type = type;
+    this._execute = execute2;
+  }
+  execute = async (inputs) => {
+    if (!this._execute) {
+      throw new Error("Tool has no execute function.");
+    }
+    updateStatistic({
+      name: "count",
+      label: this.name,
+      type: STATISTICS_TYPE_ENUM.TOOL_CALL,
+      trigger: "agent"
+    });
+    return await this._execute(inputs);
+  };
+};
+var ExuluContext = class {
+  id;
+  name;
+  active;
+  fields;
+  rateLimit;
+  description;
+  embedder;
+  queryRewriter;
+  resultReranker;
+  // todo typings
+  _sources = [];
+  configuration;
+  constructor({ id, name, description, embedder, active, rateLimit, fields, queryRewriter, resultReranker, configuration }) {
+    this.id = id;
+    this.name = name;
+    this.fields = fields || [];
+    this.configuration = configuration || {
+      calculateVectors: "manual"
+    };
+    this.description = description;
+    this.embedder = embedder;
+    this.active = active;
+    this.rateLimit = rateLimit;
+    this._sources = [];
+    this.queryRewriter = queryRewriter;
+    this.resultReranker = resultReranker;
+  }
+  deleteOne = async (id) => {
+    return {};
+  };
+  deleteAll = async () => {
+    return {};
+  };
+  getTableName = () => {
+    return sanitizeName(this.name) + "_items";
+  };
+  getChunksTableName = () => {
+    return sanitizeName(this.name) + "_chunks";
+  };
+  tableExists = async () => {
+    const { db: db2 } = await postgresClient();
+    const tableExists = await db2.schema.hasTable(this.getTableName());
+    return tableExists;
+  };
+  async updateItem(user, id, item) {
+    if (!id) {
+      throw new Error("Id is required for updating an item.");
+    }
+    const { db: db2 } = await postgresClient();
+    Object.keys(item).forEach((key) => {
+      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert") {
+        return;
+      }
+      console.log("this.fields", this.fields);
+      const field = this.fields.find((field2) => field2.name === key);
+      if (!field) {
+        throw new Error("Trying to uppdate value for field '" + key + "' that does not exist on the context fields definition. Available fields: " + this.fields.map((field2) => sanitizeName(field2.name)).join(", ") + " ,name, description, external_id");
+      }
+    });
+    delete item.id;
+    delete item.created_at;
+    delete item.upsert;
+    item.updated_at = db2.fn.now();
+    const result = await db2.from(this.getTableName()).where({ id }).update(item).returning("id");
+    if (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always") {
+      if (this.embedder.queue?.name) {
+        console.log("[EXULU] embedder is in queue mode, scheduling job.");
+        const job = await bullmqDecorator({
+          label: `Job running '${this.embedder.name}' for '${item.name} (${item.id}).'`,
+          embedder: this.embedder.id,
+          type: "embedder",
+          inputs: item,
+          queue: this.embedder.queue,
+          user
+        });
+        return {
+          id: result[0].id,
+          job: job.id
+        };
+      }
+      const { id: source, chunks } = await this.embedder.generateFromDocument({
+        ...item,
+        id
+      }, {
+        label: this.name,
+        trigger: "agent"
+      });
+      const exists = await db2.schema.hasTable(this.getChunksTableName());
+      if (!exists) {
+        await this.createChunksTable();
+      }
+      await db2.from(this.getChunksTableName()).where({ source }).delete();
+      await db2.from(this.getChunksTableName()).insert(chunks.map((chunk) => ({
+        source,
+        content: chunk.content,
+        chunk_index: chunk.index,
+        embedding: import_knex4.default.toSql(chunk.vector)
+      })));
+    }
+    return {
+      id: result[0].id,
+      job: void 0
+    };
+  }
+  async insertItem(user, item, upsert = false) {
+    if (!item.name) {
+      throw new Error("Name field is required.");
+    }
+    const { db: db2 } = await postgresClient();
+    if (item.external_id) {
+      const existingItem = await db2.from(this.getTableName()).where({ external_id: item.external_id }).first();
+      if (existingItem && !upsert) {
+        throw new Error("Item with external id " + item.external_id + " already exists.");
+      }
+      if (existingItem && upsert) {
+        await this.updateItem(user, existingItem.id, item);
+        return existingItem.id;
+      }
+    }
+    if (upsert && item.id) {
+      const existingItem = await db2.from(this.getTableName()).where({ id: item.id }).first();
+      if (existingItem && upsert) {
+        await this.updateItem(user, existingItem.id, item);
+        return existingItem.id;
+      }
+    }
+    Object.keys(item).forEach((key) => {
+      if (key === "name" || key === "description" || key === "external_id" || key === "tags" || key === "source" || key === "textLength" || key === "upsert") {
+        return;
+      }
+      console.log("this.fields", this.fields);
+      const field = this.fields.find((field2) => field2.name === key);
+      if (!field) {
+        throw new Error("Trying to insert value for field '" + key + "' that does not exist on the context fields definition. Available fields: " + this.fields.map((field2) => sanitizeName(field2.name)).join(", ") + " ,name, description, external_id");
+      }
+    });
+    delete item.id;
+    delete item.upsert;
+    const result = await db2.from(this.getTableName()).insert({
+      ...item,
+      id: db2.fn.uuid(),
+      created_at: db2.fn.now(),
+      updated_at: db2.fn.now()
+    }).returning("id");
+    if (this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always") {
+      if (this.embedder.queue?.name) {
+        console.log("[EXULU] embedder is in queue mode, scheduling job.");
+        const job = await bullmqDecorator({
+          label: `Job running '${this.embedder.name}' for '${item.name} (${item.id}).'`,
+          embedder: this.embedder.id,
+          type: "embedder",
+          inputs: item,
+          queue: this.embedder.queue,
+          user
+        });
+        return {
+          id: result[0].id,
+          job: job.id
+        };
+      }
+      console.log("[EXULU] embedder is not in queue mode, calculating vectors directly.");
+      const { id: source, chunks } = await this.embedder.generateFromDocument({
+        ...item,
+        id: result[0].id
+      }, {
+        label: this.name,
+        trigger: "agent"
+      });
+      const exists = await db2.schema.hasTable(this.getChunksTableName());
+      if (!exists) {
+        await this.createChunksTable();
+      }
+      console.log("inserting chunks");
+      await db2.from(this.getChunksTableName()).insert(chunks.map((chunk) => ({
+        source,
+        content: chunk.content,
+        chunk_index: chunk.index,
+        embedding: import_knex4.default.toSql(chunk.vector)
+      })));
+    }
+    return {
+      id: result[0].id,
+      job: void 0
+    };
+  }
+  getItems = async ({
+    statistics,
+    limit,
+    page,
+    name,
+    archived,
+    query,
+    method
+  }) => {
+    if (!query && limit > 500) {
+      throw new Error("Limit cannot be greater than 500.");
+    }
+    if (query && limit > 50) {
+      throw new Error("Limit cannot be greater than 50 when using a vector search query.");
+    }
+    if (page < 1) page = 1;
+    if (limit < 1) limit = 10;
+    let offset = (page - 1) * limit;
+    const mainTable = this.getTableName();
+    const { db: db2 } = await postgresClient();
+    const columns = await db2(mainTable).columnInfo();
+    const totalQuery = db2.count("* as count").from(mainTable).first();
+    const itemsQuery = db2.select(Object.keys(columns).map((column) => mainTable + "." + column)).from(mainTable).offset(offset).limit(limit);
+    if (typeof name === "string") {
+      itemsQuery.whereILike("name", `%${name}%`);
+      totalQuery.whereILike("name", `%${name}%`);
+    }
+    if (typeof archived === "boolean") {
+      itemsQuery.where("archived", archived);
+      totalQuery.where("archived", archived);
+    }
+    if (!query) {
+      const total = await totalQuery;
+      let items = await itemsQuery;
+      const last = Math.ceil(total.count / limit);
+      return {
+        pagination: {
+          totalCount: parseInt(total.count),
+          currentPage: page,
+          limit,
+          from: offset,
+          pageCount: last || 1,
+          to: offset + items.length,
+          lastPage: last || 1,
+          nextPage: page + 1 > last ? null : page + 1,
+          previousPage: page - 1 || null
+        },
+        filters: {
+          archived,
+          name,
+          query
+        },
+        context: {
+          name: this.name,
+          id: this.id,
+          embedder: this.embedder.name
+        },
+        items
+      };
+    }
+    if (typeof query === "string") {
+      itemsQuery.limit(limit * 5);
+      if (statistics) {
+        updateStatistic({
+          name: "count",
+          label: statistics.label,
+          type: STATISTICS_TYPE_ENUM.CONTEXT_RETRIEVE,
+          trigger: statistics.trigger
+        });
+      }
+      if (this.queryRewriter) {
+        query = await this.queryRewriter(query);
+      }
+      const chunksTable = this.getChunksTableName();
+      itemsQuery.leftJoin(chunksTable, function() {
+        this.on(chunksTable + ".source", "=", mainTable + ".id");
+      });
+      itemsQuery.select(chunksTable + ".id");
+      itemsQuery.select(chunksTable + ".source");
+      itemsQuery.select(chunksTable + ".content");
+      itemsQuery.select(chunksTable + ".chunk_index");
+      itemsQuery.select(chunksTable + ".created_at");
+      itemsQuery.select(chunksTable + ".updated_at");
+      const { chunks } = await this.embedder.generateFromQuery(query);
+      if (!chunks?.[0]?.vector) {
+        throw new Error("No vector generated for query.");
+      }
+      const vector = chunks[0].vector;
+      const vectorStr = `ARRAY[${vector.join(",")}]`;
+      const vectorExpr = `${vectorStr}::vector`;
+      switch (method) {
+        case "l1Distance":
+          itemsQuery.select(db2.raw(`?? <-> ${vectorExpr} as l1_distance`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`?? <-> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+          break;
+        case "l2Distance":
+          itemsQuery.select(db2.raw(`?? <-> ${vectorExpr} as l2_distance`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`?? <-> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+          break;
+        case "hammingDistance":
+          itemsQuery.select(db2.raw(`?? <#> ${vectorExpr} as hamming_distance`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+          break;
+        case "jaccardDistance":
+          itemsQuery.select(db2.raw(`?? <#> ${vectorExpr} as jaccard_distance`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+          break;
+        case "maxInnerProduct":
+          itemsQuery.select(db2.raw(`?? <#> ${vectorExpr} as inner_product`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+          break;
+        case "cosineDistance":
+        default:
+          itemsQuery.select(db2.raw(`1 - (?? <#> ${vectorExpr}) as cosine_distance`, [`${chunksTable}.embedding`]));
+          itemsQuery.orderByRaw(db2.raw(`1 - (?? <#> ${vectorExpr}) DESC`, [`${chunksTable}.embedding`]));
+          break;
+      }
+      let items = await itemsQuery;
+      const seenSources = /* @__PURE__ */ new Map();
+      items = items.reduce((acc, item) => {
+        if (!seenSources.has(item.source)) {
+          seenSources.set(item.source, {
+            ...Object.fromEntries(
+              Object.keys(item).filter(
+                (key) => key !== "l1_distance" && key !== "l2_distance" && key !== "hamming_distance" && key !== "jaccard_distance" && key !== "inner_product" && key !== "cosine_distance" && key !== "content" && key !== "source" && key !== "chunk_index"
+              ).map((key) => [key, item[key]])
+            ),
+            chunks: [{
+              content: item.content,
+              chunk_index: item.chunk_index,
+              ...method === "l1Distance" && { l1_distance: item.l1_distance },
+              ...method === "l2Distance" && { l2_distance: item.l2_distance },
+              ...method === "hammingDistance" && { hamming_distance: item.hamming_distance },
+              ...method === "jaccardDistance" && { jaccard_distance: item.jaccard_distance },
+              ...method === "maxInnerProduct" && { inner_product: item.inner_product },
+              ...method === "cosineDistance" && { cosine_distance: item.cosine_distance }
+            }]
+          });
+          acc.push(seenSources.get(item.source));
+        } else {
+          seenSources.get(item.source).chunks.push({
+            content: item.content,
+            chunk_index: item.chunk_index,
+            ...method === "l1Distance" && { l1_distance: item.l1_distance },
+            ...method === "l2Distance" && { l2_distance: item.l2_distance },
+            ...method === "hammingDistance" && { hamming_distance: item.hamming_distance },
+            ...method === "jaccardDistance" && { jaccard_distance: item.jaccard_distance },
+            ...method === "maxInnerProduct" && { inner_product: item.inner_product },
+            ...method === "cosineDistance" && { cosine_distance: item.cosine_distance }
+          });
+        }
+        return acc;
+      }, []);
+      if (this.resultReranker && query) {
+        items = await this.resultReranker(items);
+      }
+      return {
+        filters: {
+          archived,
+          name,
+          query
+        },
+        context: {
+          name: this.name,
+          id: this.id,
+          embedder: this.embedder.name
+        },
+        items
+      };
+    }
+  };
+  createItemsTable = async () => {
+    const { db: db2 } = await postgresClient();
+    const tableName = this.getTableName();
+    console.log("[EXULU] Creating table: " + tableName);
+    return await db2.schema.createTable(tableName, (table) => {
+      console.log("[EXULU] Creating fields for table.", this.fields);
+      table.uuid("id").primary().defaultTo(db2.fn.uuid());
+      table.string("name", 100);
+      table.text("description");
+      table.string("tags", 100);
+      table.boolean("archived").defaultTo(false);
+      table.string("external_id", 100);
+      table.integer("textLength");
+      table.string("source", 100);
+      for (const field of this.fields) {
+        const { type, name } = field;
+        if (!type || !name) {
+          continue;
+        }
+        mapType(table, type, sanitizeName(name));
+      }
+      table.timestamps(true, true);
+    });
+  };
+  createChunksTable = async () => {
+    const { db: db2 } = await postgresClient();
+    const tableName = this.getChunksTableName();
+    console.log("[EXULU] Creating table: " + tableName);
+    return await db2.schema.createTable(tableName, (table) => {
+      table.uuid("id").primary().defaultTo(db2.fn.uuid());
+      table.uuid("source").references("id").inTable(this.getTableName());
+      table.text("content");
+      table.integer("chunk_index");
+      table.specificType("embedding", `vector(${this.embedder.vectorDimensions})`);
+      table.timestamps(true, true);
+    });
+  };
+  // Exports the context as a tool that can be used by an agent
+  tool = () => {
+    return new ExuluTool({
+      id: this.id,
+      name: `${this.name} context`,
+      type: "context",
+      inputSchema: import_zod2.z.object({
+        query: import_zod2.z.string()
+      }),
+      outputSchema: import_zod2.z.object({
+        // todo check if result format is still correct based on above getItems function
+        results: import_zod2.z.array(import_zod2.z.object({
+          count: import_zod2.z.number(),
+          results: import_zod2.z.array(import_zod2.z.object({
+            id: import_zod2.z.string(),
+            content: import_zod2.z.string(),
+            metadata: import_zod2.z.record(import_zod2.z.any())
+          })),
+          errors: import_zod2.z.array(import_zod2.z.string()).optional()
+        }))
+      }),
+      description: `Gets information from the context called: ${this.name}. The context description is: ${this.description}.`,
+      execute: async ({ context }) => {
+        return await this.getItems({
+          page: 1,
+          limit: 10,
+          query: context.query,
+          statistics: {
+            label: this.name,
+            trigger: "agent"
+          }
+        });
+      }
+    });
+  };
+  sources = {
+    add: (inputs) => {
+      const source = new ExuluSource({
+        ...inputs,
+        context: this.id
+      });
+      this._sources.push(source);
+      return source;
+    },
+    get: (id) => {
+      if (id) {
+        return this._sources.find((source) => source.id === id);
+      }
+      return this._sources ?? [];
+    }
+  };
+};
+var ExuluSource = class {
+  id;
+  name;
+  description;
+  updaters;
+  context;
+  constructor({ id, name, description, updaters, context }) {
+    this.id = id;
+    this.name = name;
+    this.description = description;
+    this.context = context;
+    this.updaters = updaters.map((updater) => {
+      if (updater.type === "webhook") {
+        return {
+          ...updater,
+          slug: `/contexts/${context}/sources/${this.id}/${updater.id}/webhook`
+        };
+      }
+      return updater;
+    });
+  }
+};
+var updateStatistic = async (statistic) => {
+  const currentDate = (/* @__PURE__ */ new Date()).toISOString().split("T")[0];
+  const { db: db2 } = await postgresClient();
+  await db2.from("statistics").update({
+    total: db2.raw("total + ?", [statistic.count ?? 1]),
+    timeseries: db2.raw("CASE WHEN date = ? THEN array_append(timeseries, ?) ELSE timeseries END", [currentDate, { date: currentDate, count: statistic.count ?? 1 }])
+  }).where({
+    name: statistic.name,
+    label: statistic.label,
+    type: statistic.type
+  }).onConflict("name").merge();
+};
+
+// src/registry/index.ts
+var import_express6 = require("express");
+
+// src/registry/routes.ts
+var import_express3 = require("express");
+
+// src/registry/rate-limiter.ts
+var rateLimiter = async (key, windowSeconds, limit, points) => {
+  if (!redisClient) {
+    return {
+      status: false,
+      retryAfter: 10
+      // 10 seconds
+    };
+  }
+  const redisKey = `exulu/${key}`;
+  const current = await redisClient.incrBy(redisKey, points);
+  if (current === points) {
+    await redisClient.expire(redisKey, windowSeconds);
+  }
+  if (current > limit) {
+    const ttl = await redisClient.ttl(redisKey);
+    return {
+      status: false,
+      retryAfter: ttl
+    };
+  }
+  return {
+    status: true,
+    retryAfter: null
+  };
+};
+
+// src/registry/route-validators/index.ts
+var import_express = require("express");
+var import_jwt = require("next-auth/jwt");
+
+// src/auth/auth.ts
+var import_bcryptjs = __toESM(require("bcryptjs"), 1);
+var authentication = async ({
+  apikey,
+  authtoken,
+  internalkey,
+  db: db2
+}) => {
+  if (internalkey) {
+    if (!process.env.INTERNAL_SECRET) {
+      return {
+        error: true,
+        message: `Header "internal" provided, but no INTERNAL_SECRET was provided in the environment variables.`,
+        code: 401
+      };
+    }
+    if (process.env.INTERNAL_SECRET !== internalkey) {
+      return {
+        error: true,
+        message: `Internal key was provided in header but did not match the INTERNAL_SECRET environment variable.`,
+        code: 401
+      };
+    }
+    return {
+      error: false,
+      code: 200,
+      user: {
+        type: "api",
+        id: "XXXX-XXXX-XXXX-XXXX",
+        email: "internal@exulu.com"
+      }
+    };
+  }
+  if (authtoken) {
+    try {
+      console.log("authtoken", authtoken);
+      if (!authtoken?.email) {
+        return {
+          error: true,
+          message: `No email provided in session ${JSON.stringify(authtoken)}`,
+          code: 401
+        };
+      }
+      const user = await db2.from("users").select("*").where("email", authtoken?.email).first();
+      console.log("user", user);
+      if (!user) {
+        return {
+          error: true,
+          message: `No user found for email: ${authtoken.email}`,
+          code: 401
+        };
+      }
+      return {
+        error: false,
+        code: 200,
+        user
+      };
+    } catch (error) {
+      console.error(error);
+      return {
+        error: true,
+        message: "Invalid token.",
+        code: 401
+      };
+    }
+  }
+  if (apikey) {
+    const users = await db2.from("users").select("*").where("type", "api");
+    if (!users || users.length === 0) {
+      return {
+        error: true,
+        message: `No API users found.`,
+        code: 401
+      };
+    }
+    const keyParts = apikey.split("/");
+    const keyName = keyParts.pop();
+    const keyValue = keyParts[0];
+    if (!keyName) {
+      return {
+        error: true,
+        message: "Provided api key does not include postfix with key name ({key}/{name}).",
+        code: 401
+      };
+    }
+    if (!keyValue) {
+      return {
+        error: true,
+        message: "Provided api key is not in the correct format.",
+        code: 401
+      };
+    }
+    const filtered = users.filter(({ apiKey, id }) => apiKey.includes(keyName));
+    for (const user of filtered) {
+      const lastSlashIndex = user.apiKey.lastIndexOf("/");
+      const compareValue = lastSlashIndex !== -1 ? user.apiKey.substring(0, lastSlashIndex) : user.apiKey;
+      const isMatch = await import_bcryptjs.default.compare(keyValue, compareValue);
+      if (isMatch) {
+        await db2.from("users").where({ id: user.id }).update({
+          lastUsed: /* @__PURE__ */ new Date()
+        }).returning("id");
+        return {
+          error: false,
+          code: 200,
+          user
+        };
+      }
+    }
+  }
+  return {
+    error: true,
+    message: "Either an api key or authorization key must be provided.",
+    code: 401
+  };
+};
+
+// src/registry/route-validators/index.ts
+var requestValidators = {
+  authenticate: async (req) => {
+    const apikey = req.headers["exulu-api-key"] || null;
+    const { db: db2 } = await postgresClient();
+    let authtoken = null;
+    if (typeof apikey !== "string") {
+      const secret = process.env.NEXTAUTH_SECRET;
+      authtoken = await (0, import_jwt.getToken)({ req, secret });
+    }
+    return await authentication({
+      authtoken,
+      apikey,
+      db: db2
+    });
+  },
+  workflows: (req) => {
+    const contentType = req.headers["content-type"] || "";
+    if (!contentType.includes("application/json")) {
+      return {
+        error: true,
+        code: 400,
+        message: "Unsupported content type."
+      };
+    }
+    if (!req.body) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing body."
+      };
+    }
+    if (!req.body.agent) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing agent in body."
+      };
+    }
+    if (!req.body.session) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing session in body."
+      };
+    }
+    if (!req.body.inputs) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing inputs in body."
+      };
+    }
+    if (!req.body.label) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing label for job in body."
+      };
+    }
+    return {
+      error: false
+    };
+  },
+  embedders: (req, configuration) => {
+    const contentType = req.headers["content-type"] || "";
+    if (!contentType.includes("application/json")) {
+      return {
+        error: true,
+        code: 400,
+        message: "Unsupported content type."
+      };
+    }
+    if (!req.body) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing body."
+      };
+    }
+    if (!req.body.inputs) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing inputs."
+      };
+    }
+    if (!req.body.label) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing label for job in body."
+      };
+    }
+    if (configuration) {
+      for (const key in configuration) {
+        if (!req.body.configuration[key]) {
+          return {
+            error: true,
+            code: 400,
+            message: `Missing ${key} in body.configuration.`
+          };
+        }
+      }
+    }
+    return {
+      error: false
+    };
+  },
+  agents: (req) => {
+    console.log("[EXULU] validating request body and headers.", req.body);
+    const contentType = req.headers["content-type"] || "";
+    if (!contentType.includes("application/json")) {
+      return {
+        error: true,
+        code: 400,
+        message: "Unsupported content type."
+      };
+    }
+    if (!req.body) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing body."
+      };
+    }
+    if (!req.body.threadId) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing threadId in body."
+      };
+    }
+    if (!req.body.resourceId) {
+      return {
+        error: true,
+        code: 400,
+        message: "Missing resourceId in body."
+      };
+    }
+    if (!req.body.messages) {
+      return {
+        error: true,
+        code: 400,
+        message: 'Missing "messages" property in body.'
+      };
+    }
+    return {
+      error: false
+    };
+  }
+};
+
+// src/registry/routes.ts
+var import_zodex = require("zodex");
+
+// src/bullmq/queues.ts
+var import_bullmq5 = require("bullmq");
+var ExuluQueues = class {
+  queues;
+  constructor() {
+    this.queues = [];
+  }
+  queue(name) {
+    return this.queues.find((x) => x.name === name);
+  }
+  use(name) {
+    const existing = this.queues.find((x) => x.name === name);
+    if (existing) {
+      return existing;
+    }
+    const newQueue = new import_bullmq5.Queue(`${name}`, { connection: redisServer });
+    this.queues.push(newQueue);
+    return newQueue;
+  }
+};
+var queues = new ExuluQueues();
+
+// types/models/vector-methods.ts
+var VectorMethodEnum = {
+  "cosineDistance": "cosineDistance",
+  "l1Distance": "l1Distance",
+  "l2Distance": "l2Distance",
+  "hammingDistance": "hammingDistance",
+  "jaccardDistance": "jaccardDistance",
+  "maxInnerProduct": "maxInnerProduct"
+};
+
+// src/registry/routes.ts
+var import_express4 = __toESM(require("express"), 1);
+var import_server3 = require("@apollo/server");
+var import_cors = __toESM(require("cors"), 1);
+var import_reflect_metadata = require("reflect-metadata");
+
+// src/registry/utils/graphql.ts
+var import_schema = require("@graphql-tools/schema");
+var import_graphql_type_json = __toESM(require("graphql-type-json"), 1);
+var map = (field) => {
+  let type;
+  switch (field.type) {
+    case "text":
+    case "shortText":
+    case "longText":
+    case "code":
+      type = "String";
+      break;
+    case "number":
+      type = "Float";
+      break;
+    case "boolean":
+      type = "Boolean";
+      break;
+    case "json":
+      type = "JSON";
+      break;
+    case "date":
+      type = "String";
+      break;
+    default:
+      type = "String";
+  }
+  return type;
+};
+function createTypeDefs(table) {
+  const fields = table.fields.map((field) => {
+    let type;
+    type = map(field);
+    const required = field.required ? "!" : "";
+    return `  ${field.name}: ${type}${required}`;
+  });
+  const typeDef = `
+  type ${table.name.singular} {
+  ${fields.join("\n")}
+    id: ID!
+    createdAt: String!
+    updatedAt: String!
+  }
+  `;
+  const inputDef = `
+input ${table.name.singular}Input {
+${table.fields.map((f) => `  ${f.name}: ${map(f)}`).join("\n")}
+}
+`;
+  return typeDef + inputDef;
+}
+function createFilterTypeDefs(table) {
+  const fieldFilters = table.fields.map((field) => {
+    let type;
+    type = map(field);
+    return `
+  ${field.name}: FilterOperator${type}`;
+  });
+  const tableNameSingularUpperCaseFirst = table.name.singular.charAt(0).toUpperCase() + table.name.singular.slice(1);
+  const operatorTypes = `
+input FilterOperatorString {
+  eq: String
+  ne: String
+  in: [String]
+  contains: String
+}
+
+input FilterOperatorFloat {
+  eq: Float
+  ne: Float
+  in: [Float]
+}
+
+input FilterOperatorBoolean {
+  eq: Boolean
+  ne: Boolean
+  in: [Boolean]
+}
+
+input FilterOperatorJSON {
+  eq: JSON
+  ne: JSON
+  in: [JSON]
+}
+
+input SortBy {
+  field: String!
+  direction: SortDirection!
+}
+
+enum SortDirection {
+  ASC
+  DESC
+}
+
+input Filter${tableNameSingularUpperCaseFirst} {
+${fieldFilters.join("\n")}
+}`;
+  return operatorTypes;
+}
+var getRequestedFields = (info) => {
+  const selections = info.operation.selectionSet.selections[0].selectionSet.selections;
+  const itemsSelection = selections.find((s) => s.name.value === "items");
+  const fields = itemsSelection ? Object.keys(itemsSelection.selectionSet.selections.reduce((acc, field) => {
+    acc[field.name.value] = true;
+    return acc;
+  }, {})) : Object.keys(selections.reduce((acc, field) => {
+    acc[field.name.value] = true;
+    return acc;
+  }, {}));
+  return fields.filter((field) => field !== "pageInfo" && field !== "items");
+};
+function createMutations(table) {
+  const tableNamePlural = table.name.plural.toLowerCase();
+  const tableNameSingular = table.name.singular.toLowerCase();
+  return {
+    [`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
+      const { db: db2 } = context;
+      const requestedFields = getRequestedFields(info);
+      const results = await db2(tableNamePlural).insert({
+        ...args.input,
+        createdAt: /* @__PURE__ */ new Date(),
+        updatedAt: /* @__PURE__ */ new Date()
+      }).returning(requestedFields);
+      console.log("requestedFields", requestedFields);
+      return results[0];
+    },
+    [`${tableNamePlural}UpdateOne`]: async (_, args, context, info) => {
+      const { db: db2 } = context;
+      const { where, input } = args;
+      await db2(tableNamePlural).where(where).update({
+        ...input,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      const requestedFields = getRequestedFields(info);
+      const result = await db2.from(tableNamePlural).select(requestedFields).where(where).first();
+      return result;
+    },
+    [`${tableNamePlural}UpdateOneById`]: async (_, args, context, info) => {
+      const { id, input } = args;
+      const { db: db2 } = context;
+      await db2(tableNamePlural).where({ id }).update({
+        ...input,
+        updatedAt: /* @__PURE__ */ new Date()
+      });
+      const requestedFields = getRequestedFields(info);
+      const result = await db2.from(tableNamePlural).select(requestedFields).where({ id }).first();
+      return result;
+    },
+    [`${tableNamePlural}RemoveOne`]: async (_, args, context, info) => {
+      const { db: db2 } = context;
+      const { where } = args;
+      const requestedFields = getRequestedFields(info);
+      const result = await db2.from(tableNamePlural).select(requestedFields).where(where).first();
+      await db2(tableNamePlural).where(where).del();
+      return result;
+    },
+    [`${tableNamePlural}RemoveOneById`]: async (_, args, context, info) => {
+      const { id } = args;
+      const { db: db2 } = context;
+      const requestedFields = getRequestedFields(info);
+      const result = await db2.from(tableNamePlural).select(requestedFields).where({ id }).first();
+      await db2(tableNamePlural).where({ id }).del();
+      return result;
+    }
+  };
+}
+function createQueries(table) {
+  const tableNamePlural = table.name.plural.toLowerCase();
+  const tableNameSingular = table.name.singular.toLowerCase();
+  const applyFilters = (query, filters) => {
+    filters.forEach((filter) => {
+      Object.entries(filter).forEach(([fieldName, operators]) => {
+        if (operators) {
+          if (operators.eq !== void 0) {
+            query = query.where(fieldName, operators.eq);
+          }
+          if (operators.ne !== void 0) {
+            query = query.whereRaw(`?? IS DISTINCT FROM ?`, [fieldName, operators.ne]);
+          }
+          if (operators.in !== void 0) {
+            query = query.whereIn(fieldName, operators.in);
+          }
+          if (operators.contains !== void 0) {
+            query = query.where(fieldName, "like", `%${operators.contains}%`);
+          }
+        }
+      });
+    });
+    return query;
+  };
+  const applySorting = (query, sort) => {
+    if (sort) {
+      query = query.orderBy(sort.field, sort.direction.toLowerCase());
+    }
+    return query;
+  };
+  return {
+    [`${tableNameSingular}ById`]: async (_, args, context, info) => {
+      const { db: db2 } = context;
+      const requestedFields = getRequestedFields(info);
+      const result = await db2.from(tableNamePlural).select(requestedFields).where({ id: args.id }).first();
+      return result;
+    },
+    [`${tableNameSingular}One`]: async (_, args, context, info) => {
+      const { filters = [], sort } = args;
+      const { db: db2 } = context;
+      const requestedFields = getRequestedFields(info);
+      let query = db2.from(tableNamePlural).select(requestedFields);
+      query = applyFilters(query, filters);
+      query = applySorting(query, sort);
+      const result = await query.first();
+      return result;
+    },
+    [`${tableNamePlural}Pagination`]: async (_, args, context, info) => {
+      const { limit = 10, page = 0, filters = [], sort } = args;
+      const { db: db2 } = context;
+      console.log("page", page);
+      let baseQuery = db2(tableNamePlural);
+      baseQuery = applyFilters(baseQuery, filters);
+      const [{ count }] = await baseQuery.clone().count("* as count");
+      const itemCount = Number(count);
+      const pageCount = Math.ceil(itemCount / limit);
+      const currentPage = page;
+      const hasPreviousPage = currentPage > 1;
+      const hasNextPage = currentPage < pageCount - 1;
+      let dataQuery = baseQuery.clone();
+      const requestedFields = getRequestedFields(info);
+      dataQuery = applySorting(dataQuery, sort);
+      if (page > 1) {
+        dataQuery = dataQuery.offset((page - 1) * limit);
+      }
+      const items = await dataQuery.select(requestedFields).limit(limit);
+      console.log("items", items);
+      console.log("query", dataQuery.toQuery());
+      return {
+        pageInfo: {
+          pageCount,
+          itemCount,
+          currentPage,
+          hasPreviousPage,
+          hasNextPage
+        },
+        items
+      };
+    }
+  };
+}
+function createSDL(tables) {
+  let typeDefs = `
+    scalar JSON
+    
+    type Query {
+    `;
+  let mutationDefs = `
+    type Mutation {
+    `;
+  let modelDefs = "";
+  const resolvers = { JSON: import_graphql_type_json.default, Query: {}, Mutation: {} };
+  for (const table of tables) {
+    const tableNamePlural = table.name.plural.toLowerCase();
+    const tableNameSingular = table.name.singular.toLowerCase();
+    const tableNameSingularUpperCaseFirst = table.name.singular.charAt(0).toUpperCase() + table.name.singular.slice(1);
+    typeDefs += `
+      ${tableNameSingular}ById(id: ID!): ${tableNameSingular}
+      ${tableNamePlural}Pagination(limit: Int, page: Int, filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingularUpperCaseFirst}PaginationResult
+      ${tableNameSingular}One(filters: [Filter${tableNameSingularUpperCaseFirst}], sort: SortBy): ${tableNameSingular}
+    `;
+    mutationDefs += `
+      ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!): ${tableNameSingular}
+      ${tableNamePlural}UpdateOne(where: JSON!, input: ${tableNameSingular}Input!): ${tableNameSingular}
+      ${tableNamePlural}UpdateOneById(id: ID!, input: ${tableNameSingular}Input!): ${tableNameSingular}
+      ${tableNamePlural}RemoveOne(where: JSON!): ${tableNameSingular}
+      ${tableNamePlural}RemoveOneById(id: ID!): ${tableNameSingular}
+    `;
+    modelDefs += createTypeDefs(table);
+    modelDefs += createFilterTypeDefs(table);
+    modelDefs += `
+type ${tableNameSingularUpperCaseFirst}PaginationResult {
+  pageInfo: PageInfo!
+  items: [${tableNameSingular}]!
+}
+
+type PageInfo {
+  pageCount: Int!
+  itemCount: Int!
+  currentPage: Int!
+  hasPreviousPage: Boolean!
+  hasNextPage: Boolean!
+}
+`;
+    Object.assign(resolvers.Query, createQueries(table));
+    Object.assign(resolvers.Mutation, createMutations(table));
+  }
+  typeDefs += "}\n";
+  mutationDefs += "}\n";
+  const fullSDL = typeDefs + mutationDefs + modelDefs;
+  const schema = (0, import_schema.makeExecutableSchema)({
+    typeDefs: fullSDL,
+    resolvers
+  });
+  console.log("\n\u{1F4CA} GraphQL Schema Overview\n");
+  const queriesTable = Object.keys(resolvers.Query).map((query) => ({
+    "Operation Type": "Query",
+    "Name": query,
+    "Description": "Retrieves data"
+  }));
+  const mutationsTable = Object.keys(resolvers.Mutation).map((mutation) => ({
+    "Operation Type": "Mutation",
+    "Name": mutation,
+    "Description": "Modifies data"
+  }));
+  const typesTable = tables.flatMap(
+    (table) => table.fields.map((field) => ({
+      "Type": table.name.singular,
+      "Field": field.name,
+      "Field Type": field.type,
+      "Required": field.required ? "Yes" : "No"
+    }))
+  );
+  console.log("\u{1F50D} Operations:");
+  console.table([...queriesTable, ...mutationsTable]);
+  console.log("\n\u{1F4DD} Types and Fields:");
+  console.table(typesTable);
+  console.log("\n");
+  return schema;
+}
+
+// src/registry/routes.ts
+var import_express5 = require("@as-integrations/express5");
+
+// src/registry/uppy.ts
+var import_express2 = require("express");
+var import_jwt2 = require("next-auth/jwt");
+var bodyParser = require("body-parser");
+var createUppyRoutes = async (app) => {
+  const {
+    S3Client,
+    AbortMultipartUploadCommand,
+    CompleteMultipartUploadCommand,
+    CreateMultipartUploadCommand,
+    GetObjectCommand,
+    ListPartsCommand,
+    PutObjectCommand,
+    UploadPartCommand,
+    ListObjectsV2Command
+  } = require("@aws-sdk/client-s3");
+  const { getSignedUrl } = require("@aws-sdk/s3-request-presigner");
+  const {
+    STSClient,
+    GetFederationTokenCommand
+  } = require("@aws-sdk/client-sts");
+  const policy = {
+    Version: "2012-10-17",
+    Statement: [
+      {
+        Effect: "Allow",
+        Action: [
+          "s3:PutObject"
+        ],
+        Resource: [
+          `arn:aws:s3:::${process.env.COMPANION_S3_BUCKET}/*`,
+          `arn:aws:s3:::${process.env.COMPANION_S3_BUCKET}`
+        ]
+      }
+    ]
+  };
+  let s3Client;
+  let stsClient;
+  const expiresIn = 60 * 60 * 24 * 1;
+  function getS3Client() {
+    s3Client ??= new S3Client({
+      region: process.env.COMPANION_S3_REGION,
+      ...process.env.COMPANION_S3_ENDPOINT && {
+        forcePathStyle: true,
+        endpoint: process.env.COMPANION_S3_ENDPOINT
+      },
+      credentials: {
+        accessKeyId: process.env.COMPANION_S3_KEY,
+        secretAccessKey: process.env.COMPANION_S3_SECRET
+      }
+    });
+    return s3Client;
+  }
+  function getSTSClient() {
+    stsClient ??= new STSClient({
+      region: process.env.COMPANION_S3_REGION,
+      ...process.env.COMPANION_S3_ENDPOINT && { endpoint: process.env.COMPANION_S3_ENDPOINT },
+      credentials: {
+        accessKeyId: process.env.COMPANION_S3_KEY,
+        secretAccessKey: process.env.COMPANION_S3_SECRET
+      }
+    });
+    return stsClient;
+  }
+  app.use(bodyParser.urlencoded({ extended: true }), bodyParser.json());
+  app.get("/", (req, res) => {
+    res.json("Exulu upload server.");
+  });
+  app.get("/s3/list", async (req, res, next) => {
+    const apikey = req.headers["exulu-api-key"] || null;
+    let authtoken = null;
+    if (typeof apikey !== "string") {
+      const secret = process.env.NEXTAUTH_SECRET;
+      authtoken = await (0, import_jwt2.getToken)({ req, secret });
+    }
+    const { db: db2 } = await postgresClient();
+    const authenticationResult = await authentication({
+      authtoken,
+      apikey,
+      db: db2
+    });
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { prefix = "" } = req.query;
+    if (typeof prefix !== "string") {
+      res.status(400).json({ error: "Invalid prefix parameter. Must be a string." });
+      return;
+    }
+    if (authenticationResult.user.type !== "api" && !prefix.includes(authenticationResult.user.id)) {
+      res.status(405).json({ error: "Not allowed to list files in this folder based on authenticated user." });
+      return;
+    }
+    try {
+      const command = new ListObjectsV2Command({
+        Bucket: process.env.COMPANION_S3_BUCKET,
+        Prefix: prefix,
+        MaxKeys: 1e3
+        // Adjust this value based on your needs
+      });
+      const data = await getS3Client().send(command);
+      const files = data.Contents?.map((item) => ({
+        key: item.Key,
+        size: item.Size,
+        lastModified: item.LastModified
+      })) || [];
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.status(200).json({
+        files,
+        isTruncated: data.IsTruncated,
+        nextContinuationToken: data.NextContinuationToken
+      });
+    } catch (err) {
+      next(err);
+    }
+  });
+  app.get("/s3/download", async (req, res, next) => {
+    const apikey = req.headers["exulu-api-key"] || null;
+    const internalkey = req.headers["internal-key"] || null;
+    const { db: db2 } = await postgresClient();
+    let authtoken = null;
+    if (typeof apikey !== "string" && typeof internalkey !== "string") {
+      const secret = process.env.NEXTAUTH_SECRET;
+      authtoken = await (0, import_jwt2.getToken)({ req, secret });
+    }
+    const authenticationResult = await authentication({
+      authtoken,
+      apikey,
+      internalkey,
+      db: db2
+    });
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { key } = req.query;
+    if (typeof key !== "string" || key.trim() === "") {
+      res.status(400).json({ error: "Missing or invalid `key` query parameter." });
+      return;
+    }
+    if (authenticationResult.user.type !== "api" && !key.includes(authenticationResult.user.id)) {
+      res.status(405).json({ error: "Not allowed to access the files in the folder based on authenticated user." });
+      return;
+    }
+    try {
+      const url = await getSignedUrl(
+        getS3Client(),
+        new GetObjectCommand({
+          Bucket: process.env.COMPANION_S3_BUCKET,
+          Key: key
+        }),
+        { expiresIn }
+      );
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.json({ url, method: "GET", expiresIn });
+    } catch (err) {
+      next(err);
+    }
+  });
+  app.get("/s3/sts", (req, res, next) => {
+    getSTSClient().send(new GetFederationTokenCommand({
+      Name: "Exulu",
+      // The duration, in seconds, of the role session. The value specified
+      // can range from 900 seconds (15 minutes) up to the maximum session
+      // duration set for the role.
+      DurationSeconds: expiresIn,
+      Policy: JSON.stringify(policy)
+    })).then((response) => {
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.setHeader("Cache-Control", `public,max-age=${expiresIn}`);
+      res.json({
+        credentials: response.Credentials,
+        bucket: process.env.COMPANION_S3_BUCKET,
+        region: process.env.COMPANION_S3_REGION
+      });
+    }, next);
+  });
+  const validateFileParameters = (filename, contentType) => {
+    if (!filename || !contentType) {
+      throw new Error("Missing required parameters: filename and content type are required");
+    }
+  };
+  const extractFileParameters = (req) => {
+    const isPostRequest = req.method === "POST";
+    const params = isPostRequest ? req.body : req.query;
+    return {
+      filename: params.filename,
+      contentType: params.type
+    };
+  };
+  const generateS3Key = (filename) => `${crypto.randomUUID()}-${filename}`;
+  const signOnServer = async (req, res, next) => {
+    const apikey = req.headers["exulu-api-key"] || null;
+    const { db: db2 } = await postgresClient();
+    let authtoken = null;
+    if (typeof apikey !== "string") {
+      const secret = process.env.NEXTAUTH_SECRET;
+      authtoken = await (0, import_jwt2.getToken)({ req, secret });
+    }
+    const authenticationResult = await authentication({
+      authtoken,
+      apikey,
+      db: db2
+    });
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { filename, contentType } = extractFileParameters(req);
+    validateFileParameters(filename, contentType);
+    const key = generateS3Key(filename);
+    let folder = "";
+    if (authenticationResult.user.type === "api") {
+      folder = `api/`;
+    } else {
+      folder = `${authenticationResult.user.id}/`;
+    }
+    getSignedUrl(
+      getS3Client(),
+      new PutObjectCommand({
+        Bucket: process.env.COMPANION_S3_BUCKET,
+        Key: folder + key,
+        ContentType: contentType
+      }),
+      { expiresIn }
+    ).then((url) => {
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.json({
+        url,
+        method: "PUT"
+      });
+      res.end();
+    }, next);
+  };
+  app.get("/s3/params", async (req, res, next) => {
+    return await signOnServer(req, res, next);
+  });
+  app.post("/s3/sign", async (req, res, next) => {
+    return await signOnServer(req, res, next);
+  });
+  app.post("/s3/multipart", async (req, res, next) => {
+    const apikey = req.headers["exulu-api-key"] || null;
+    const { db: db2 } = await postgresClient();
+    let authtoken = null;
+    if (typeof apikey !== "string") {
+      const secret = process.env.NEXTAUTH_SECRET;
+      authtoken = await (0, import_jwt2.getToken)({ req, secret });
+    }
+    const authenticationResult = await authentication({
+      authtoken,
+      apikey,
+      db: db2
+    });
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const client = getS3Client();
+    const { type, metadata, filename } = req.body;
+    if (typeof filename !== "string") {
+      return res.status(400).json({ error: "s3: content filename must be a string" });
+    }
+    if (typeof type !== "string") {
+      return res.status(400).json({ error: "s3: content type must be a string" });
+    }
+    const key = `${crypto.randomUUID()}-${filename}`;
+    let folder = "";
+    if (authenticationResult.user.type === "api") {
+      folder = `api/`;
+    } else {
+      folder = `${authenticationResult.user.id}/`;
+    }
+    const params = {
+      Bucket: process.env.COMPANION_S3_BUCKET,
+      Key: folder + key,
+      ContentType: type,
+      Metadata: metadata
+    };
+    const command = new CreateMultipartUploadCommand(params);
+    return client.send(command, (err, data) => {
+      if (err) {
+        next(err);
+        return;
+      }
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.json({
+        key: data.Key,
+        uploadId: data.UploadId
+      });
+    });
+  });
+  function validatePartNumber(partNumber) {
+    partNumber = Number(partNumber);
+    return Number.isInteger(partNumber) && partNumber >= 1 && partNumber <= 1e4;
+  }
+  app.get("/s3/multipart/:uploadId/:partNumber", (req, res, next) => {
+    const { uploadId, partNumber } = req.params;
+    const { key } = req.query;
+    if (!validatePartNumber(partNumber)) {
+      return res.status(400).json({ error: "s3: the part number must be an integer between 1 and 10000." });
+    }
+    if (typeof key !== "string") {
+      return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
+    }
+    return getSignedUrl(getS3Client(), new UploadPartCommand({
+      Bucket: process.env.COMPANION_S3_BUCKET,
+      Key: key,
+      UploadId: uploadId,
+      PartNumber: partNumber,
+      Body: ""
+    }), { expiresIn }).then((url) => {
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.json({ url, expires: expiresIn });
+    }, next);
+  });
+  app.get("/s3/multipart/:uploadId", (req, res, next) => {
+    const client = getS3Client();
+    const { uploadId } = req.params;
+    const { key } = req.query;
+    if (typeof key !== "string") {
+      res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
+      return;
+    }
+    const parts = [];
+    function listPartsPage(startAt) {
+      client.send(new ListPartsCommand({
+        Bucket: process.env.COMPANION_S3_BUCKET,
+        Key: key,
+        UploadId: uploadId,
+        PartNumberMarker: startAt
+      }), (err, data) => {
+        if (err) {
+          next(err);
+          return;
+        }
+        parts.push(...data.Parts);
+        if (data.IsTruncated) {
+          listPartsPage(data.NextPartNumberMarker);
+        } else {
+          res.json(parts);
+        }
+      });
+    }
+    listPartsPage(0);
+  });
+  function isValidPart(part) {
+    return part && typeof part === "object" && Number(part.PartNumber) && typeof part.ETag === "string";
+  }
+  app.post("/s3/multipart/:uploadId/complete", (req, res, next) => {
+    const client = getS3Client();
+    const { uploadId } = req.params;
+    const { key } = req.query;
+    const { parts } = req.body;
+    if (typeof key !== "string") {
+      return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
+    }
+    if (!Array.isArray(parts) || !parts.every(isValidPart)) {
+      return res.status(400).json({ error: "s3: `parts` must be an array of {ETag, PartNumber} objects." });
+    }
+    return client.send(new CompleteMultipartUploadCommand({
+      Bucket: process.env.COMPANION_S3_BUCKET,
+      Key: key,
+      UploadId: uploadId,
+      MultipartUpload: {
+        Parts: parts
+      }
+    }), (err, data) => {
+      if (err) {
+        next(err);
+        return;
+      }
+      res.setHeader("Access-Control-Allow-Origin", "*");
+      res.json({
+        location: data.Location
+      });
+    });
+  });
+  app.delete("/s3/multipart/:uploadId", (req, res, next) => {
+    const client = getS3Client();
+    const { uploadId } = req.params;
+    const { key } = req.query;
+    if (typeof key !== "string") {
+      return res.status(400).json({ error: 's3: the object key must be passed as a query parameter. For example: "?key=abc.jpg"' });
+    }
+    return client.send(new AbortMultipartUploadCommand({
+      Bucket: process.env.COMPANION_S3_BUCKET,
+      Key: key,
+      UploadId: uploadId
+    }), (err) => {
+      if (err) {
+        next(err);
+        return;
+      }
+      res.json({});
+    });
+  });
+  return app;
+};
+
+// src/registry/routes.ts
+var Papa = require("papaparse");
+var global_queues = {
+  logs_cleaner: "logs-cleaner"
+};
+var createRecurringJobs = async () => {
+  const recurringJobSchedulersLogs = [];
+  const queue = queues.use(global_queues.logs_cleaner);
+  recurringJobSchedulersLogs.push({
+    name: global_queues.logs_cleaner,
+    pattern: "0 10 * * * *",
+    ttld: "30 days",
+    opts: {
+      backoff: 3,
+      attempts: 5,
+      removeOnFail: 1e3
+    }
+  });
+  await queue.upsertJobScheduler(
+    "logs-cleaner-scheduler",
+    { pattern: "0 10 * * * *" },
+    // every 10 minutes
+    {
+      name: global_queues.logs_cleaner,
+      data: { ttld: 30 },
+      // time to live in days
+      opts: {
+        backoff: 3,
+        attempts: 5,
+        removeOnFail: 1e3
+      }
+    }
+  );
+  console.log("Recurring job schedulers:");
+  console.table(recurringJobSchedulersLogs);
+  return queue;
+};
+var createExpressRoutes = async (app, agents, embedders, tools, workflows, contexts) => {
+  const routeLogs = [];
+  var corsOptions = {
+    origin: "*",
+    optionsSuccessStatus: 200
+    // some legacy browsers (IE11, various SmartTVs) choke on 204
+  };
+  app.use((0, import_cors.default)(corsOptions));
+  console.log(`
+    \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2557  \u2588\u2588\u2557\u2588\u2588\u2557   \u2588\u2588\u2557\u2588\u2588\u2557      \u2588\u2588\u2557   \u2588\u2588\u2557
+    \u2588\u2588\u2554\u2550\u2550\u2550\u2550\u255D\u255A\u2588\u2588\u2557\u2588\u2588\u2554\u255D\u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+    \u2588\u2588\u2588\u2588\u2588\u2557   \u255A\u2588\u2588\u2588\u2554\u255D \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+    \u2588\u2588\u2554\u2550\u2550\u255D   \u2588\u2588\u2554\u2588\u2588\u2557 \u2588\u2588\u2551   \u2588\u2588\u2551\u2588\u2588\u2551      \u2588\u2588\u2551   \u2588\u2588\u2551
+    \u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u2588\u2588\u2554\u255D \u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2557\u255A\u2588\u2588\u2588\u2588\u2588\u2588\u2554\u255D
+    \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D\u255A\u2550\u255D  \u255A\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u2550\u255D \u255A\u2550\u2550\u2550\u2550\u2550\u255D 
+    `);
+  console.log("Agents:");
+  console.table(agents.map((agent) => {
+    return {
+      id: agent.id,
+      name: agent.name,
+      description: agent.description,
+      slug: "/agents/" + agent.id,
+      active: true
+    };
+  }));
+  console.log("Contexts:");
+  console.table(contexts.map((context) => {
+    const sources = context.sources.get();
+    return {
+      id: context.id,
+      name: context.name,
+      description: context.description,
+      embedder: context.embedder.name,
+      slug: "/contexts/" + context.id,
+      active: context.active,
+      sources: Array.isArray(sources) ? sources.length : 0,
+      sources_details: Array.isArray(sources) ? sources.map((source) => `${source.name} (${source.id})`).join(", ") : "No sources"
+    };
+  }));
+  routeLogs.push(
+    { route: "/agents", method: "GET", note: "List all agents" },
+    { route: "/agents/:id", method: "GET", note: "Get specific agent" },
+    { route: "/workflows", method: "GET", note: "List all workflows" },
+    { route: "/workflows/:id", method: "GET", note: "Get specific workflow" },
+    { route: "/contexts", method: "GET", note: "List all contexts" },
+    { route: "/contexts/:id", method: "GET", note: "Get specific context" },
+    { route: "/contexts/statistics", method: "GET", note: "Get context statistics" },
+    { route: "/tools", method: "GET", note: "List all tools" },
+    { route: "/tools/:id", method: "GET", note: "Get specific tool" },
+    { route: "/statistics/timeseries", method: "POST", note: "Get time series statistics" },
+    { route: "/statistics/totals", method: "POST", note: "Get totals statistics" },
+    { route: "/items/:context", method: "POST", note: "Create new item in context" },
+    { route: "/items/:context", method: "GET", note: "Get items from context" },
+    { route: "/items/export/:context", method: "GET", note: "Export items from context" },
+    { route: "/graphql", method: "POST", note: "GraphQL endpoint" }
+  );
+  await createRecurringJobs();
+  const schema = createSDL([usersSchema, rolesSchema, agentsSchema, jobsSchema]);
+  const server = new import_server3.ApolloServer({ schema, introspection: true });
+  await server.start();
+  app.use(
+    "/graphql",
+    (0, import_cors.default)(),
+    import_express4.default.json(),
+    (0, import_express5.expressMiddleware)(server, {
+      context: async ({ req }) => {
+        const authenticationResult = await requestValidators.authenticate(req);
+        if (!authenticationResult.user?.id) {
+          throw new Error(authenticationResult.message);
+        }
+        const { db: db2 } = await postgresClient();
+        return {
+          req,
+          db: db2
+        };
+      }
+    })
+  );
+  app.get(`/agents`, async (req, res) => {
+    res.status(200).json(agents);
+  });
+  app.get(`/agents/:id`, async (req, res) => {
+    const { db: db2 } = await postgresClient();
+    const id = req.params.id;
+    if (!id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const agent = await db2.from("agents").where({ id }).first();
+    if (!agent) {
+      res.status(400).json({
+        message: "Agent not found in database."
+      });
+      return;
+    }
+    console.log("[EXULU] agent", agent);
+    const backend = agents.find((a) => a.id === agent.backend);
+    res.status(200).json({
+      ...{
+        name: agent.name,
+        id: agent.id,
+        description: agent.description,
+        active: agent.active,
+        public: agent.public,
+        slug: backend?.slug,
+        rateLimit: backend?.rateLimit,
+        streaming: backend?.streaming,
+        capabilities: backend?.capabilities,
+        // todo add contexts
+        availableTools: backend?.tools,
+        enabledTools: agent.tools
+      }
+    });
+  });
+  console.log("tools", tools);
+  app.get("/tools", async (req, res) => {
+    res.status(200).json(tools.map((tool) => ({
+      id: tool.id,
+      name: tool.name,
+      description: tool.description,
+      type: tool.type || "tool",
+      inputSchema: tool.inputSchema ? (0, import_zodex.zerialize)(tool.inputSchema) : null,
+      outputSchema: tool.outputSchema ? (0, import_zodex.zerialize)(tool.outputSchema) : null
+    })));
+  });
+  app.get("/tools/:id", async (req, res) => {
+    const id = req.params.id;
+    if (!id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const tool = tools.find((tool2) => tool2.id === id);
+    if (!tool) {
+      res.status(400).json({
+        message: "Tool not found."
+      });
+      return;
+    }
+    res.status(200).json(tool);
+  });
+  const deleteItem = async ({
+    id,
+    external_id,
+    contextId
+  }) => {
+    if (!contextId) {
+      throw new Error("Missing context in request.");
+    }
+    if (!id && !external_id) {
+      throw new Error("Missing id or external_id in request.");
+    }
+    const { db: db2 } = await postgresClient();
+    const context = contexts.find((context2) => context2.id === contextId);
+    if (!context) {
+      throw new Error("Context not found in registry.");
+    }
+    const exists = await context.tableExists();
+    if (!exists) {
+      throw new Error("Table with name " + context.getTableName() + " does not exist.");
+    }
+    const mutation = db2.from(context.getTableName()).delete().returning("id");
+    if (id) {
+      mutation.where({ id });
+    }
+    if (external_id) {
+      mutation.where({ external_id });
+    }
+    const result = await mutation;
+    return result;
+  };
+  app.delete("/items/:context/:id", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const result = await deleteItem({
+      id: req.params.id,
+      contextId: req.params.context
+    });
+    res.status(200).json(result);
+  });
+  app.delete("/items/:context/external/:id", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const result = await deleteItem({
+      external_id: req.params.id,
+      contextId: req.params.context
+    });
+    res.status(200).json(result);
+  });
+  app.get("/items/:context/:id", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    if (!req.params.id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const { db: db2 } = await postgresClient();
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const item = await db2.from(context.getTableName()).where({ id: req.params.id }).select("*").first();
+    if (!item) {
+      res.status(404).json({
+        message: "Item not found."
+      });
+      return;
+    }
+    console.log("[EXULU] chunks table name.", context.getChunksTableName());
+    const chunks = await db2.from(context.getChunksTableName()).where({ source: req.params.id }).select("id", "content", "source", "embedding", "chunk_index", "created_at", "updated_at");
+    console.log("[EXULU] chunks", chunks);
+    res.status(200).json({
+      ...item,
+      chunks: chunks.map((chunk) => ({
+        id: chunk.id,
+        content: chunk.content,
+        source: chunk.source,
+        index: chunk.chunk_index,
+        embedding: chunk.embedding?.length > 0 ? JSON.parse(chunk.embedding)?.length : null,
+        createdAt: chunk.created_at,
+        updatedAt: chunk.updated_at
+      }))
+    });
+  });
+  app.post("/items/:context/:id", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    if (!req.params.id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const exists = await context.tableExists();
+    if (!exists) {
+      await context.createItemsTable();
+    }
+    const result = await context.updateItem(authenticationResult.user.id, req.params.id, req.body);
+    res.status(200).json({
+      message: "Item updated successfully.",
+      id: result
+    });
+  });
+  app.post("/items/:context", async (req, res) => {
+    try {
+      if (!req.params.context) {
+        res.status(400).json({
+          message: "Missing context in request."
+        });
+        return;
+      }
+      const authenticationResult = await requestValidators.authenticate(req);
+      if (!authenticationResult.user?.id) {
+        res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+        return;
+      }
+      const context = contexts.find((context2) => context2.id === req.params.context);
+      if (!context) {
+        res.status(400).json({
+          message: "Context not found in registry."
+        });
+        return;
+      }
+      const exists = await context.tableExists();
+      if (!exists) {
+        await context.createItemsTable();
+      }
+      const result = await context.insertItem(authenticationResult.user.id, req.body, !!req.body.upsert);
+      res.status(200).json({
+        message: "Item created successfully.",
+        id: result
+      });
+    } catch (error) {
+      res.status(500).json({
+        message: error?.message || "An error occurred while creating the item."
+      });
+    }
+  });
+  app.get("/items/:context", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    let limit = req.query.limit ? parseInt(req.query.limit) : 10;
+    let page = req.query.page ? parseInt(req.query.page) : 1;
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const exists = await context.tableExists();
+    if (!exists) {
+      await context.createItemsTable();
+    }
+    if (req.query.method && !Object.values(VectorMethodEnum).includes(req.query.method)) {
+      res.status(400).json({
+        message: "Invalid vector lookup method, must be one of: " + Object.values(VectorMethodEnum).join(", ")
+      });
+      return;
+    }
+    const result = await context.getItems({
+      page,
+      limit,
+      archived: req.query.archived === "true",
+      name: typeof req.query.name === "string" ? req.query.name : void 0,
+      method: req.query.method ? req.query.method : void 0,
+      query: req.query.query ? req.query.query : void 0,
+      statistics: {
+        label: context.name,
+        trigger: "api"
+      }
+    });
+    res.status(200).json(result);
+  });
+  routeLogs.push({
+    route: "/items/:context",
+    method: "DELETE",
+    note: `Delete all embeddings for a context.`
+  });
+  app.delete(`items/:context`, async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    await context.deleteAll();
+    res.status(200).json({
+      message: "All embeddings deleted."
+    });
+  });
+  routeLogs.push({
+    route: `/items/:context/:id`,
+    method: "DELETE",
+    note: `Delete specific embedding for a context.`
+  });
+  app.delete(`items/:context/:id`, async (req, res) => {
+    const id = req.params.id;
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    if (!id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    await context.deleteOne(id);
+    res.status(200).json({
+      message: "Embedding deleted."
+    });
+  });
+  app.post("/statistics/timeseries", async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db2 } = await postgresClient();
+    const type = req.body.type;
+    if (!Object.values(STATISTICS_TYPE_ENUM).includes(type)) {
+      res.status(400).json({
+        message: "Invalid type, must be one of: " + Object.values(STATISTICS_TYPE_ENUM).join(", ")
+      });
+      return;
+    }
+    let from = new Date(req.body.from);
+    let to = new Date(req.body.to);
+    if (!from || !to) {
+      from = new Date(Date.now() - 7 * 24 * 60 * 60 * 1e3);
+      to = /* @__PURE__ */ new Date();
+    }
+    const query = db2.from("statistics").select("*");
+    query.where("name", "count");
+    query.andWhere("type", type);
+    query.andWhere("createdAt", ">=", from);
+    query.andWhere("createdAt", "<=", to);
+    const results = await query;
+    const dates = [];
+    for (let i = 0; i < (to.getTime() - from.getTime()) / (1e3 * 60 * 60 * 24); i++) {
+      dates.push(new Date(from.getTime() + i * (1e3 * 60 * 60 * 24)));
+    }
+    const data = dates.map((date) => {
+      const result = results.find((result2) => result2.date === date);
+      if (result) {
+        return result;
+      }
+      return {
+        date,
+        count: 0
+      };
+    });
+    res.status(200).json({
+      data,
+      filter: {
+        from,
+        to
+      }
+    });
+  });
+  app.post("/statistics/totals", async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db2 } = await postgresClient();
+    let from = new Date(req.body.from);
+    let to = new Date(req.body.to);
+    if (!from || !to) {
+      from = new Date(Date.now() - 7 * 24 * 60 * 60 * 1e3);
+      to = /* @__PURE__ */ new Date();
+    }
+    let promises2 = Object.values(STATISTICS_TYPE_ENUM).map(async (type) => {
+      const result = await db2.from("statistics").where("name", "count").andWhere("type", type).andWhere("createdAt", ">=", from).andWhere("createdAt", "<=", to).sum("total as total");
+      return {
+        [type]: result[0]?.total || 0
+      };
+    });
+    const results = await Promise.all(promises2);
+    res.status(200).json({
+      data: { ...Object.assign({}, ...results) },
+      filter: {
+        from,
+        to
+      }
+    });
+  });
+  app.get("/contexts/statistics", async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db2 } = await postgresClient();
+    const statistics = await db2("statistics").where("name", "count").andWhere("type", "context.retrieve").sum("total as total").first();
+    const response = await db2("jobs").select(db2.raw(`to_char("createdAt", 'YYYY-MM-DD') as date`)).count("* as count").where("type", "embedder").groupByRaw(`to_char("createdAt", 'YYYY-MM-DD')`).then((rows) => ({
+      jobs: rows
+    }));
+    console.log({ response });
+    let jobs = [];
+    if (response[0]) {
+      jobs = response[0].jobs.map((job) => ({
+        date: job.id,
+        count: job.count
+      }));
+    }
+    const embeddingsCountResult = await db2("jobs").where("type", "embedder").count("* as count").first();
+    res.status(200).json({
+      active: contexts.filter((context) => context.active).length,
+      inactive: contexts.filter((context) => !context.active).length,
+      sources: contexts.reduce((acc, context) => acc + context.sources.get().length, 0),
+      queries: statistics?.total || 0,
+      jobs,
+      totals: {
+        embeddings: embeddingsCountResult?.count || 0
+      }
+    });
+  });
+  app.get(`/contexts/:id`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const id = req.params.id;
+    if (!id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === id);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found."
+      });
+      return;
+    }
+    res.status(200).json({
+      ...{
+        id: context.id,
+        name: context.name,
+        description: context.description,
+        embedder: context.embedder.name,
+        slug: "/contexts/" + context.id,
+        active: context.active,
+        fields: context.fields,
+        sources: context.sources.get().map((source) => ({
+          id: source.id,
+          name: source.name,
+          description: source.description,
+          updaters: source.updaters.map((updater) => ({
+            id: updater.id,
+            slug: updater.slug,
+            type: updater.type,
+            configuration: updater.configuration
+          }))
+        }))
+      },
+      agents: []
+      // todo
+    });
+  });
+  app.get(`/items/export/:context`, async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found."
+      });
+      return;
+    }
+    const items = await context.getItems({
+      page: 1,
+      // todo add pagination
+      limit: 500
+    });
+    const csv = Papa.unparse(items);
+    const ISOTime = (/* @__PURE__ */ new Date()).toISOString();
+    res.status(200).attachment(`${context.name}-items-export-${ISOTime}.csv`).send(csv);
+  });
+  app.get(`/contexts`, async (req, res) => {
+    console.log("contexts!!");
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    console.log("contexts", contexts?.length);
+    res.status(200).json(contexts.map((context) => ({
+      id: context.id,
+      name: context.name,
+      description: context.description,
+      embedder: context.embedder.name,
+      slug: "/contexts/" + context.id,
+      active: context.active,
+      fields: context.fields,
+      sources: context.sources.get().map((source) => ({
+        id: source.id,
+        name: source.name,
+        description: source.description,
+        updaters: source.updaters.map((updater) => ({
+          id: updater.id,
+          slug: updater.slug,
+          type: updater.type,
+          configuration: updater.configuration
+        }))
+      }))
+    })));
+  });
+  app.get(`/workflows`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    res.status(200).json(workflows.map((workflow) => ({
+      id: workflow.id,
+      name: workflow.name,
+      slug: workflow.slug,
+      enable_batch: workflow.enable_batch,
+      queue: workflow.queue?.name,
+      inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null
+    })));
+  });
+  app.get(`/workflows/:id`, async (req, res) => {
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const id = req.params.id;
+    if (!id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const workflow = workflows.find((workflow2) => workflow2.id === id);
+    if (!workflow) {
+      res.status(400).json({
+        message: "Workflow not found."
+      });
+      return;
+    }
+    res.status(200).json({
+      ...workflow,
+      queue: workflow.queue?.name,
+      inputSchema: workflow.inputSchema ? (0, import_zodex.zerialize)(workflow.inputSchema) : null,
+      workflow: void 0
+    });
+  });
+  contexts.forEach((context) => {
+    const sources = context.sources.get();
+    if (!Array.isArray(sources)) {
+      return;
+    }
+    sources.forEach((source) => {
+      source.updaters.forEach((updater) => {
+        if (!updater.slug) return;
+        if (updater.type === "webhook" || updater.type === "manual") {
+          routeLogs.push({
+            route: `${updater.slug}/${updater.type}/:context`,
+            method: "POST",
+            note: `Webhook updater for ${context.name}`
+          });
+          app.post(`${updater.slug}/${updater.type}/:context`, async (req, res) => {
+            const { context: id } = req.params;
+            if (!id) {
+              res.status(400).json({
+                message: "Missing context id in request."
+              });
+              return;
+            }
+            const context2 = contexts.find((context3) => context3.id === id);
+            if (!context2) {
+              res.status(400).json({
+                message: `Context for provided id: ${id} not found.`
+              });
+              return;
+            }
+            if (!context2.embedder.queue) {
+              res.status(500).json({ detail: "No queue set for embedder." });
+              return;
+            }
+            const authenticationResult = await requestValidators.authenticate(req);
+            if (!authenticationResult.user?.id) {
+              res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+              return;
+            }
+            const requestValidationResult = requestValidators.embedders(req, updater.configuration);
+            if (requestValidationResult.error) {
+              res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
+              return;
+            }
+            const documents = await updater.fn(req.body.configuration);
+            const batches = [];
+            for (let i = 0; i < documents.length; i += context2.embedder.batchSize) {
+              batches.push(documents.slice(i, i + context2.embedder.batchSize));
+            }
+            let promises2 = [];
+            if (batches.length > 0) {
+              promises2 = batches.map((documents2) => {
+                return bullmqDecorator({
+                  label: `Job running context '${context2.name}' with embedder '${context2.embedder.name}' for '${req.body.label}'`,
+                  type: "embedder",
+                  embedder: context2.embedder.id,
+                  updater: updater.id,
+                  context: context2.id,
+                  trigger: updater.type,
+                  source: source.id,
+                  inputs: req.body.inputs,
+                  ...updater.configuration && { configuration: req.body.configuration },
+                  documents: documents2,
+                  queue: context2.embedder.queue,
+                  user: authenticationResult.user.id
+                });
+              });
+            }
+            const jobs = await Promise.all(promises2);
+            res.status(200).json(jobs);
+            return;
+          });
+        }
+      });
+    });
+  });
+  agents.forEach((agent) => {
+    const slug = agent.slug;
+    if (!slug) return;
+    routeLogs.push({
+      route: slug + "/:instance",
+      method: "POST",
+      note: `Agent endpoint for ${agent.id}`
+    });
+    app.post(slug + "/:instance", async (req, res) => {
+      const instance = req.params.instance;
+      if (!instance) {
+        res.status(400).json({
+          message: "Missing instance in request."
+        });
+        return;
+      }
+      const { db: db2 } = await postgresClient();
+      const agentInstance = await db2.from("agents").where({
+        id: instance
+      }).first();
+      if (!agentInstance) {
+        res.status(400).json({
+          message: "Agent instance not found."
+        });
+        return;
+      }
+      if (agent.rateLimit) {
+        const limit = await rateLimiter(
+          agent.rateLimit.name || agent.id,
+          agent.rateLimit.rate_limit.time,
+          agent.rateLimit.rate_limit.limit,
+          1
+        );
+        if (!limit.status) {
+          res.status(429).json({
+            message: "Rate limit exceeded.",
+            retryAfter: limit.retryAfter
+          });
+          return;
+        }
+      }
+      const stream = req.headers["stream"] || false;
+      const requestValidationResult = requestValidators.agents(req);
+      if (requestValidationResult.error) {
+        res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
+        return;
+      }
+      const authenticationResult = await requestValidators.authenticate(req);
+      if (!authenticationResult.user?.id) {
+        res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+        return;
+      }
+      if (!!stream) {
+        const chatClient = await agent.chat(agentInstance.id);
+        if (!chatClient) {
+          res.status(500).json({
+            message: "Agent instantiation not successful."
+          });
+          return;
+        }
+        const { textStream } = await chatClient.stream(req.body.messages, {
+          threadId: `${req.body.threadId}`,
+          // conversation id
+          resourceId: `${req.body.resourceId}`,
+          // user id
+          ...agent.outputSchema && { output: agent.outputSchema },
+          maxRetries: 2,
+          // todo make part of ExuluAgent class
+          maxSteps: 5,
+          // todo make part of ExuluAgent class
+          onError: (error) => console.error("[EXULU] chat stream error.", error),
+          onFinish: ({ response, usage }) => console.info(
+            "[EXULU] chat stream finished.",
+            usage
+          )
+        });
+        for await (const delta of textStream) {
+          res.write(`data: ${delta}
+
+`);
+        }
+        res.end();
+        return;
+      } else {
+        const response = await agent.chat.generate(req.body.messages, {
+          resourceId: `${authenticationResult.user.id}`,
+          output: agent.outputSchema,
+          threadId: `${req.body.threadId}`,
+          // conversation id
+          maxRetries: 2,
+          // todo make part of ExuluAgent class
+          maxSteps: 5
+          // todo make part of ExuluAgent class
+        });
+        res.status(200).json(response);
+        return;
+      }
+    });
+  });
+  workflows.forEach((workflow) => {
+    routeLogs.push({
+      route: workflow.slug,
+      method: "POST",
+      note: `Execute workflow ${workflow.name}`
+    });
+    app.post(`${workflow.slug}`, async (req, res) => {
+      if (!workflow.queue) {
+        res.status(500).json({ detail: "No queue set for workflow." });
+        return;
+      }
+      const authenticationResult = await requestValidators.authenticate(req);
+      if (!authenticationResult.user?.id) {
+        res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+        return;
+      }
+      const requestValidationResult = requestValidators.workflows(req);
+      if (requestValidationResult.error) {
+        res.status(requestValidationResult.code || 500).json({ detail: `${requestValidationResult.message}` });
+        return;
+      }
+      const inputs = await preprocessInputs(req.body.inputs);
+      if (workflow.queue) {
+        const job = await bullmqDecorator({
+          label: `Job running '${workflow.name}' for '${req.body.label}'`,
+          agent: req.body.agent,
+          workflow: workflow.id,
+          type: "workflow",
+          inputs,
+          session: req.body.session,
+          queue: workflow.queue,
+          user: authenticationResult.user.id
+        });
+        res.status(200).json({
+          "job": {
+            "status": "waiting",
+            "name": job.name,
+            "queue": workflow.queue.name,
+            "redisId": job.redis,
+            "jobId": job.id
+          },
+          "output": {}
+        });
+        return;
+      }
+      const { runId, start, watch } = workflow.workflow.createRun();
+      console.log("[EXULU] running workflow with inputs.", inputs);
+      const output = await start({
+        triggerData: {
+          ...inputs,
+          user: authenticationResult.user.id
+        }
+      });
+      const failedSteps = Object.entries(output.results).filter(([_, step]) => step.status === "failed").map(([id, step]) => `${id}: ${step.error}`);
+      if (failedSteps.length > 0) {
+        const message = `Workflow has failed steps: ${failedSteps.join("\n - ")}`;
+        throw new Error(message);
+      }
+      res.status(200).json({
+        "job": {},
+        "output": output
+      });
+      return;
+    });
+  });
+  await createUppyRoutes(app);
+  console.log("Routes:");
+  console.table(routeLogs);
+};
+var preprocessInputs = async (data) => {
+  for (const key in data) {
+    if (key.includes("exulu_file_")) {
+      const url = await getPresignedFileUrl(data[key]);
+      const newKey = key.replace("exulu_file_", "");
+      data[newKey] = url;
+      delete data[key];
+    } else if (Array.isArray(data[key])) {
+      for (let i = 0; i < data[key].length; i++) {
+        if (typeof data[key][i] === "object") {
+          await preprocessInputs(data[key][i]);
+        }
+      }
+    } else if (typeof data[key] === "object") {
+      await preprocessInputs(data[key]);
+    }
+  }
+  return data;
+};
+var getPresignedFileUrl = async (key) => {
+  if (!process.env.NEXT_PUBLIC_UPLOAD_URL) {
+    throw new Error("Missing process.env.NEXT_PUBLIC_UPLOAD_URL");
+  }
+  if (!process.env.INTERNAL_SECRET) {
+    throw new Error("Missing process.env.NEXT_PUBLIC_UPLOAD_URL");
+  }
+  console.log(`[EXULU] fetching presigned url for file with key: ${key}`);
+  let url = `${process.env.NEXT_PUBLIC_UPLOAD_URL}/s3/download?key=${key}`;
+  const response = await fetch(url, {
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      "Internal-Key": process.env.INTERNAL_SECRET
+    }
+  });
+  const json = await response.json();
+  if (!json.url) {
+    throw new Error(`Could not generate presigned url for file with key: ${key}`);
+  }
+  console.log(`[EXULU] presigned url for file with key: ${key}, generated: ${json.url}`);
+  return json.url;
+};
+
+// src/registry/workers.ts
+var import_ioredis = __toESM(require("ioredis"), 1);
+var import_bullmq8 = require("bullmq");
+
+// src/registry/utils.ts
+var import_bullmq7 = require("bullmq");
+var bullmq = {
+  validate: (job) => {
+    if (!job.data) {
+      throw new Error(`Missing job data for job ${job.id}.`);
+    }
+    if (!job.data.type) {
+      throw new Error(`Missing property "type" in data for job ${job.id}.`);
+    }
+    if (!job.data.inputs) {
+      throw new Error(`Missing property "inputs" in data for job ${job.id}.`);
+    }
+    if (job.data.type !== "embedder" && job.data.type !== "workflow") {
+      throw new Error(`Property "type" in data for job ${job.id} must be of value "embedder" or "workflow".`);
+    }
+    if (!job.data.workflow && !job.data.embedder) {
+      throw new Error(`Property "backend" in data for job ${job.id} missing. Job  data: ${JSON.stringify(job)}`);
+    }
+  },
+  process: {
+    workflow: async (job, workflow, logsDir) => {
+      if (!workflow) {
+        throw new Error(`Workflow function with id: ${job.data.backend} not found in registry.`);
+      }
+      const { runId, start, watch } = workflow.workflow.createRun();
+      console.log("[EXULU] starting workflow with job inputs.", job.data.inputs);
+      const logger = new ExuluLogger(job, logsDir);
+      const output = await start({ triggerData: {
+        ...job.data.inputs,
+        redis: job.id,
+        logger
+      } });
+      const failedSteps = Object.entries(output.results).filter(([_, step]) => step.status === "failed").map(([id, step]) => `${id}: ${step.error}`);
+      if (failedSteps.length > 0) {
+        const message = `Workflow has failed steps: ${failedSteps.join("\n - ")}`;
+        logger.write(message, "ERROR");
+        throw new Error(message);
+      }
+      await logger.write(`Workflow completed. ${JSON.stringify(output.results)}`, "INFO");
+      return output;
+    }
+  }
+};
+
+// src/registry/workers.ts
+var fs2 = __toESM(require("fs"), 1);
+var import_path = __toESM(require("path"), 1);
+var defaultLogsDir = import_path.default.join(process.cwd(), "logs");
+var redisConnection = new import_ioredis.default({
+  ...redisServer,
+  maxRetriesPerRequest: null
+});
+var createWorkers = async (queues2, contexts, embedders, workflows, _logsDir) => {
+  const logsDir = _logsDir || defaultLogsDir;
+  const workers = queues2.map((queue) => {
+    console.log(`[EXULU] creating worker for queue ${queue}.`);
+    const worker = new import_bullmq8.Worker(
+      `${queue}`,
+      async (job) => {
+        const { db: db2 } = await postgresClient();
+        try {
+          bullmq.validate(job);
+          if (job.data.type === "embedder") {
+            if (!job.data.updater) {
+              throw new Error("No updater set for embedder job.");
+            }
+            const context = contexts.find((context2) => context2.id === job.data.context);
+            if (!context) {
+              throw new Error(`Context ${job.data.context} not found in the registry.`);
+            }
+            if (!job.data.embedder) {
+              throw new Error(`No embedder set for embedder job.`);
+            }
+            const embedder = embedders.find((embedder2) => embedder2.id === job.data.embedder);
+            if (!embedder) {
+              throw new Error(`Embedder ${job.data.embedder} not found in the registry.`);
+            }
+            if (!job.data.source) {
+              throw new Error("No source set for embedder job.");
+            }
+            const source = context.sources.get(job.data.source);
+            if (!source) {
+              throw new Error(`Source ${job.data.source} not found in the registry.`);
+            }
+            if (!job.data.updater) {
+              throw new Error("No updater set for embedder job.");
+            }
+            const updater = source.updaters.find((updater2) => updater2.id === job.data.updater);
+            if (!updater) {
+              throw new Error(`Updater ${job.data.updater} not found in the registry.`);
+            }
+            if (!job.data.documents) {
+              throw new Error("No input documents set for embedder job.");
+            }
+            if (!Array.isArray(job.data.documents)) {
+              throw new Error("Input documents must be an array.");
+            }
+            const result = await embedder.upsert(job.data.context, job.data.documents, {
+              label: context.name,
+              trigger: job.data.trigger || "unknown"
+            });
+            const mongoRecord = await db2.from("jobs").where({ redis: job.id }).first();
+            if (!mongoRecord) {
+              throw new Error("Job not found in the database.");
+            }
+            const finishedAt = /* @__PURE__ */ new Date();
+            const duration = (finishedAt.getTime() - new Date(mongoRecord.createdAt).getTime()) / 1e3;
+            await db2.from("jobs").where({ redis: job.id }).update({
+              status: "completed",
+              finishedAt,
+              duration,
+              result: JSON.stringify(result)
+            });
+            return result;
+          }
+          if (job.data.type === "workflow") {
+            const workflow = workflows.find((workflow2) => workflow2.id === job.data.workflow);
+            if (!workflow) {
+              throw new Error(`Workflow ${job.data.workflow} not found in the registry.`);
+            }
+            const result = await bullmq.process.workflow(job, workflow, logsDir);
+            const mongoRecord = await db2.from("jobs").where({ redis: job.id }).first();
+            if (!mongoRecord) {
+              throw new Error("Job not found in the database.");
+            }
+            const finishedAt = /* @__PURE__ */ new Date();
+            const duration = (finishedAt.getTime() - new Date(mongoRecord.createdAt).getTime()) / 1e3;
+            await db2.from("jobs").where({ redis: job.id }).update({
+              status: "completed",
+              finishedAt,
+              duration,
+              result: JSON.stringify(result)
+            });
+            return result;
+          }
+        } catch (error) {
+          await db2.from("jobs").where({ redis: job.id }).update({
+            status: "failed",
+            finishedAt: /* @__PURE__ */ new Date(),
+            error: error instanceof Error ? error.message : String(error)
+          });
+          throw new Error(error instanceof Error ? error.message : String(error));
+        }
+      },
+      { connection: redisConnection }
+    );
+    worker.on("completed", (job, returnvalue) => {
+      console.log(`[EXULU] completed job ${job.id}.`, returnvalue);
+    });
+    worker.on("failed", (job, error, prev) => {
+      if (job?.id) {
+        console.error(`[EXULU] failed job ${job.id}.`);
+      }
+      console.error(`[EXULU] job error.`, error);
+    });
+    worker.on("progress", (job, progress) => {
+      console.log(`[EXULU] job progress ${job.id}.`, progress);
+    });
+    return worker;
+  });
+  const logsCleaner = createLogsCleanerWorker(logsDir);
+  workers.push(logsCleaner);
+  return workers;
+};
+var createLogsCleanerWorker = (logsDir) => {
+  const logsCleaner = new import_bullmq8.Worker(
+    global_queues.logs_cleaner,
+    async (job) => {
+      console.log(`[EXULU] recurring job ${job.id}.`);
+      const folder = fs2.readdirSync(logsDir);
+      const files = folder.filter((file) => file.endsWith(".log"));
+      const now = /* @__PURE__ */ new Date();
+      const daysToKeep = job.data.ttld;
+      const dateToKeep = new Date(now.getTime() - daysToKeep * 24 * 60 * 60 * 1e3);
+      files.forEach((file) => {
+        const filePath = import_path.default.join(logsDir, file);
+        const fileStats = fs2.statSync(filePath);
+        if (fileStats.mtime < dateToKeep) {
+          fs2.unlinkSync(filePath);
+        }
+      });
+    },
+    { connection: redisConnection }
+  );
+  logsCleaner.on("completed", (job, returnvalue) => {
+    console.log(`[EXULU] completed logs cleaner ${job.id}.`, returnvalue);
+  });
+  logsCleaner.on("failed", (job, error, prev) => {
+    if (job?.id) {
+      console.error(`[EXULU] failed logs cleaner ${job.id}.`);
+    }
+    console.error(`[EXULU] job error logs cleaner.`, error);
+  });
+  return logsCleaner;
+};
+
+// src/registry/index.ts
+var ExuluApp = class {
+  _agents;
+  _workflows;
+  _config;
+  _embedders;
+  _queues = [];
+  _contexts;
+  _tools;
+  constructor({ contexts, embedders, agents, workflows, config, tools }) {
+    this._embedders = embedders ?? [];
+    this._workflows = workflows ?? [];
+    this._contexts = contexts ?? {};
+    this._agents = agents ?? [];
+    this._config = config;
+    this._tools = tools ?? [];
+    const queues2 = [
+      ...embedders?.length ? embedders.map((agent) => agent.queue?.name || null) : [],
+      ...workflows?.length ? workflows.map((workflow) => workflow.queue?.name || null) : []
+    ];
+    this._queues = [...new Set(queues2.filter((o) => !!o))];
+  }
+  embedder(id) {
+    return this._embedders.find((x) => x.id === id);
+  }
+  tool(id) {
+    return this._tools.find((x) => x.id === id);
+  }
+  tools() {
+    return this._tools;
+  }
+  context(id) {
+    return Object.values(this._contexts ?? {}).find((x) => x.id === id);
+  }
+  agent(id) {
+    return this._agents.find((x) => x.id === id);
+  }
+  workflow(id) {
+    return this._workflows.find((x) => x.id === id);
+  }
+  get embedders() {
+    return this._embedders;
+  }
+  get contexts() {
+    return Object.values(this._contexts ?? {});
+  }
+  get workflows() {
+    return this._workflows;
+  }
+  get agents() {
+    return this._agents;
+  }
+  bullmq = {
+    workers: {
+      create: async () => {
+        return await createWorkers(
+          this._queues,
+          Object.values(this._contexts ?? {}),
+          this._embedders,
+          this._workflows,
+          this._config.workers?.logsDir
+        );
+      }
+    }
+  };
+  server = {
+    express: {
+      init: async (app) => {
+        return await createExpressRoutes(
+          app,
+          this._agents,
+          this._embedders,
+          this._tools,
+          this._workflows,
+          Object.values(this._contexts ?? {})
+        );
+      }
+    }
+  };
+};
+
+// src/index.ts
+var ExuluJobs = {
+  redis: redisClient,
+  jobs: {
+    validate: validateJob
+  }
+};
+var ExuluDatabase = {
+  init: async () => {
+    await execute();
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  EXULU_STATISTICS_TYPE_ENUM,
+  ExuluAgent,
+  ExuluApp,
+  ExuluAuthentication,
+  ExuluContext,
+  ExuluDatabase,
+  ExuluEmbedder,
+  ExuluJobs,
+  ExuluQueues,
+  ExuluSource,
+  ExuluTool,
+  ExuluWorkflow,
+  ExuluZodFileType
+});