@exulu/backend 1.46.0 → 1.47.0

package/ee/queues/decorator.ts

@@ -0,0 +1,140 @@
+import { Queue } from "bullmq";
+import { v4 as uuidv4 } from "uuid";
+import type { UIMessage } from "ai";
+import type { STATISTICS_LABELS } from "@EXULU_TYPES/statistics";
+
+type ExuluJobType = "embedder" | "workflow" | "eval" | "processor";
+
+type ExuluBullMqDecoratorData = {
+  queue: Queue;
+  label: string;
+  embedder?: string;
+  processor?: string;
+  inputs: any;
+  user?: number;
+  role?: string;
+  trigger: STATISTICS_LABELS;
+  workflow?: string;
+  evaluation?: string;
+  item?: string;
+  context?: string;
+  retries?: number;
+  backoff?: {
+    type: "exponential" | "linear";
+    delay: number; // in milliseconds
+  };
+  timeoutInSeconds: number;
+};
+
+export type BullMqJobData = {
+  label: string;
+  type: string;
+  source?: string;
+  inputs: any;
+  timeoutInSeconds: number;
+  user?: number;
+  role?: string;
+  trigger: STATISTICS_LABELS;
+  messages?: UIMessage[];
+  eval_run_id?: string;
+  eval_run_name?: string;
+  test_case_id?: string;
+  test_case_name?: string;
+  eval_functions?: {
+    id: string;
+    config: Record<string, any>;
+  }[];
+  agent_id?: string;
+  expected_output?: string;
+  expected_tools?: string[];
+  expected_knowledge_sources?: string[];
+  expected_agent_tools?: string[];
+  config?: Record<string, any>;
+  scoring_method?: string;
+  pass_threshold?: number;
+  workflow?: string;
+  embedder?: string;
+  processor?: string;
+  evaluation?: string;
+  item?: string;
+  context?: string;
+};
+
+export const bullmqDecorator = async ({
+  queue,
+  label,
+  embedder,
+  processor,
+  inputs,
+  evaluation,
+  user,
+  role,
+  trigger,
+  workflow,
+  item,
+  context,
+  retries,
+  backoff,
+  timeoutInSeconds,
+}: ExuluBullMqDecoratorData) => {
+  const types = [embedder, workflow, processor, evaluation];
+
+  if (types.filter((type) => type).length > 1) {
+    throw new Error(
+      "Cannot have multiple types in the same job, must be one of the following: embedder, workflow, processor or eval.",
+    );
+  }
+
+  let type: ExuluJobType = "embedder";
+
+  if (workflow) {
+    type = "workflow";
+  }
+
+  if (processor) {
+    type = "processor";
+  }
+
+  if (evaluation) {
+    type = "eval";
+  }
+
+  if (embedder) {
+    type = "embedder";
+  }
+
+  const jobData: BullMqJobData = {
+    label,
+    type: `${type}`,
+    timeoutInSeconds: timeoutInSeconds || 180, // 3 minutes default
+    inputs,
+    ...(user && { user }),
+    ...(role && { role }),
+    ...(trigger && { trigger }),
+    ...(workflow && { workflow }),
+    ...(embedder && { embedder }),
+    ...(processor && { processor }),
+    ...(evaluation && { evaluation }),
+    ...(item && { item }),
+    ...(context && { context }),
+  };
+
+  const redisId = uuidv4();
+  const job = await queue.add(`${embedder || workflow || processor || evaluation}`, jobData, {
+    jobId: redisId,
+    // Setting it to 3 as a sensible default, as
+    // many AI services are quite unstable.
+    attempts: retries || 3, // todo make this configurable?
+    removeOnComplete: 5000,
+    removeOnFail: 10000,
+    backoff: backoff || {
+      type: "exponential",
+      delay: 2000,
+    },
+  });
+
+  return {
+    ...job,
+    redis: job.id,
+  };
+};

package/ee/queues/queues.ts

@@ -0,0 +1,156 @@
+import { Queue } from "bullmq";
+import { redisServer } from "./server";
+import { BullMQOtel } from "bullmq-otel";
+import type { ExuluQueueConfig } from "@EXULU_TYPES/queue-config";
+import { checkLicense } from "@EE/entitlements";
+
+// Used for workflows and embedders
+class ExuluQueues {
+  queues: {
+    queue: Queue;
+    ratelimit: number;
+    concurrency: {
+      worker: number;
+      queue: number;
+    };
+    timeoutInSeconds: number;
+  }[];
+  constructor() {
+    this.queues = [];
+  }
+
+  public list: Map<
+    string,
+    {
+      name: string;
+      concurrency: {
+        worker: number;
+        queue: number;
+      };
+      ratelimit: number;
+      timeoutInSeconds: number;
+      use: () => Promise<ExuluQueueConfig>;
+    }
+  > = new Map(); // list of queue names
+
+  queue(name: string):
+    | {
+        queue: Queue;
+        ratelimit: number;
+        concurrency: {
+          worker: number;
+          queue: number;
+        };
+      }
+    | undefined {
+    return this.queues.find((x) => x.queue?.name === name) as
+      | {
+          queue: Queue;
+          ratelimit: number;
+          concurrency: {
+            worker: number;
+            queue: number;
+          };
+        }
+      | undefined;
+  }
+
+  // name: string
+  // concurrency: global concurrency for the queue
+  // ratelimit: maximum number of jobs per second
+  // Rate limit is set on workers (see workers.ts), even global rate limits,
+  // that is a bit counter-intuitive. Since queues are registered using .user
+  // method of ExuluQueues we need to store the desired rate limit on the queue
+  // here so we can use the value when creating workers for the queue instance
+  // as there is no way to store a rate limit value natively on a bullm queue.
+  register = (
+    name: string,
+    concurrency: {
+      worker: number;
+      queue: number;
+    },
+    ratelimit: number = 1,
+    timeoutInSeconds: number = 180,
+  ): {
+    use: () => Promise<ExuluQueueConfig>;
+  } => {
+    const license = checkLicense();
+    if (!license["queues"]) {
+      throw new Error(`[EXULU] You are not licensed to use queues so cannot register a queue. Please set your EXULU_ENTERPRISE_LICENSE env variable.`);
+    }
+    const queueConcurrency = concurrency.queue || 1;
+    const workerConcurrency = concurrency.worker || 1;
+
+    const use = async (): Promise<ExuluQueueConfig> => {
+      const existing = this.queues.find((x) => x.queue?.name === name);
+      if (existing) {
+        const globalConcurrency = await existing.queue.getGlobalConcurrency();
+        if (globalConcurrency !== queueConcurrency) {
+          await existing.queue.setGlobalConcurrency(queueConcurrency);
+        }
+        return {
+          queue: existing.queue,
+          ratelimit,
+          concurrency: {
+            worker: workerConcurrency,
+            queue: queueConcurrency,
+          },
+          timeoutInSeconds,
+        };
+      }
+
+      if (!redisServer.host?.length || !redisServer.port?.length) {
+        console.error(
+          `[EXULU] no redis server configured, but you are trying to use a queue ( ${name}), likely in an agent or embedder (look for ExuluQueues.register().use() ).`,
+        );
+        console.error("Stack trace:");
+        console.trace();
+        throw new Error(`[EXULU] no redis server configured.`);
+      }
+
+      const newQueue = new Queue(`${name}`, {
+        connection: {
+          ...redisServer,
+          enableOfflineQueue: false,
+        },
+        telemetry: new BullMQOtel("simple-guide"),
+      });
+      await newQueue.setGlobalConcurrency(queueConcurrency);
+      this.queues.push({
+        queue: newQueue,
+        ratelimit,
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency,
+        },
+        timeoutInSeconds,
+      });
+      return {
+        queue: newQueue,
+        ratelimit,
+        concurrency: {
+          worker: workerConcurrency,
+          queue: queueConcurrency,
+        },
+        timeoutInSeconds,
+      };
+    };
+
+    this.list.set(name, {
+      name,
+      concurrency: {
+        worker: workerConcurrency,
+        queue: queueConcurrency,
+      },
+      ratelimit,
+      timeoutInSeconds,
+      use,
+    });
+
+    return {
+      use,
+    };
+  };
+}
+
+export const queues = new ExuluQueues();

package/ee/queues/server.ts

@@ -0,0 +1,6 @@
+export const redisServer = {
+  host: process.env.REDIS_HOST ?? "",
+  port: (process.env.REDIS_PORT as any) ?? "",
+  password: process.env.REDIS_PASSWORD || undefined,
+  username: process.env.REDIS_USER || "",
+};

package/ee/rbac-resolver.ts

@@ -0,0 +1,51 @@
+import { checkLicense } from "./entitlements";
+
+export const RBACResolver = async (
+  db: any,
+  entityName: string,
+  resourceId: string,
+  rights_mode: string,
+): Promise<{
+  type: string;
+  users: any[];
+  roles: any[];
+}> => {
+
+  // If RBAC is not available
+  // the system defaults to public.
+  const license = checkLicense()
+  if (!license.rbac) {
+    return {
+      type: "public",
+      users: [],
+      roles: []
+    }
+  }
+  // Get RBAC records for this resource
+  const rbacRecords = await db
+    .from("rbac")
+    .where({
+      entity: entityName,
+      target_resource_id: resourceId,
+    })
+    .select("*");
+
+  const users = rbacRecords
+    .filter((r) => r.access_type === "User")
+    ?.map((r) => ({ id: r.user_id, rights: r.rights }));
+
+  const roles = rbacRecords
+    .filter((r) => r.access_type === "Role")
+    ?.map((r) => ({ id: r.role_id, rights: r.rights }));
+
+  // Determine the type based on rights_mode or presence of records
+  let type = rights_mode || "private";
+  if (type === "users" && users.length === 0) type = "private";
+  if (type === "roles" && roles.length === 0) type = "private";
+
+  return {
+    type,
+    users,
+    roles,
+  };
+};

package/ee/rbac-update.ts

@@ -0,0 +1,111 @@
+// Helper function to handle RBAC updates
+import { checkLicense } from "./entitlements.ts";
+
+export const handleRBACUpdate = async (
+  db: any,
+  entityName: string,
+  resourceId: string,
+  rbacData: any,
+  existingRbacRecords: any[],
+): Promise<void> => {
+
+  const license = checkLicense()
+  if (!license.rbac) {
+    console.warn(`[EXULU] You are not licensed to use RBAC.`);
+    return;
+  }
+  
+  const { users = [], roles = [] /* projects = [] */ } = rbacData;
+
+  // Get existing RBAC records if not provided
+  if (!existingRbacRecords) {
+    existingRbacRecords = await db
+      .from("rbac")
+      .where({
+        entity: entityName,
+        target_resource_id: resourceId,
+      })
+      .select("*");
+  }
+
+  // Create sets for comparison
+  const newUserRecords = new Set(users.map((u: any) => `${u.id}:${u.rights}`));
+  const newRoleRecords = new Set(roles.map((r: any) => `${r.id}:${r.rights}`));
+  // const newProjectRecords = new Set(projects.map((p: any) => `${p.id}:${p.rights}`));
+  const existingUserRecords = new Set(
+    existingRbacRecords
+      .filter((r) => r.access_type === "User")
+      .map((r) => `${r.user_id}:${r.rights}`),
+  );
+  const existingRoleRecords = new Set(
+    existingRbacRecords
+      .filter((r) => r.access_type === "Role")
+      .map((r) => `${r.role_id}:${r.rights}`),
+  );
+
+  // Records to create
+  const usersToCreate = users.filter((u: any) => !existingUserRecords.has(`${u.id}:${u.rights}`));
+  const rolesToCreate = roles.filter((r: any) => !existingRoleRecords.has(`${r.id}:${r.rights}`));
+  // const projectsToCreate = projects.filter((p: any) => !existingProjectRecords.has(`${p.id}:${p.rights}`));
+
+  // Records to remove
+  const usersToRemove = existingRbacRecords.filter(
+    (r) => r.access_type === "User" && !newUserRecords.has(`${r.user_id}:${r.rights}`),
+  );
+  const rolesToRemove = existingRbacRecords.filter(
+    (r) => r.access_type === "Role" && !newRoleRecords.has(`${r.role_id}:${r.rights}`),
+  );
+  // const projectsToRemove = existingRbacRecords
+  //     .filter(r => r.access_type === 'Project' && !newProjectRecords.has(`${r.project_id}:${r.rights}`));
+
+  // Remove obsolete records
+  if (usersToRemove.length > 0) {
+    await db
+      .from("rbac")
+      .whereIn(
+        "id",
+        usersToRemove.map((r) => r.id),
+      )
+      .del();
+  }
+  if (rolesToRemove.length > 0) {
+    await db
+      .from("rbac")
+      .whereIn(
+        "id",
+        rolesToRemove.map((r) => r.id),
+      )
+      .del();
+  }
+
+  // Create new records
+  const recordsToInsert: any[] = [];
+
+  usersToCreate.forEach((user: any) => {
+    recordsToInsert.push({
+      entity: entityName,
+      access_type: "User",
+      target_resource_id: resourceId,
+      user_id: user.id,
+      rights: user.rights,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    });
+  });
+
+  rolesToCreate.forEach((role: any) => {
+    recordsToInsert.push({
+      entity: entityName,
+      access_type: "Role",
+      target_resource_id: resourceId,
+      role_id: role.id,
+      rights: role.rights,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    });
+  });
+
+  if (recordsToInsert.length > 0) {
+    await db.from("rbac").insert(recordsToInsert);
+  }
+};