@exulu/backend 1.18.0 → 1.19.0

package/dist/index.cjs

@@ -531,6 +531,9 @@ var ExuluZodFileType = ({
   });
 };
 var ExuluAgent = class {
+  // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
+  // underscores and be a max length of 80 characters and at least 5 characters long.
+  // The ID is used for storing references to agents so it is important it does not change.
   id;
   name;
   description = "";
@@ -938,38 +941,10 @@ var ExuluEval = class {
     }
   };
 };
-var calculateStatistics = (items, method) => {
-  let methodProperty = "";
-  if (method === "l1Distance") {
-    methodProperty = "l1_distance";
-  }
-  if (method === "l2Distance") {
-    methodProperty = "l2_distance";
-  }
-  if (method === "hammingDistance") {
-    methodProperty = "hamming_distance";
-  }
-  if (method === "jaccardDistance") {
-    methodProperty = "jaccard_distance";
-  }
-  if (method === "maxInnerProduct") {
-    methodProperty = "inner_product";
-  }
-  if (method === "cosineDistance") {
-    methodProperty = "cosine_distance";
-  }
-  const average = items.reduce((acc, item) => {
-    return acc + item[methodProperty];
-  }, 0) / items.length;
-  const total = items.reduce((acc, item) => {
-    return acc + item[methodProperty];
-  }, 0);
-  return {
-    average,
-    total
-  };
-};
 var ExuluTool = class {
+  // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
+  // underscores and be a max length of 80 characters and at least 5 characters long.
+  // The ID is used for storing references to tools so it is important it does not change.
   id;
   name;
   description;
@@ -991,13 +966,16 @@ var ExuluTool = class {
     });
   }
 };
-var getTableName = (name) => {
-  return sanitizeName(name) + "_items";
+var getTableName = (id) => {
+  return sanitizeName(id) + "_items";
 };
-var getChunksTableName = (name) => {
-  return sanitizeName(name) + "_chunks";
+var getChunksTableName = (id) => {
+  return sanitizeName(id) + "_chunks";
 };
 var ExuluContext = class {
+  // Must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or 
+  // underscores and be a max length of 80 characters and at least 5 characters long.
+  // The ID is used for the table name in the database, so it is important it does not change.
   id;
   name;
   active;
@@ -1014,7 +992,9 @@ var ExuluContext = class {
     this.name = name;
     this.fields = fields || [];
     this.configuration = configuration || {
-      calculateVectors: "manual"
+      calculateVectors: "manual",
+      language: "english",
+      defaultRightsMode: "private"
     };
     this.description = description;
     this.embedder = embedder;
@@ -1025,8 +1005,8 @@ var ExuluContext = class {
   }
   deleteAll = async () => {
     const { db: db3 } = await postgresClient();
-    await db3.from(getTableName(this.name)).delete();
-    await db3.from(getChunksTableName(this.name)).delete();
+    await db3.from(getTableName(this.id)).delete();
+    await db3.from(getChunksTableName(this.id)).delete();
     return {
       count: 0,
       results: []
@@ -1034,38 +1014,50 @@ var ExuluContext = class {
   };
   tableExists = async () => {
     const { db: db3 } = await postgresClient();
-    const tableExists = await db3.schema.hasTable(getTableName(this.name));
+    const tableName = getTableName(this.id);
+    console.log("[EXULU] checking if table exists.", tableName);
+    const tableExists = await db3.schema.hasTable(tableName);
     return tableExists;
   };
-  createAndUpsertEmbeddings = async (item, user, statistics, role) => {
+  chunksTableExists = async () => {
+    const { db: db3 } = await postgresClient();
+    const chunksTableName = getChunksTableName(this.id);
+    const chunksTableExists = await db3.schema.hasTable(chunksTableName);
+    return chunksTableExists;
+  };
+  createAndUpsertEmbeddings = async (item, user, statistics, role, job) => {
     if (!this.embedder) {
       throw new Error("Embedder is not set for this context.");
     }
+    if (!item.id) {
+      throw new Error("Item id is required for generating embeddings.");
+    }
     const { db: db3 } = await postgresClient();
     const { id: source, chunks } = await this.embedder.generateFromDocument({
       ...item,
       id: item.id
     }, {
-      label: statistics.label || this.name,
-      trigger: statistics.trigger || "agent"
+      label: statistics?.label || this.name,
+      trigger: statistics?.trigger || "agent"
     }, user, role);
-    const exists = await db3.schema.hasTable(getChunksTableName(this.name));
+    const exists = await db3.schema.hasTable(getChunksTableName(this.id));
     if (!exists) {
       await this.createChunksTable();
     }
-    await db3.from(getChunksTableName(this.name)).where({ source }).delete();
-    await db3.from(getChunksTableName(this.name)).insert(chunks.map((chunk) => ({
+    await db3.from(getChunksTableName(this.id)).where({ source }).delete();
+    await db3.from(getChunksTableName(this.id)).insert(chunks.map((chunk) => ({
       source,
       content: chunk.content,
       chunk_index: chunk.index,
       embedding: import_knex4.default.toSql(chunk.vector)
     })));
-    await db3.from(getTableName(this.name)).where({ id: item.id }).update({
+    await db3.from(getTableName(this.id)).where({ id: item.id }).update({
       embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
     }).returning("id");
     return {
       id: item.id,
-      chunks: chunks?.length || 0
+      chunks: chunks?.length || 0,
+      job
     };
   };
   async updateItem(user, item, role, trigger) {
@@ -1091,43 +1083,105 @@ var ExuluContext = class {
     delete payloadWithSanitizedPropertyNames.created_at;
     delete payloadWithSanitizedPropertyNames.upsert;
     payloadWithSanitizedPropertyNames.updated_at = db3.fn.now();
-    const result = await db3.from(getTableName(this.name)).where({ id }).update(payloadWithSanitizedPropertyNames).returning("id");
+    const result = await db3.from(getTableName(this.id)).where({ id }).update(payloadWithSanitizedPropertyNames).returning("id");
     if (this.embedder && (this.configuration.calculateVectors === "onUpdate" || this.configuration.calculateVectors === "always")) {
-      if (this.embedder.queue?.name) {
-        console.log("[EXULU] embedder is in queue mode, scheduling job.");
-        const job = await bullmqDecorator({
-          label: `${this.embedder.name}`,
-          embedder: this.embedder.id,
-          context: this.id,
-          inputs: item,
-          item: item.id,
-          queue: this.embedder.queue,
-          user,
-          role,
-          trigger: trigger || "agent"
-        });
-        return {
-          id: result[0].id,
-          job: job.id
-        };
-      }
-      await this.createAndUpsertEmbeddings(item, user, {
-        label: this.embedder.name,
+      const { job } = await this.embeddings.generate.one({
+        item: {
+          ...item,
+          id: result[0].id
+        },
+        user,
+        role,
         trigger: trigger || "agent"
-      }, role);
+      });
+      return {
+        id: result[0].id,
+        job
+      };
     }
     return {
       id: result[0].id,
       job: void 0
     };
   }
+  embeddings = {
+    generate: {
+      one: async ({
+        item,
+        user,
+        role,
+        trigger
+      }) => {
+        if (!this.embedder) {
+          throw new Error("Embedder is not set for this context.");
+        }
+        if (!item.id) {
+          throw new Error("Item id is required for generating embeddings.");
+        }
+        if (this.embedder.queue?.name) {
+          console.log("[EXULU] embedder is in queue mode, scheduling job.");
+          const job = await bullmqDecorator({
+            label: `${this.embedder.name}`,
+            embedder: this.embedder.id,
+            context: this.id,
+            inputs: item,
+            item: item.id,
+            queue: this.embedder.queue,
+            user,
+            role,
+            trigger: trigger || "agent"
+          });
+          return {
+            id: item.id,
+            job: job.id,
+            chunks: 0
+          };
+        }
+        return await this.createAndUpsertEmbeddings(item, user, {
+          label: this.embedder.name,
+          trigger: trigger || "agent"
+        }, role, void 0);
+      },
+      all: async (context, userId, roleId) => {
+        const exists = await context.tableExists();
+        if (!exists) {
+          await context.createItemsTable();
+        }
+        const chunksTableExists = await context.chunksTableExists();
+        if (!chunksTableExists && context.embedder) {
+          await context.createChunksTable();
+        }
+        const { db: db3 } = await postgresClient();
+        const items = await db3.from(getTableName(context.id)).select("*");
+        const jobs = [];
+        if (!context.embedder?.queue?.name && items.length > 2e3) {
+          throw new Error(`Embedder is not in queue mode, cannot generate embeddings for more than 
+                        2.000 items at once, if you need to generate embeddings for more items please configure 
+                        the embedder to use a queue. You can configure the embedder to use a queue by setting 
+                        the queue property in the embedder configuration.`);
+        }
+        for (const item of items) {
+          const { job } = await context.embeddings.generate.one({
+            item,
+            user: userId,
+            role: roleId,
+            trigger: "api"
+          });
+          if (job) {
+            jobs.push(job);
+          }
+        }
+        return jobs || [];
+      }
+    }
+  };
   async insertItem(user, item, upsert = false, role, trigger) {
     if (!item.name) {
       throw new Error("Name field is required.");
     }
     const { db: db3 } = await postgresClient();
     if (item.external_id) {
-      const existingItem = await db3.from(getTableName(this.name)).where({ external_id: item.external_id }).first();
+      const existingItem = await db3.from(getTableName(this.id)).where({ external_id: item.external_id }).first();
       if (existingItem && !upsert) {
         throw new Error("Item with external id " + item.external_id + " already exists.");
       }
@@ -1140,7 +1194,7 @@ var ExuluContext = class {
       }
     }
     if (upsert && item.id) {
-      const existingItem = await db3.from(getTableName(this.name)).where({ id: item.id }).first();
+      const existingItem = await db3.from(getTableName(this.id)).where({ id: item.id }).first();
       if (existingItem && upsert) {
         await this.updateItem(user, {
           ...item,
@@ -1164,39 +1218,26 @@ var ExuluContext = class {
     });
     delete payloadWithSanitizedPropertyNames.id;
     delete payloadWithSanitizedPropertyNames.upsert;
-    const result = await db3.from(getTableName(this.name)).insert({
+    const result = await db3.from(getTableName(this.id)).insert({
       ...payloadWithSanitizedPropertyNames,
       id: db3.fn.uuid(),
       created_at: db3.fn.now(),
       updated_at: db3.fn.now()
     }).returning("id");
     if (this.embedder && (this.configuration.calculateVectors === "onInsert" || this.configuration.calculateVectors === "always")) {
-      if (this.embedder.queue?.name) {
-        console.log("[EXULU] embedder is in queue mode, scheduling job.");
-        const job = await bullmqDecorator({
-          label: `${this.embedder.name}`,
-          embedder: this.embedder.id,
-          context: this.id,
-          inputs: item,
-          item: item.id,
-          queue: this.embedder.queue,
-          user,
-          role,
-          trigger: trigger || "agent"
-        });
-        return {
-          id: result[0].id,
-          job: job.id
-        };
-      }
-      console.log("[EXULU] embedder is not in queue mode, calculating vectors directly.");
-      await this.createAndUpsertEmbeddings({
-        ...item,
-        id: result[0].id
-      }, user, {
-        label: this.embedder.name,
+      const { job } = await this.embeddings.generate.one({
+        item: {
+          ...item,
+          id: result[0].id
+        },
+        user,
+        role,
         trigger: trigger || "agent"
-      }, role);
+      });
+      return {
+        id: result[0].id,
+        job
+      };
     }
     return {
       id: result[0].id,
@@ -1225,7 +1266,7 @@ var ExuluContext = class {
     if (page < 1) page = 1;
     if (limit < 1) limit = 10;
     let offset = (page - 1) * limit;
-    const mainTable = getTableName(this.name);
+    const mainTable = getTableName(this.id);
     const { db: db3 } = await postgresClient();
     const columns = await db3(mainTable).columnInfo();
     const totalQuery = db3.count("* as count").from(mainTable).first();
@@ -1288,7 +1329,7 @@ var ExuluContext = class {
       if (this.queryRewriter) {
         query = await this.queryRewriter(query);
       }
-      const chunksTable = getChunksTableName(this.name);
+      const chunksTable = getChunksTableName(this.id);
       itemsQuery.leftJoin(chunksTable, function() {
         this.on(chunksTable + ".source", "=", mainTable + ".id");
       });
@@ -1308,52 +1349,133 @@ var ExuluContext = class {
       const vector = chunks[0].vector;
       const vectorStr = `ARRAY[${vector.join(",")}]`;
       const vectorExpr = `${vectorStr}::vector`;
+      const language = this.configuration.language || "english";
+      let items = [];
       switch (method) {
-        case "l1Distance":
-          itemsQuery.select(db3.raw(`?? <-> ${vectorExpr} as l1_distance`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`?? <-> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
-          break;
-        case "l2Distance":
-          itemsQuery.select(db3.raw(`?? <-> ${vectorExpr} as l2_distance`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`?? <-> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
-          break;
-        case "hammingDistance":
-          itemsQuery.select(db3.raw(`?? <#> ${vectorExpr} as hamming_distance`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
-          break;
-        case "jaccardDistance":
-          itemsQuery.select(db3.raw(`?? <#> ${vectorExpr} as jaccard_distance`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
-          break;
-        case "maxInnerProduct":
-          itemsQuery.select(db3.raw(`?? <#> ${vectorExpr} as inner_product`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`?? <#> ${vectorExpr} ASC`, [`${chunksTable}.embedding`]));
+        case "tsvector":
+          itemsQuery.select(db3.raw(
+            `ts_rank(${chunksTable}.fts, websearch_to_tsquery(?, ?)) as fts_rank`,
+            [language, query]
+          )).whereRaw(
+            `${chunksTable}.fts @@ websearch_to_tsquery(?, ?)`,
+            [language, query]
+          ).orderByRaw(`fts_rank DESC`);
+          items = await itemsQuery;
           break;
         case "cosineDistance":
         default:
-          itemsQuery.select(db3.raw(`1 - (?? <#> ${vectorExpr}) as cosine_distance`, [`${chunksTable}.embedding`]));
-          itemsQuery.orderByRaw(db3.raw(`1 - (?? <#> ${vectorExpr}) DESC`, [`${chunksTable}.embedding`]));
+          itemsQuery.whereNotNull(`${chunksTable}.embedding`);
+          itemsQuery.select(
+            db3.raw(`1 - (${chunksTable}.embedding <=> ${vectorExpr}) AS cosine_distance`)
+          );
+          itemsQuery.orderByRaw(
+            `${chunksTable}.embedding <=> ${vectorExpr} ASC NULLS LAST`
+          );
+          items = await itemsQuery;
           break;
+        case "hybridSearch":
+          const matchCount = Math.min(limit * 5, 30);
+          const fullTextWeight = 1;
+          const semanticWeight = 1;
+          const rrfK = 50;
+          const hybridSQL = `
+                    WITH full_text AS (
+                      SELECT
+                        c.id,
+                        c.source,
+                        row_number() OVER (
+                          ORDER BY ts_rank_cd(c.fts, websearch_to_tsquery(?, ?)) DESC
+                        ) AS rank_ix
+                      FROM ${chunksTable} c
+                      WHERE c.fts @@ websearch_to_tsquery(?, ?)
+                      ORDER BY rank_ix
+                      LIMIT LEAST(?, 30) * 2
+                    ),
+                    semantic AS (
+                      SELECT
+                        c.id,
+                        c.source,
+                        row_number() OVER (
+                          ORDER BY c.embedding <=> ${vectorExpr} ASC
+                        ) AS rank_ix
+                      FROM ${chunksTable} c
+                      WHERE c.embedding IS NOT NULL
+                      ORDER BY rank_ix
+                      LIMIT LEAST(?, 30) * 2
+                    )
+                    SELECT
+                      m.*,
+                      c.id AS chunk_id,
+                      c.source,
+                      c.content,
+                      c.chunk_index,
+                      c.created_at AS chunk_created_at,
+                      c.updated_at AS chunk_updated_at,
+                
+                      /* Per-signal scores for introspection */
+                      ts_rank(c.fts, websearch_to_tsquery(?, ?)) AS fts_rank,
+                      (1 - (c.embedding <=> ${vectorExpr})) AS cosine_distance,
+                
+                      /* Hybrid RRF score */
+                      (
+                        COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
+                        +
+                        COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
+                      ) AS hybrid_score
+                
+                    FROM full_text ft
+                    FULL OUTER JOIN semantic se
+                      ON ft.id = se.id
+                    JOIN ${chunksTable} c
+                      ON COALESCE(ft.id, se.id) = c.id
+                    JOIN ${mainTable} m
+                      ON m.id = c.source
+                    ORDER BY hybrid_score DESC
+                    LIMIT LEAST(?, 30)
+                    OFFSET 0
+                  `;
+          const bindings = [
+            // full_text: websearch_to_tsquery(lang, query) in rank and where
+            language,
+            query,
+            language,
+            query,
+            matchCount,
+            // full_text limit
+            matchCount,
+            // semantic limit
+            // fts_rank (ts_rank) call
+            language,
+            query,
+            // RRF fusion parameters
+            rrfK,
+            fullTextWeight,
+            rrfK,
+            semanticWeight,
+            matchCount
+            // final limit
+          ];
+          items = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
       }
-      let items = await itemsQuery;
+      console.log("items", items.map((item) => item.name + " " + item.id));
       const seenSources = /* @__PURE__ */ new Map();
       items = items.reduce((acc, item) => {
         if (!seenSources.has(item.source)) {
           seenSources.set(item.source, {
             ...Object.fromEntries(
               Object.keys(item).filter(
-                (key) => key !== "l1_distance" && key !== "l2_distance" && key !== "hamming_distance" && key !== "jaccard_distance" && key !== "inner_product" && key !== "cosine_distance" && key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at"
+                (key) => key !== "cosine_distance" && // kept per chunk below
+                key !== "fts_rank" && // kept per chunk below
+                key !== "hybrid_score" && // we will compute per item below
+                key !== "content" && key !== "source" && key !== "chunk_index" && key !== "chunk_id" && key !== "chunk_created_at" && key !== "chunk_updated_at"
               ).map((key) => [key, item[key]])
             ),
             chunks: [{
               content: item.content,
               chunk_index: item.chunk_index,
-              ...method === "l1Distance" && { l1_distance: item.l1_distance },
-              ...method === "l2Distance" && { l2_distance: item.l2_distance },
-              ...method === "hammingDistance" && { hamming_distance: item.hamming_distance },
-              ...method === "jaccardDistance" && { jaccard_distance: item.jaccard_distance },
-              ...method === "maxInnerProduct" && { inner_product: item.inner_product },
-              ...method === "cosineDistance" && { cosine_distance: item.cosine_distance }
+              ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
+              ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
+              ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
             }]
           });
           acc.push(seenSources.get(item.source));
@@ -1361,12 +1483,9 @@ var ExuluContext = class {
           seenSources.get(item.source).chunks.push({
             content: item.content,
             chunk_index: item.chunk_index,
-            ...method === "l1Distance" && { l1_distance: item.l1_distance },
-            ...method === "l2Distance" && { l2_distance: item.l2_distance },
-            ...method === "hammingDistance" && { hamming_distance: item.hamming_distance },
-            ...method === "jaccardDistance" && { jaccard_distance: item.jaccard_distance },
-            ...method === "maxInnerProduct" && { inner_product: item.inner_product },
-            ...method === "cosineDistance" && { cosine_distance: item.cosine_distance }
+            ...method === "cosineDistance" && { cosine_distance: item.cosine_distance },
+            ...(method === "tsvector" || method === "hybridSearch") && { fts_rank: item.fts_rank },
+            ...method === "hybridSearch" && { hybrid_score: item.hybrid_score }
           });
         }
         return acc;
@@ -1375,16 +1494,34 @@ var ExuluContext = class {
         if (!item.chunks?.length) {
           return;
         }
-        const {
-          average,
-          total
-        } = calculateStatistics(item.chunks, method ?? "cosineDistance");
-        item.averageRelevance = average;
-        item.totalRelevance = total;
+        if (method === "tsvector") {
+          const ranks = item.chunks.map((c) => typeof c.fts_rank === "number" ? c.fts_rank : 0);
+          const total = ranks.reduce((a, b) => a + b, 0);
+          const average = ranks.length ? total / ranks.length : 0;
+          item.averageRelevance = average;
+          item.totalRelevance = total;
+        } else if (method === "cosineDistance") {
+          let methodProperty = "cosine_distance";
+          const average = item.chunks.reduce((acc, item2) => {
+            return acc + item2[methodProperty];
+          }, 0) / item.chunks.length;
+          const total = item.chunks.reduce((acc, item2) => {
+            return acc + item2[methodProperty];
+          }, 0);
+          item.averageRelevance = average;
+          item.totalRelevance = total;
+        } else if (method === "hybridSearch") {
+          const scores = item.chunks.map((c) => typeof c.hybrid_score === "number" ? c.hybrid_score : 0);
+          const total = scores.reduce((a, b) => a + b, 0);
+          const average = scores.length ? total / scores.length : 0;
+          item.averageRelevance = average;
+          item.totalRelevance = total;
+        }
       });
       if (this.resultReranker && query) {
         items = await this.resultReranker(items);
       }
+      items = items.slice(0, limit);
       return {
         filters: {
           archived,
@@ -1402,7 +1539,7 @@ var ExuluContext = class {
   };
   createItemsTable = async () => {
     const { db: db3 } = await postgresClient();
-    const tableName = getTableName(this.name);
+    const tableName = getTableName(this.id);
     console.log("[EXULU] Creating table: " + tableName);
     return await db3.schema.createTable(tableName, (table) => {
       console.log("[EXULU] Creating fields for table.", this.fields);
@@ -1428,19 +1565,33 @@ var ExuluContext = class {
   };
   createChunksTable = async () => {
     const { db: db3 } = await postgresClient();
-    const tableName = getChunksTableName(this.name);
+    const tableName = getChunksTableName(this.id);
     console.log("[EXULU] Creating table: " + tableName);
-    return await db3.schema.createTable(tableName, (table) => {
+    await db3.schema.createTable(tableName, (table) => {
       if (!this.embedder) {
         throw new Error("Embedder must be set for context " + this.name + " to create chunks table.");
       }
       table.uuid("id").primary().defaultTo(db3.fn.uuid());
-      table.uuid("source").references("id").inTable(getTableName(this.name));
+      table.uuid("source").references("id").inTable(getTableName(this.id));
       table.text("content");
       table.integer("chunk_index");
       table.specificType("embedding", `vector(${this.embedder.vectorDimensions})`);
+      table.specificType(
+        "fts",
+        `tsvector GENERATED ALWAYS AS (to_tsvector('${this.configuration.language || "english"}', coalesce(content, ''))) STORED`
+      );
+      table.index(["fts"], `${tableName}_fts_gin_idx`, "gin");
+      table.index(["source"], `${tableName}_source_idx`);
       table.timestamps(true, true);
     });
+    await db3.raw(`
+            CREATE INDEX IF NOT EXISTS ${tableName}_embedding_hnsw_cosine
+            ON ${tableName}
+            USING hnsw (embedding vector_cosine_ops)
+            WHERE embedding IS NOT NULL
+            WITH (m = 16, ef_construction = 64)
+        `);
+    return;
   };
   // Exports the context as a tool that can be used by an agent
   tool = () => {
@@ -1858,9 +2009,6 @@ var requestValidators = {
   }
 };
 
-// src/registry/routes.ts
-var import_zodex = require("zodex");
-
 // src/bullmq/queues.ts
 var import_bullmq4 = require("bullmq");
 var import_bullmq_otel = require("bullmq-otel");
@@ -1897,17 +2045,13 @@ var queues = new ExuluQueues();
 // types/models/vector-methods.ts
 var VectorMethodEnum = {
   "cosineDistance": "cosineDistance",
-  "l1Distance": "l1Distance",
-  "l2Distance": "l2Distance",
-  "hammingDistance": "hammingDistance",
-  "jaccardDistance": "jaccardDistance",
-  "maxInnerProduct": "maxInnerProduct"
+  "hybridSearch": "hybridSearch",
+  "tsvector": "tsvector"
 };
 
 // src/registry/routes.ts
 var import_express4 = __toESM(require("express"), 1);
 var import_server3 = require("@apollo/server");
-var Papa = require("papaparse");
 var import_cors = __toESM(require("cors"), 1);
 var import_reflect_metadata = require("reflect-metadata");
 
@@ -3149,7 +3293,7 @@ function createSDL(tables, contexts, agents, tools) {
   const contextSchemas = [];
   console.log("============= Agents =============", agents?.length);
   contexts.forEach((context) => {
-    const tableName = getTableName(context.name);
+    const tableName = getTableName(context.id);
     const definition = {
       name: {
         singular: tableName,
@@ -3476,30 +3620,6 @@ type StatisticsResult {
     typeDefs: fullSDL,
     resolvers
   });
-  console.log("\n\u{1F4CA} GraphQL Schema Overview\n");
-  const queriesTable = Object.keys(resolvers.Query).map((query) => ({
-    "Operation Type": "Query",
-    "Name": query,
-    "Description": "Retrieves data"
-  }));
-  const mutationsTable = Object.keys(resolvers.Mutation).map((mutation) => ({
-    "Operation Type": "Mutation",
-    "Name": mutation,
-    "Description": "Modifies data"
-  }));
-  const typesTable = tables.flatMap(
-    (table) => table.fields.map((field) => ({
-      "Type": table.name.singular,
-      "Field": field.name,
-      "Field Type": field.type,
-      "Required": field.required ? "Yes" : "No"
-    }))
-  );
-  console.log("\u{1F50D} Operations:");
-  console.table([...queriesTable, ...mutationsTable]);
-  console.log("\n\u{1F4DD} Types and Fields:");
-  console.table(typesTable);
-  console.log("\n");
   return schema;
 }
 var encryptSensitiveFields = (input) => {
@@ -4001,12 +4121,9 @@ var createRecurringJobs = async () => {
       }
     }
   );
-  console.log("[EXULU] recurring job schedulers:");
-  console.table(recurringJobSchedulersLogs);
   return queue;
 };
 var createExpressRoutes = async (app, logger, agents, tools, contexts, config, tracer) => {
-  const routeLogs = [];
   console.log("============= agents =============", agents?.length);
   var corsOptions = {
     origin: "*",
@@ -4029,35 +4146,6 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     Intelligence Management Platform
 
     `);
-  console.log("Agents:");
-  console.table(agents.map((agent) => {
-    return {
-      id: agent.id,
-      name: agent.name,
-      description: agent.description,
-      slug: "/agents/" + agent.id,
-      active: true
-    };
-  }));
-  console.log("Contexts:");
-  console.table(contexts.map((context) => {
-    return {
-      id: context.id,
-      name: context.name,
-      description: context.description,
-      embedder: context.embedder?.name || void 0,
-      slug: "/contexts/" + context.id,
-      active: context.active
-    };
-  }));
-  routeLogs.push(
-    { route: "/contexts", method: "GET", note: "List all contexts" },
-    { route: "/contexts/:id", method: "GET", note: "Get specific context" },
-    { route: "/items/:context", method: "POST", note: "Create new item in context" },
-    { route: "/items/:context", method: "GET", note: "Get items from context" },
-    { route: "/items/export/:context", method: "GET", note: "Export items from context" },
-    { route: "/graphql", method: "POST", note: "GraphQL endpoint" }
-  );
   if (redisServer.host?.length && redisServer.port?.length) {
     await createRecurringJobs();
   } else {
@@ -4076,15 +4164,12 @@ var createExpressRoutes = async (app, logger, agents, tools, contexts, config, t
     statisticsSchema2(),
     rbacSchema2()
   ], contexts, agents, tools);
-  console.log("[EXULU] graphql server");
   const server = new import_server3.ApolloServer({
     cache: new import_utils2.InMemoryLRUCache(),
     schema,
     introspection: true
   });
-  console.log("[EXULU] starting graphql server");
   await server.start();
-  console.log("[EXULU] graphql server started");
   app.use(
     "/graphql",
     (0, import_cors.default)(corsOptions),
@@ -4223,9 +4308,9 @@ Mood: friendly and intelligent.
     }
     const exists = await context.tableExists();
     if (!exists) {
-      throw new Error("Table with name " + getTableName(context.name) + " does not exist.");
+      throw new Error("Table with name " + getTableName(context.id) + " does not exist.");
     }
-    const query = db3.from(getTableName(context.name)).select("id");
+    const query = db3.from(getTableName(context.id)).select("id");
     if (id) {
       query.where({ id });
     }
@@ -4237,12 +4322,12 @@ Mood: friendly and intelligent.
       return null;
     }
     if (context.embedder) {
-      const chunks = await db3.from(getChunksTableName(context.name)).where({ source: item.id }).select("id");
+      const chunks = await db3.from(getChunksTableName(context.id)).where({ source: item.id }).select("id");
       if (chunks.length > 0) {
-        await db3.from(getChunksTableName(context.name)).where({ source: item.id }).delete();
+        await db3.from(getChunksTableName(context.id)).where({ source: item.id }).delete();
       }
     }
-    const mutation = db3.from(getTableName(context.name)).where({ id: item.id }).delete().returning("id");
+    const mutation = db3.from(getTableName(context.id)).where({ id: item.id }).delete().returning("id");
     const result = await mutation;
     return result;
   };
@@ -4284,6 +4369,47 @@ Mood: friendly and intelligent.
     }
     res.status(200).json(result);
   });
+  app.delete("/items/:context/:id/chunks/delete", async (req, res) => {
+    console.log("[EXULU] deleting chunks for item.", req.params.id);
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db3 } = await postgresClient();
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    await db3.from(getChunksTableName(context.id)).where({ source: req.params.id }).delete();
+    res.status(200).json({
+      message: "Chunks deleted successfully."
+    });
+  });
+  app.delete("/items/:context/chunks/delete", async (req, res) => {
+    console.log("[EXULU] deleting chunks for context.", req.params.context);
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const { db: db3 } = await postgresClient();
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    console.log("[EXULU] truncating chunks table for context.", context.id);
+    await db3.from(getChunksTableName(context.id)).truncate();
+    res.status(200).json({
+      message: "Chunks deleted successfully."
+    });
+  });
   app.get("/items/:context/:id", async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -4306,14 +4432,15 @@ Mood: friendly and intelligent.
       return;
     }
     const itemsTableExists = await context.tableExists();
+    console.log("[EXULU] items table exists.", itemsTableExists);
     if (!itemsTableExists) {
       await context.createItemsTable();
     }
-    const chunksTableExists = await db3.schema.hasTable(getChunksTableName(context.name));
+    const chunksTableExists = await context.chunksTableExists();
     if (!chunksTableExists && context.embedder) {
       await context.createChunksTable();
     }
-    const item = await db3.from(getTableName(context.name)).where({ id: req.params.id }).select("*").first();
+    const item = await db3.from(getTableName(context.id)).where({ id: req.params.id }).select("*").first();
     if (!item) {
       res.status(404).json({
         message: "Item not found."
@@ -4326,8 +4453,8 @@ Mood: friendly and intelligent.
       });
       return;
     }
-    console.log("[EXULU] chunks table name.", getChunksTableName(context.name));
-    const chunks = await db3.from(getChunksTableName(context.name)).where({ source: req.params.id }).select("id", "content", "source", "embedding", "chunk_index", "created_at", "updated_at");
+    console.log("[EXULU] chunks table name.", getChunksTableName(context.id));
+    const chunks = await db3.from(getChunksTableName(context.id)).where({ source: req.params.id }).select("id", "content", "source", "embedding", "chunk_index", "created_at", "updated_at");
     console.log("[EXULU] chunks", chunks);
     res.status(200).json({
       ...item,
@@ -4342,6 +4469,88 @@ Mood: friendly and intelligent.
       }))
     });
   });
+  app.post("/items/:context/chunks/generate/all", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const user = authenticationResult.user;
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const jobs = await context.embeddings.generate.all(context, user.id, user.role?.id);
+    res.status(200).json({
+      message: "Embeddings generated successfully.",
+      jobs
+    });
+  });
+  app.post("/items/:context/chunks/generate/one/:id", async (req, res) => {
+    if (!req.params.context) {
+      res.status(400).json({
+        message: "Missing context in request."
+      });
+      return;
+    }
+    if (!req.params.id) {
+      res.status(400).json({
+        message: "Missing id in request."
+      });
+      return;
+    }
+    const authenticationResult = await requestValidators.authenticate(req);
+    if (!authenticationResult.user?.id) {
+      res.status(authenticationResult.code || 500).json({ detail: `${authenticationResult.message}` });
+      return;
+    }
+    const user = authenticationResult.user;
+    const context = contexts.find((context2) => context2.id === req.params.context);
+    if (!context) {
+      res.status(400).json({
+        message: "Context not found in registry."
+      });
+      return;
+    }
+    const exists = await context.tableExists();
+    if (!exists) {
+      await context.createItemsTable();
+    }
+    const chunksTableExists = await context.chunksTableExists();
+    if (!chunksTableExists && context.embedder) {
+      await context.createChunksTable();
+    }
+    const { db: db3 } = await postgresClient();
+    const item = await db3.from(getTableName(context.id)).where({ id: req.params.id }).select("*").first();
+    if (!item) {
+      res.status(404).json({
+        message: "Item not found."
+      });
+      return;
+    }
+    const { job } = await context.embeddings.generate.one({
+      item: {
+        ...item,
+        id: item.id
+      },
+      user: user.id,
+      role: user.role?.id,
+      trigger: "api"
+    });
+    res.status(200).json({
+      message: job ? "Job scheduled to generate embeddings successfully." : "Embeddings generated successfully.",
+      job
+    });
+  });
   app.post("/items/:context/:id", async (req, res) => {
     if (!req.params.context) {
       res.status(400).json({
@@ -4387,7 +4596,6 @@ Mood: friendly and intelligent.
   });
   app.post("/items/:context", async (req, res) => {
     try {
-      console.log("[EXULU] post items");
       if (!req.params.context) {
         res.status(400).json({
           message: "Missing context in request."
@@ -4484,6 +4692,10 @@ Mood: friendly and intelligent.
     if (!exists) {
       await context.createItemsTable();
     }
+    const chunksTableExists = await context.chunksTableExists();
+    if (!chunksTableExists && context.embedder) {
+      await context.createChunksTable();
+    }
     if (req.query.method && !Object.values(VectorMethodEnum).includes(req.query.method)) {
       res.status(400).json({
         message: "Invalid vector lookup method, must be one of: " + Object.values(VectorMethodEnum).join(", ")
@@ -4523,11 +4735,6 @@ Mood: friendly and intelligent.
   agents.forEach((agent) => {
     const slug = agent.slug;
     if (!slug) return;
-    routeLogs.push({
-      route: slug + "/:instance",
-      method: "POST",
-      note: `Agent endpoint for ${agent.id}`
-    });
     app.post(slug + "/:instance", async (req, res) => {
       const instance = req.params.instance;
       if (!instance) {
@@ -4649,8 +4856,6 @@ Mood: friendly and intelligent.
   } else {
     console.log("[EXULU] skipping uppy file upload routes, because no S3 compatible region, key or secret is set in the environment.");
   }
-  console.log("Routes:");
-  console.table(routeLogs);
   const TARGET_API = "https://api.anthropic.com";
   app.use("/gateway/anthropic/:id", import_express4.default.raw({ type: "*/*", limit: REQUEST_SIZE_LIMIT }), async (req, res) => {
     const path3 = req.url;
@@ -4995,8 +5200,6 @@ ${code}`
         }]
       })
     );
-    console.log("Contexts:");
-    console.table();
   };
   connect = async () => {
     if (!this.express) {
@@ -5051,14 +5254,6 @@ ${code}`
     };
     this.express.get("/mcp", handleSessionRequest);
     this.express.delete("/mcp", handleSessionRequest);
-    const routeLogs = [];
-    routeLogs.push(
-      { route: "/mcp", method: "GET", note: "Get MCP server status" },
-      { route: "/mcp", method: "POST", note: "Send MCP request" },
-      { route: "/mcp", method: "DELETE", note: "Terminate MCP session" }
-    );
-    console.log("MCP Routes:");
-    console.table(routeLogs);
     return this.express;
   };
 };
@@ -5067,9 +5262,8 @@ ${code}`
 var import_express8 = __toESM(require("express"), 1);
 
 // src/templates/agents/claude-code.ts
-var agentId = "0832-5178-1145-2194";
 var claudeCodeAgent = new ExuluAgent({
-  id: `${agentId}-claude-code-agent`,
+  id: `claude_code_agent`,
   name: `Claude Code Agent`,
   description: `Claude Code agent, enabling the creation of multiple Claude Code Agent instances with different configurations (rate limits, functions, etc).`,
   type: "custom"
@@ -5077,9 +5271,8 @@ var claudeCodeAgent = new ExuluAgent({
 
 // src/templates/agents/claude-opus-4.ts
 var import_anthropic = require("@ai-sdk/anthropic");
-var agentId2 = "5434-5678-9143-2590";
 var defaultAgent = new ExuluAgent({
-  id: `${agentId2}-default-claude-4-opus-agent`,
+  id: `default_claude_4_opus_agent`,
   name: `Default Claude 4 Opus Agent`,
   description: `Basic agent without any defined tools, that can support MCP's.`,
   type: "agent",
@@ -5146,7 +5339,7 @@ var logger_default = createLogger;
 
 // src/templates/contexts/code-standards.ts
 var codeStandardsContext = new ExuluContext({
-  id: "code-standards",
+  id: "code_standards",
   name: "Code Standards",
   description: "Code standards that can be used with the Exulu CLI.",
   configuration: {
@@ -5184,6 +5377,13 @@ var projectsContext = new ExuluContext({
 });
 
 // src/registry/index.ts
+var isValidPostgresName = (id) => {
+  console.log("[EXULU] validating context id.", id);
+  const regex = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
+  const isValid = regex.test(id);
+  const length = id.length;
+  return isValid && length <= 80 && length > 5;
+};
 var ExuluApp = class {
   _agents = [];
   _config;
@@ -5214,6 +5414,28 @@ var ExuluApp = class {
       // Add agents as tools
       ...(agents || []).map((agent) => agent.tool())
     ];
+    const checks = [
+      ...Object.keys(this._contexts || {}).map((x) => ({
+        name: this._contexts?.[x]?.name ?? "",
+        id: this._contexts?.[x]?.id ?? "",
+        type: "context"
+      })),
+      ...this._agents.map((agent) => ({
+        name: agent.name ?? "",
+        id: agent.id ?? "",
+        type: "agent"
+      })),
+      ...this._tools.map((tool2) => ({
+        name: tool2.name ?? "",
+        id: tool2.id ?? "",
+        type: "tool"
+      }))
+    ];
+    const invalid = checks.filter((x) => !isValidPostgresName(x?.id ?? ""));
+    if (invalid.length > 0) {
+      console.error(`%c[EXULU] Invalid ID found for a context, tool or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`, "color: orange; font-weight: bold; \n \n");
+      throw new Error(`Invalid ID found for a context, tool or agent: ${invalid.map((x) => x.id).join(", ")}. An ID must begin with a letter (a-z) or underscore (_). Subsequent characters in a name can be letters, digits (0-9), or underscores and be a max length of 80 characters and at least 5 characters long.`);
+    }
     const contextsArray = Object.values(contexts || {});
     const queues2 = [
       ...contextsArray?.length ? contextsArray.map((context) => context.embedder?.queue?.name || null) : []
@@ -5250,6 +5472,34 @@ var ExuluApp = class {
   get agents() {
     return this._agents;
   }
+  embeddings = {
+    generate: {
+      one: async ({
+        context: contextId,
+        item: itemId
+      }) => {
+        const { db: db3 } = await postgresClient();
+        const item = await db3.from(getTableName(contextId)).where({ id: itemId }).select("*").first();
+        const context = this.contexts.find((x) => contextId === x.id);
+        if (!context) {
+          throw new Error(`Context ${contextId} not found in registry.`);
+        }
+        return await context.embeddings.generate.one({
+          item,
+          trigger: "api"
+        });
+      },
+      all: async ({
+        context: contextId
+      }) => {
+        const context = this.contexts.find((x) => contextId === x.id);
+        if (!context) {
+          throw new Error(`Context ${contextId} not found in registry.`);
+        }
+        return await context.embeddings.generate.all(context, void 0, void 0);
+      }
+    }
+  };
   bullmq = {
     workers: {
       create: async () => {