@exulu/backend 1.42.2 → 1.43.0

package/dist/index.js

@@ -1435,6 +1435,10 @@ var addCoreFields = (schema) => {
     name: "last_processed_at",
     type: "date"
   });
+  schema.fields.push({
+    name: "embeddings_updated_at",
+    type: "date"
+  });
   if (schema.RBAC) {
     if (!schema.fields.some((field) => field.name === "rights_mode")) {
       schema.fields.push({
@@ -1648,7 +1652,8 @@ var checkRecordAccess = async (record, request, user) => {
   const isPublic = record.rights_mode === "public";
   const byUsers = record.rights_mode === "users";
   const byRoles = record.rights_mode === "roles";
-  const isCreator = user ? record.created_by === user.id.toString() : false;
+  const createdBy = typeof record.created_by === "string" ? record.created_by : record.created_by?.toString();
+  const isCreator = user ? createdBy === user.id.toString() : false;
   const isAdmin = user ? user.super_admin : false;
   const isApi = user ? user.type === "api" : false;
   let hasAccess = "none";
@@ -1855,6 +1860,7 @@ ${enumValues}
     fields.push("  maxContextLength: Int");
     fields.push("  provider: String");
     fields.push("  authenticationInformation: String");
+    fields.push("  systemInstructions: String");
     fields.push("  slug: String");
   }
   const rbacField = table.RBAC ? "  RBAC: RBACData" : "";
@@ -1919,6 +1925,8 @@ input FilterOperatorDate {
 input FilterOperatorFloat {
   eq: Float
   ne: Float
+  lte: Float
+  gte: Float
   in: [Float]
   and: [FilterOperatorFloat]
   or: [FilterOperatorFloat]
@@ -1982,7 +1990,11 @@ var getRequestedFields = (info) => {
   return fields.filter((field) => field !== "pageInfo" && field !== "items" && field !== "RBAC");
 };
 var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRbacRecords) => {
-  const { users = [], roles = [], projects = [] } = rbacData;
+  const {
+    users = [],
+    roles = []
+    /* projects = [] */
+  } = rbacData;
   if (!existingRbacRecords) {
     existingRbacRecords = await db3.from("rbac").where({
       entity: entityName,
@@ -1991,25 +2003,19 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
   }
   const newUserRecords = new Set(users.map((u) => `${u.id}:${u.rights}`));
   const newRoleRecords = new Set(roles.map((r) => `${r.id}:${r.rights}`));
-  const newProjectRecords = new Set(projects.map((p) => `${p.id}:${p.rights}`));
   const existingUserRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "User").map((r) => `${r.user_id}:${r.rights}`));
   const existingRoleRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Role").map((r) => `${r.role_id}:${r.rights}`));
   const existingProjectRecords = new Set(existingRbacRecords.filter((r) => r.access_type === "Project").map((r) => `${r.project_id}:${r.rights}`));
   const usersToCreate = users.filter((u) => !existingUserRecords.has(`${u.id}:${u.rights}`));
   const rolesToCreate = roles.filter((r) => !existingRoleRecords.has(`${r.id}:${r.rights}`));
-  const projectsToCreate = projects.filter((p) => !existingProjectRecords.has(`${p.id}:${p.rights}`));
   const usersToRemove = existingRbacRecords.filter((r) => r.access_type === "User" && !newUserRecords.has(`${r.user_id}:${r.rights}`));
   const rolesToRemove = existingRbacRecords.filter((r) => r.access_type === "Role" && !newRoleRecords.has(`${r.role_id}:${r.rights}`));
-  const projectsToRemove = existingRbacRecords.filter((r) => r.access_type === "Project" && !newProjectRecords.has(`${r.project_id}:${r.rights}`));
   if (usersToRemove.length > 0) {
     await db3.from("rbac").whereIn("id", usersToRemove.map((r) => r.id)).del();
   }
   if (rolesToRemove.length > 0) {
     await db3.from("rbac").whereIn("id", rolesToRemove.map((r) => r.id)).del();
   }
-  if (projectsToRemove.length > 0) {
-    await db3.from("rbac").whereIn("id", projectsToRemove.map((r) => r.id)).del();
-  }
   const recordsToInsert = [];
   usersToCreate.forEach((user) => {
     recordsToInsert.push({
@@ -2033,17 +2039,6 @@ var handleRBACUpdate = async (db3, entityName, resourceId, rbacData, existingRba
       updatedAt: /* @__PURE__ */ new Date()
     });
   });
-  projectsToCreate.forEach((project) => {
-    recordsToInsert.push({
-      entity: entityName,
-      access_type: "Project",
-      target_resource_id: resourceId,
-      project_id: project.id,
-      rights: project.rights,
-      createdAt: /* @__PURE__ */ new Date(),
-      updatedAt: /* @__PURE__ */ new Date()
-    });
-  });
   if (recordsToInsert.length > 0) {
     await db3.from("rbac").insert(recordsToInsert);
   }
@@ -2057,7 +2052,7 @@ function createMutations(table, agents, contexts, tools, config) {
       if (user.super_admin === true) {
         return true;
       }
-      if (!user.super_admin && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
+      if (!user.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user.role || !(table.name.plural === "agents" && user.role.agents === "write") && !(table.name.plural === "workflow_templates" && user.role.workflows === "write") && !(table.name.plural === "variables" && user.role.variables === "write") && !(table.name.plural === "users" && user.role.users === "write") && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && user.role.evals === "write"))) {
         console.error("Access control error: no role found for current user or no access to entity type.");
         throw new Error("Access control error: no role found for current user or no access to entity type.");
       }
@@ -2111,6 +2106,61 @@ function createMutations(table, agents, contexts, tools, config) {
     }
   };
   const mutations = {
+    [`${tableNamePlural}CopyOneById`]: async (_, args, context, info) => {
+      const { db: db3 } = context;
+      const requestedFields = getRequestedFields(info);
+      let { id } = args;
+      if (!id) {
+        throw new Error("ID is required for copying a record.");
+      }
+      await validateWriteAccess(id, context);
+      const item = await db3.from(tableNamePlural).select("*").where({ id }).first();
+      if (!item) {
+        throw new Error("Record not found");
+      }
+      if (item.rights_mode) {
+        item.rights_mode = "private";
+      }
+      if (item.created_at) {
+        item.created_at = /* @__PURE__ */ new Date();
+      }
+      if (item.createdAt) {
+        item.createdAt = /* @__PURE__ */ new Date();
+      }
+      if (item.updated_at) {
+        item.updated_at = /* @__PURE__ */ new Date();
+      }
+      if (item.updatedAt) {
+        item.updatedAt = /* @__PURE__ */ new Date();
+      }
+      if (item.created_by) {
+        item.created_by = context.user.id;
+      }
+      if (item.createdBy) {
+        item.createdBy = context.user.id;
+      }
+      if (item.name) {
+        item.name = item.name + " (Copy)";
+      }
+      Object.keys(item).forEach((key) => {
+        if (table.fields.find((field) => field.name === key)?.type === "json") {
+          if (typeof item[key] === "object" || Array.isArray(item[key])) {
+            item[key] = JSON.stringify(item[key]);
+          }
+        }
+      });
+      const insert = db3(tableNamePlural).insert({
+        ...item,
+        id: db3.fn.uuid()
+      }).returning("*");
+      const result = await insert;
+      if (!result[0]) {
+        throw new Error("Failed to copy record.");
+      }
+      return {
+        item: finalizeRequestedFields({ args, table, requestedFields, agents, contexts, tools, result: result[0], user: context.user })
+      };
+    },
     [`${tableNamePlural}CreateOne`]: async (_, args, context, info) => {
       const { db: db3 } = context;
       const requestedFields = getRequestedFields(info);
@@ -2368,7 +2418,7 @@ function createMutations(table, agents, contexts, tools, config) {
         }
         const { limit = 10, filters = [], sort } = args;
         const { db: db3 } = context;
-        const { items } = await paginationRequest({
+        const { items } = await itemsPaginationRequest({
           db: db3,
           limit,
           page: 0,
@@ -2467,9 +2517,6 @@ function createMutations(table, agents, contexts, tools, config) {
       };
     };
     mutations[`${tableNameSingular}GenerateChunks`] = async (_, args, context, info) => {
-      if (!context.user?.super_admin) {
-        throw new Error("You are not authorized to generate chunks via API, user must be super admin.");
-      }
       const { db: db3 } = await postgresClient();
       const exists = contexts.find((context2) => context2.id === table.id);
       if (!exists) {
@@ -2480,13 +2527,17 @@ function createMutations(table, agents, contexts, tools, config) {
       const columns = await db3(mainTable).columnInfo();
       let query = db3.from(mainTable).select(Object.keys(columns));
       if (!args.where) {
+        if (!context.user?.super_admin) {
+          throw new Error("You are not authorized to generate all chunks via API, user must be super admin.");
+        }
         const {
           jobs: jobs2,
           items: items2
         } = await embeddings.generate.all(
           config,
           context.user.id,
-          context.user.role?.id
+          context.user.role?.id,
+          args.limit
         );
         return {
           message: "Chunks generated successfully.",
@@ -2495,6 +2546,9 @@ function createMutations(table, agents, contexts, tools, config) {
         };
       }
       query = applyFilters(query, args.where, table);
+      if (args.limit) {
+        query = query.limit(args.limit);
+      }
       const items = await query;
       if (items.length === 0) {
         throw new Error("No items found to generate chunks for.");
@@ -2519,9 +2573,6 @@ function createMutations(table, agents, contexts, tools, config) {
       };
     };
     mutations[`${tableNameSingular}DeleteChunks`] = async (_, args, context, info) => {
-      if (!context.user?.super_admin) {
-        throw new Error("You are not authorized to delete chunks via API, user must be super admin.");
-      }
       const { db: db3 } = await postgresClient();
       const id = contexts.find((context2) => context2.id === table.id)?.id;
       if (!id) {
@@ -2530,6 +2581,10 @@ function createMutations(table, agents, contexts, tools, config) {
       if (args.where) {
         let query = db3.from(getTableName(id)).select("id");
         query = applyFilters(query, args.where, table);
+        query = applyAccessControl(table, query, context.user);
+        if (args.limit) {
+          query = query.limit(args.limit);
+        }
         const items = await query;
         if (items.length === 0) {
           throw new Error("No items found to delete chunks for.");
@@ -2543,11 +2598,20 @@ function createMutations(table, agents, contexts, tools, config) {
           jobs: []
         };
       } else {
-        const count = await db3.from(getChunksTableName(id)).count();
-        await db3.from(getChunksTableName(id)).truncate();
+        if (!context.user?.super_admin) {
+          throw new Error("You are not authorized to delete all chunks via API, user must be super admin.");
+        }
+        let count = 0;
+        if (!args.limit) {
+          const result = await db3.from(getChunksTableName(id)).count();
+          count = parseInt(result[0].count);
+          await db3.from(getChunksTableName(id)).truncate();
+        } else {
+          count = await db3.from(getChunksTableName(id)).limit(args.limit).delete();
+        }
         return {
           message: "Chunks deleted successfully.",
-          items: parseInt(count[0].count),
+          items: count,
           jobs: []
         };
       }
@@ -2562,8 +2626,8 @@ var applyAccessControl = (table, query, user, field_prefix) => {
   }
   console.log("[EXULU] user.role", user?.role);
   console.log("[EXULU] table.name.plural", table.name.plural);
-  if (user && !user?.super_admin && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
-    console.error("==== Access control error: no role found or no access to entity type. ====");
+  if (user && !user?.super_admin && (table.name.plural === "agents" || table.name.plural === "workflow_templates" || table.name.plural === "variables" || table.name.plural === "users" || table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (!user?.role || !(table.name.plural === "agents" && (user.role.agents === "read" || user.role.agents === "write")) && !(table.name.plural === "workflow_templates" && (user.role.workflows === "read" || user.role.workflows === "write")) && !(table.name.plural === "variables" && (user.role.variables === "read" || user.role.variables === "write")) && !(table.name.plural === "users" && (user.role.users === "read" || user.role.users === "write")) && !((table.name.plural === "test_cases" || table.name.plural === "eval_sets" || table.name.plural === "eval_runs") && (user.role.evals === "read" || user.role.evals === "write")))) {
+    console.error("==== Access control error: no role found or no access to entity type. ====", user, table.name.plural);
     throw new Error("Access control error: no role found or no access to entity type.");
   }
   const hasRBAC = table.RBAC === true;
@@ -2606,6 +2670,7 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
   const isJsonField = field?.type === "json";
   const prefix = field_prefix ? field_prefix + "." : "";
   fieldName = prefix + fieldName;
+  console.log("[EXULU] operators", operators);
   if (operators.eq !== void 0) {
     if (isJsonField) {
       query = query.whereRaw(`?? = ?::jsonb`, [fieldName, JSON.stringify(operators.eq)]);
@@ -2637,7 +2702,13 @@ var converOperatorToQuery = (query, fieldName, operators, table, field_prefix) =
     }
   }
   if (operators.lte !== void 0) {
-    query = query.where(fieldName, "<=", operators.lte);
+    console.log("[EXULU] operators.lte", operators.lte);
+    console.log("[EXULU] fieldName", fieldName);
+    if (operators.lte === 0 || operators.lte === "0") {
+      query = query.whereNull(fieldName).orWhere(fieldName, "=", 0);
+    } else {
+      query = query.where(fieldName, "<=", operators.lte);
+    }
   }
   if (operators.gte !== void 0) {
     query = query.where(fieldName, ">=", operators.gte);
@@ -2653,7 +2724,8 @@ var backendAgentFields = [
   "capabilities",
   "maxContextLength",
   "provider",
-  "authenticationInformation"
+  "authenticationInformation",
+  "systemInstructions"
 ];
 var removeAgentFields = (requestedFields) => {
   const filtered = requestedFields.filter((field) => !backendAgentFields.includes(field));
@@ -2738,6 +2810,9 @@ var addAgentFields = async (args, requestedFields, agents, result, tools, user,
   if (requestedFields.includes("provider")) {
     result.provider = backend?.provider || "";
   }
+  if (requestedFields.includes("systemInstructions")) {
+    result.systemInstructions = backend?.config?.instructions || void 0;
+  }
   if (!requestedFields.includes("backend")) {
     delete result.backend;
   }
@@ -2962,7 +3037,7 @@ var applySorting = (query, sort, field_prefix) => {
   }
   return query;
 };
-var paginationRequest = async ({
+var itemsPaginationRequest = async ({
   db: db3,
   limit,
   page,
@@ -2992,7 +3067,8 @@ var paginationRequest = async ({
   if (page > 1) {
     dataQuery = dataQuery.offset((page - 1) * limit);
   }
-  let items = await dataQuery.select(fields ? fields : "*").limit(limit);
+  dataQuery = dataQuery.select(fields ? fields : "*").limit(limit);
+  let items = await dataQuery;
   return {
     items,
     pageInfo: {
@@ -3043,7 +3119,7 @@ function createQueries(table, agents, tools, contexts) {
       const { limit = 10, page = 0, filters = [], sort } = args;
       const requestedFields = getRequestedFields(info);
       const sanitizedFields = sanitizeRequestedFields(table, requestedFields);
-      const { items, pageInfo } = await paginationRequest({
+      const { items, pageInfo } = await itemsPaginationRequest({
         db: db3,
         limit,
         page,
@@ -3118,6 +3194,46 @@ function createQueries(table, agents, tools, contexts) {
         expand: args.expand
       });
     };
+    queries[`${tableNameSingular}ChunkById`] = async (_, args, context, info) => {
+      const exists = contexts.find((ctx) => ctx.id === table.id);
+      if (!exists) {
+        throw new Error("Context " + table.id + " not found in registry.");
+      }
+      const { db: db3 } = context;
+      const chunksTable = getChunksTableName(exists.id);
+      const mainTable = getTableName(exists.id);
+      const chunk = await db3(chunksTable + " as chunks").select([
+        "chunks.id as chunk_id",
+        "chunks.source as chunk_source",
+        "chunks.content as chunk_content",
+        "chunks.chunk_index",
+        "chunks.metadata as chunk_metadata",
+        db3.raw('chunks."createdAt" as chunk_created_at'),
+        db3.raw('chunks."updatedAt" as chunk_updated_at'),
+        "items.id as item_id",
+        "items.name as item_name",
+        "items.external_id as item_external_id",
+        db3.raw('items."updatedAt" as item_updated_at'),
+        db3.raw('items."createdAt" as item_created_at')
+      ]).leftJoin(mainTable + " as items", "chunks.source", "items.id").where("chunks.id", args.id).first();
+      if (!chunk) {
+        return null;
+      }
+      return {
+        chunk_content: chunk.chunk_content,
+        chunk_index: chunk.chunk_index,
+        chunk_id: chunk.chunk_id,
+        chunk_source: chunk.chunk_source,
+        chunk_metadata: chunk.chunk_metadata,
+        chunk_created_at: chunk.chunk_created_at,
+        chunk_updated_at: chunk.chunk_updated_at,
+        item_id: chunk.item_id,
+        item_name: chunk.item_name,
+        item_external_id: chunk.item_external_id,
+        item_updated_at: chunk.item_updated_at,
+        item_created_at: chunk.item_created_at
+      };
+    };
   }
   return queries;
 }
@@ -3250,120 +3366,52 @@ var vectorSearch = async ({
       const fullTextWeight = 2;
       const semanticWeight = 1;
       const rrfK = 50;
-      const hybridSQL = `
-            WITH full_text AS (
-              SELECT
-                chunks.id,
-                chunks.source,
-                row_number() OVER (
-                  ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC
-                ) AS rank_ix
-              FROM ${chunksTable} as chunks
-              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
-              WHERE chunks.fts @@ plainto_tsquery(?, ?)
-                AND ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?
-                AND (items.archived IS FALSE OR items.archived IS NULL)
-              ORDER BY rank_ix
-              LIMIT LEAST(?, 250) * 2
-            ),
-            semantic AS (
-              SELECT
-                chunks.id,
-                chunks.source,
-                row_number() OVER (
-                  ORDER BY chunks.embedding <=> ${vectorExpr} ASC
-                ) AS rank_ix
-              FROM ${chunksTable} as chunks
-              LEFT JOIN ${mainTable} as items ON items.id = chunks.source
-              WHERE chunks.embedding IS NOT NULL
-                AND (1 - (chunks.embedding <=> ${vectorExpr})) >= ?
-                AND (items.archived IS FALSE OR items.archived IS NULL)
-              ORDER BY rank_ix
-              LIMIT LEAST(?, 250) * 2
-            )
-            SELECT
-              items.id as item_id,
-              items.name as item_name,
-              items.external_id as item_external_id,
-              chunks.id AS chunk_id,
-              chunks.source,
-              chunks.content,
-              chunks.chunk_index,
-              chunks.metadata,
-              chunks."createdAt" as chunk_created_at,
-              chunks."updatedAt" as chunk_updated_at,
-              items."updatedAt" as item_updated_at,
-              items."createdAt" as item_created_at,
-              /* Per-signal scores for introspection */
-              ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank,
-              (1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance,
-        
-              /* Hybrid RRF score */
-              (
-                COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
-                +
-                COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
-              )::float AS hybrid_score
-        
-            FROM full_text ft
-            FULL OUTER JOIN semantic se
-              ON ft.id = se.id
-            JOIN ${chunksTable} as chunks
-              ON COALESCE(ft.id, se.id) = chunks.id
-            JOIN ${mainTable} as items
-              ON items.id = chunks.source
-            WHERE (
-              COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
-              +
-              COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
-            ) >= ?
-            AND (chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)
-            AND (chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)
-            ORDER BY hybrid_score DESC
-            LIMIT LEAST(?, 250)
-            OFFSET 0
-          `;
-      const bindings = [
-        // full_text: plainto_tsquery(lang, query) in rank and where
-        language,
-        query,
-        language,
-        query,
-        language,
-        query,
-        cutoffs?.tsvector || 0,
-        // full_text tsvector cutoff
-        matchCount,
-        // full_text limit
-        cutoffs?.cosineDistance || 0,
-        // semantic cosine distance cutoff
-        matchCount,
-        // semantic limit
-        // fts_rank (ts_rank) call
-        language,
-        query,
-        // RRF fusion parameters
-        rrfK,
-        fullTextWeight,
-        rrfK,
-        semanticWeight,
-        // WHERE clause hybrid_score filter
-        rrfK,
-        fullTextWeight,
-        rrfK,
-        semanticWeight,
-        cutoffs?.hybrid || 0,
-        // Additional cutoff filters in main WHERE clause
-        language,
-        query,
-        cutoffs?.tsvector || 0,
-        // tsvector cutoff for results from semantic CTE
-        cutoffs?.cosineDistance || 0,
-        // cosine distance cutoff for results from full_text CTE
-        matchCount
-        // final limit
-      ];
-      resultChunks = await db3.raw(hybridSQL, bindings).then((r) => r.rows ?? r);
+      let fullTextQuery = db3(chunksTable + " as chunks").select([
+        "chunks.id",
+        "chunks.source",
+        db3.raw(`row_number() OVER (ORDER BY ts_rank(chunks.fts, plainto_tsquery(?, ?)) DESC) AS rank_ix`, [language, query])
+      ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereRaw(`chunks.fts @@ plainto_tsquery(?, ?)`, [language, query]).whereRaw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
+      fullTextQuery = applyFilters(fullTextQuery, filters, table, "items");
+      fullTextQuery = applyAccessControl(table, fullTextQuery, user, "items");
+      let semanticQuery = db3(chunksTable + " as chunks").select([
+        "chunks.id",
+        "chunks.source",
+        db3.raw(`row_number() OVER (ORDER BY chunks.embedding <=> ${vectorExpr} ASC) AS rank_ix`)
+      ]).leftJoin(mainTable + " as items", "items.id", "chunks.source").whereNotNull("chunks.embedding").whereRaw(`(1 - (chunks.embedding <=> ${vectorExpr})) >= ?`, [cutoffs?.cosineDistance || 0]).whereRaw(`(items.archived IS FALSE OR items.archived IS NULL)`).limit(Math.min(matchCount * 2, 500));
+      semanticQuery = applyFilters(semanticQuery, filters, table, "items");
+      semanticQuery = applyAccessControl(table, semanticQuery, user, "items");
+      let hybridQuery = db3.with("full_text", fullTextQuery).with("semantic", semanticQuery).select([
+        "items.id as item_id",
+        "items.name as item_name",
+        "items.external_id as item_external_id",
+        "chunks.id as chunk_id",
+        "chunks.source",
+        "chunks.content",
+        "chunks.chunk_index",
+        "chunks.metadata",
+        db3.raw('chunks."createdAt" as chunk_created_at'),
+        db3.raw('chunks."updatedAt" as chunk_updated_at'),
+        db3.raw('items."updatedAt" as item_updated_at'),
+        db3.raw('items."createdAt" as item_created_at'),
+        db3.raw(`ts_rank(chunks.fts, plainto_tsquery(?, ?)) AS fts_rank`, [language, query]),
+        db3.raw(`(1 - (chunks.embedding <=> ${vectorExpr})) AS cosine_distance`),
+        db3.raw(`
+                        (
+                            COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
+                            +
+                            COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
+                        )::float AS hybrid_score
+                    `, [rrfK, fullTextWeight, rrfK, semanticWeight])
+      ]).from("full_text as ft").fullOuterJoin("semantic as se", "ft.id", "se.id").join(chunksTable + " as chunks", function() {
+        this.on(db3.raw("COALESCE(ft.id, se.id)"), "=", "chunks.id");
+      }).join(mainTable + " as items", "items.id", "chunks.source").whereRaw(`
+                    (
+                        COALESCE(1.0 / (? + ft.rank_ix), 0.0) * ?
+                        +
+                        COALESCE(1.0 / (? + se.rank_ix), 0.0) * ?
+                    ) >= ?
+                `, [rrfK, fullTextWeight, rrfK, semanticWeight, cutoffs?.hybrid || 0]).whereRaw(`(chunks.fts IS NULL OR ts_rank(chunks.fts, plainto_tsquery(?, ?)) > ?)`, [language, query, cutoffs?.tsvector || 0]).whereRaw(`(chunks.embedding IS NULL OR (1 - (chunks.embedding <=> ${vectorExpr})) >= ?)`, [cutoffs?.cosineDistance || 0]).orderByRaw("hybrid_score DESC").limit(Math.min(matchCount, 250));
+      resultChunks = await hybridQuery;
   }
   console.log("[EXULU] Vector search chunk results:", resultChunks?.length);
   let results = resultChunks.map((chunk) => ({
@@ -3599,8 +3647,8 @@ var contextToTableDefinition = (context) => {
     type: "text"
   });
   definition.fields.push({
-    name: "embeddings_updated_at",
-    type: "date"
+    name: "chunks_count",
+    type: "number"
   });
   definition.fields.push({
     name: "name",
@@ -3703,10 +3751,13 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     if (table.type === "items") {
       typeDefs += `
       ${tableNamePlural}VectorSearch(query: String!, method: VectorMethodEnum!, filters: [Filter${tableNameSingularUpperCaseFirst}], cutoffs: SearchCutoffs, expand: SearchExpand): ${tableNameSingular}VectorSearchResult
+      ${tableNameSingular}ChunkById(id: ID!): ${tableNameSingular}VectorSearchChunk
     `;
     }
     mutationDefs += `
       ${tableNamePlural}CreateOne(input: ${tableNameSingular}Input!, upsert: Boolean): ${tableNameSingular}MutationPayload
+      ${tableNamePlural}CopyOneById(id: ID!): ${tableNameSingular}MutationPayload
+
       ${tableNamePlural}UpdateOne(where: [Filter${tableNameSingularUpperCaseFirst}], input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
       ${tableNamePlural}UpdateOneById(id: ID!, input: ${tableNameSingular}Input!): ${tableNameSingular}MutationPayload
       ${tableNamePlural}RemoveOneById(id: ID!): ${tableNameSingular}
@@ -3714,9 +3765,9 @@ function createSDL(tables, contexts, agents, tools, config, evals, queues2) {
     `;
     if (table.type === "items") {
       mutationDefs += `
-    ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}GenerateChunksReturnPayload
+    ${tableNameSingular}GenerateChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}GenerateChunksReturnPayload
     ${tableNameSingular}ExecuteSource(source: ID!, inputs: JSON!): ${tableNameSingular}ExecuteSourceReturnPayload
-    ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}]): ${tableNameSingular}DeleteChunksReturnPayload
+    ${tableNameSingular}DeleteChunks(where: [Filter${tableNameSingularUpperCaseFirst}], limit: Int): ${tableNameSingular}DeleteChunksReturnPayload
     `;
       if (table.processor) {
         mutationDefs += `
@@ -3850,6 +3901,9 @@ type PageInfo {
   typeDefs += `
     contextById(id: ID!): Context
     `;
+  typeDefs += `
+    getUniquePromptTags: [String!]!
+    `;
   mutationDefs += `
     runEval(id: ID!, test_case_ids: [ID!]): RunEvalReturnPayload
     `;
@@ -4307,6 +4361,38 @@ type PageInfo {
     array.push("agents");
     return [...new Set(array)].sort();
   };
+  resolvers.Query["getUniquePromptTags"] = async (_, args, context, info) => {
+    const { db: db3 } = context;
+    const user = context.user;
+    const promptTable = tables.find((t) => t.name.plural === "prompt_library");
+    if (!promptTable) {
+      throw new Error("Prompt library table not found");
+    }
+    let query = db3.from("prompt_library").select("tags");
+    query = applyAccessControl(promptTable, query, user);
+    const results = await query;
+    const allTags = [];
+    for (const row of results) {
+      if (row.tags) {
+        let tags = [];
+        if (typeof row.tags === "string") {
+          try {
+            tags = JSON.parse(row.tags);
+          } catch (e) {
+            tags = [row.tags];
+          }
+        } else if (Array.isArray(row.tags)) {
+          tags = row.tags;
+        }
+        tags.forEach((tag) => {
+          if (tag && typeof tag === "string" && tag.trim()) {
+            allTags.push(tag.trim().toLowerCase());
+          }
+        });
+      }
+    }
+    return [...new Set(allTags)].sort();
+  };
   modelDefs += `
     type ProviderPaginationResult {
         items: [Provider]!
@@ -4697,14 +4783,16 @@ var addBucketPrefixToKey = (key, bucket) => {
   }
   return `${bucket}/${key}`;
 };
-var uploadFile = async (file, fileName, config, options = {}, user, customBucket) => {
+var uploadFile = async (file, fileName, config, options = {}, user, customBucket, global) => {
   if (!config.fileUploads) {
     throw new Error("File uploads are not configured (in the exported uploadFile function)");
   }
   const client2 = getS3Client(config);
   let defaultBucket = config.fileUploads.s3Bucket;
   let key = fileName;
-  key = addUserPrefixToKey(key, user || "api");
+  if (!global) {
+    key = addUserPrefixToKey(key, user || "api");
+  }
   key = addGeneralPrefixToKey(key, config);
   const sanitizedMetadata = sanitizeMetadata(options.metadata);
   const command = new PutObjectCommand({
@@ -4855,7 +4943,7 @@ var createUppyRoutes = async (app, contexts, config) => {
       return;
     }
     let allowed = false;
-    if (user.type === "api" || user.super_admin || key.includes(`user_${user.id}/`)) {
+    if (user.type === "api" || user.super_admin || !key.includes(`user_`) || key.includes(`user_${user.id}/`)) {
       allowed = true;
     }
     if (!allowed) {
@@ -4933,9 +5021,16 @@ var createUppyRoutes = async (app, contexts, config) => {
       return;
     }
     const client2 = getS3Client(config);
+    let prefix = `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}`;
+    if (!req.headers.global) {
+      prefix += `user_${authenticationResult.user.id}`;
+    } else {
+      prefix += "global";
+    }
+    console.log("[EXULU] prefix", prefix);
     const command = new ListObjectsV2Command({
       Bucket: config.fileUploads.s3Bucket,
-      Prefix: `${config.fileUploads.s3prefix ? config.fileUploads.s3prefix.replace(/\/$/, "") + "/" : ""}user_${authenticationResult.user.id}`,
+      Prefix: prefix,
       MaxKeys: 9,
       ...req.query.continuationToken && { ContinuationToken: req.query.continuationToken }
     });
@@ -5018,7 +5113,13 @@ var createUppyRoutes = async (app, contexts, config) => {
     const { filename, contentType } = extractFileParameters(req);
     validateFileParameters(filename, contentType);
     const key = generateS3Key2(filename);
-    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    let fullKey = key;
+    console.log("[EXULU] global", req.headers.global);
+    if (!req.headers.global) {
+      fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    } else {
+      fullKey = "global/" + key;
+    }
     fullKey = addGeneralPrefixToKey(fullKey, config);
     console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     getSignedUrl(
@@ -5074,7 +5175,13 @@ var createUppyRoutes = async (app, contexts, config) => {
       return res.status(400).json({ error: "s3: content type must be a string" });
     }
     const key = `${randomUUID()}-_EXULU_${filename}`;
-    let fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    let fullKey = key;
+    console.log("[EXULU] global", req.headers.global);
+    if (!req.headers.global) {
+      fullKey = addUserPrefixToKey(key, user.type === "api" ? "api" : user.id);
+    } else {
+      fullKey = "global/" + key;
+    }
     fullKey = addGeneralPrefixToKey(fullKey, config);
     console.log("[EXULU] signing on server for user", user.id, "with key", fullKey);
     const params = {
@@ -5722,6 +5829,31 @@ var ExuluAgent2 = class {
                 they are talking with the current date in mind as a reference.`;
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
+    const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
+    console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
+    console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
+    if (includesContextSearchTool) {
+      system += `
+
+
+
+            When you use a context search tool, you will include references to the items
+            retrieved from the context search tool inline in the response using this exact JSON format
+            (all on one line, no line breaks):
+            {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
+
+            IMPORTANT formatting rules:
+            - Use the exact format shown above, all on ONE line
+            - Do NOT use quotes around field names or values
+            - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
+            - Include the file/item name, not the full path
+            - Separate multiple citations with spaces
+
+            Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
+
+            The citations will be rendered as interactive badges in the UI.
+            `;
+    }
     if (prompt) {
       let result = { object: null, text: "" };
       let inputTokens = 0;
@@ -5982,6 +6114,31 @@ ${extractedText}
     const genericContext = "IMPORTANT: \n\n The current date is " + (/* @__PURE__ */ new Date()).toLocaleDateString() + " and the current time is " + (/* @__PURE__ */ new Date()).toLocaleTimeString() + ". If the user does not explicitly provide the current date, for examle when saying ' this weekend', you should assume they are talking with the current date in mind as a reference.";
     let system = instructions || "You are a helpful assistant. When you use a tool to answer a question do not explicitly comment on the result of the tool call unless the user has explicitly you to do something with the result.";
     system += "\n\n" + genericContext;
+    const includesContextSearchTool = currentTools?.some((tool2) => tool2.name.toLowerCase().includes("context_search") || tool2.id.includes("context_search"));
+    console.log("[EXULU] Current tools: " + currentTools?.map((tool2) => tool2.name));
+    console.log("[EXULU] Includes context search tool: " + includesContextSearchTool);
+    if (includesContextSearchTool) {
+      system += `
+
+
+
+            When you use a context search tool, you will include references to the items
+            retrieved from the context search tool inline in the response using this exact JSON format
+            (all on one line, no line breaks):
+            {item_name: <item_name>, item_id: <item_id>, context: <context_id>, chunk_id: <chunk_id>, chunk_index: <chunk_index>}
+
+            IMPORTANT formatting rules:
+            - Use the exact format shown above, all on ONE line
+            - Do NOT use quotes around field names or values
+            - Use the context ID (e.g., "dx-newlift-newton-knowledge-g3y6r1") from the tool result
+            - Include the file/item name, not the full path
+            - Separate multiple citations with spaces
+
+            Example: {item_name: document.pdf, item_id: abc123, context: my-context-id, chunk_id: chunk_456, chunk_index: 0}
+
+            The citations will be rendered as interactive badges in the UI.
+            `;
+    }
     const result = streamText({
       model,
       // Should be a LanguageModelV1
@@ -6525,6 +6682,7 @@ var ExuluContext = class {
       })));
     }
     await db3.from(getTableName(this.id)).where({ id: item.id }).update({
+      chunks_count: chunks?.length || 0,
       embeddings_updated_at: (/* @__PURE__ */ new Date()).toISOString()
     }).returning("id");
     return {
@@ -6794,9 +6952,13 @@ var ExuluContext = class {
           trigger: trigger || "agent"
         }, role, void 0);
       },
-      all: async (config, userId, roleId) => {
+      all: async (config, userId, roleId, limit) => {
         const { db: db3 } = await postgresClient();
-        const items = await db3.from(getTableName(this.id)).select("*");
+        let query = db3.from(getTableName(this.id)).select("*");
+        if (limit) {
+          query = query.limit(limit);
+        }
+        const items = await query;
         const jobs = [];
         const queue = await this.embedder?.queue;
         if (!queue?.queue.name && items.length > 2e3) {
@@ -6841,7 +7003,7 @@ var ExuluContext = class {
       table.text("rights_mode").defaultTo(this.configuration?.defaultRightsMode ?? "private");
       table.integer("textlength");
       table.text("source");
-      table.timestamp("embeddings_updated_at");
+      table.integer("chunks_count").defaultTo(0);
       for (const field of this.fields) {
         let { type, name, unique } = field;
         if (!type || !name) {
@@ -7370,12 +7532,6 @@ Mood: friendly and intelligent.
           return;
         }
       }
-      if (user?.type !== "api" && !user?.super_admin && req.body.resourceId !== user?.id) {
-        res.status(400).json({
-          message: "The provided user id in the resourceId field is not the same as the authenticated user. Only super admins and API users can impersonate other users."
-        });
-        return;
-      }
       console.log("[EXULU] agent tools", agentInstance.tools?.map((x) => x.name + " (" + x.id + ")"));
       const disabledTools = req.body.disabledTools ? req.body.disabledTools : [];
       let enabledTools = await getEnabledTools(agentInstance, tools, disabledTools, agents, user);
@@ -9420,45 +9576,6 @@ var createLogger = ({
 };
 var logger_default = createLogger;
 
-// src/templates/contexts/code-standards.ts
-var codeStandardsContext = new ExuluContext({
-  id: "code_standards",
-  name: "Code Standards",
-  description: "Code standards that can be used with the Exulu CLI.",
-  configuration: {
-    defaultRightsMode: "public"
-  },
-  fields: [{
-    name: "Best practices",
-    type: "longText"
-  }, {
-    name: "Code style",
-    type: "longText"
-  }, {
-    name: "Tech stack",
-    type: "longText"
-  }],
-  active: true
-});
-
-// src/templates/contexts/outputs.ts
-var outputsContext = new ExuluContext({
-  id: "outputs_default_context",
-  name: "Outputs",
-  description: "Outputs from agent sessions that you have saved for re-used later.",
-  configuration: {
-    defaultRightsMode: "private",
-    calculateVectors: "manual"
-  },
-  fields: [
-    {
-      name: "content",
-      type: "longText"
-    }
-  ],
-  active: true
-});
-
 // src/registry/index.ts
 import winston2 from "winston";
 import util from "util";
@@ -10585,9 +10702,7 @@ var ExuluApp = class {
       ...evals ?? []
     ] : [];
     this._contexts = {
-      ...contexts,
-      codeStandardsContext,
-      outputsContext
+      ...contexts
     };
     this._agents = [
       claudeSonnet4Agent,
@@ -12323,10 +12438,6 @@ var ExuluJobs = {
     validate: validateJob
   }
 };
-var ExuluDefaultContexts = {
-  codeStandards: codeStandardsContext,
-  outputs: outputsContext
-};
 var ExuluDefaultAgents = {
   anthropic: {
     opus4: claudeOpus4Agent,
@@ -12484,7 +12595,6 @@ export {
   ExuluChunkers,
   ExuluContext,
   ExuluDefaultAgents,
-  ExuluDefaultContexts,
   ExuluEmbedder,
   ExuluEval2 as ExuluEval,
   ExuluJobs,