@superatomai/sdk-node 0.0.24 → 0.0.26

package/dist/index.js

@@ -164,6 +164,7 @@ var init_utils = __esm({
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  BM25L: () => BM25L,
   CONTEXT_CONFIG: () => CONTEXT_CONFIG,
   CleanupService: () => CleanupService,
   LLM: () => LLM,
@@ -175,7 +176,10 @@ __export(index_exports, {
   UIBlock: () => UIBlock,
   UILogCollector: () => UILogCollector,
   UserManager: () => UserManager,
-  logger: () => logger
+  hybridRerank: () => hybridRerank,
+  logger: () => logger,
+  rerankChromaResults: () => rerankChromaResults,
+  rerankConversationResults: () => rerankConversationResults
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -381,6 +385,7 @@ var UserSchema = import_zod3.z.object({
   password: import_zod3.z.string().min(1, "Password is required"),
   fullname: import_zod3.z.string().optional(),
   role: import_zod3.z.string().optional(),
+  userInfo: import_zod3.z.record(import_zod3.z.unknown()).optional(),
   wsIds: import_zod3.z.array(import_zod3.z.string()).optional()
   // Only in memory, not persisted to file
 });
@@ -510,6 +515,7 @@ var UsersRequestPayloadSchema = import_zod3.z.object({
     password: import_zod3.z.string().optional(),
     fullname: import_zod3.z.string().optional(),
     role: import_zod3.z.string().optional(),
+    userInfo: import_zod3.z.record(import_zod3.z.unknown()).optional(),
     // Query operation fields
     filters: UserQueryFiltersSchema.optional(),
     limit: import_zod3.z.number().optional(),
@@ -555,7 +561,8 @@ var DashboardQueryFiltersSchema = import_zod3.z.object({
   projectId: import_zod3.z.string().optional(),
   createdBy: import_zod3.z.number().optional(),
   updatedBy: import_zod3.z.number().optional(),
-  name: import_zod3.z.string().optional()
+  name: import_zod3.z.string().optional(),
+  published: import_zod3.z.boolean().optional()
 });
 var DashboardsRequestPayloadSchema = import_zod3.z.object({
   operation: import_zod3.z.enum(["create", "update", "delete", "getAll", "getOne", "query"]),
@@ -565,6 +572,7 @@ var DashboardsRequestPayloadSchema = import_zod3.z.object({
     projectId: import_zod3.z.string().optional(),
     name: import_zod3.z.string().optional(),
     description: import_zod3.z.string().optional(),
+    published: import_zod3.z.boolean().optional(),
     createdBy: import_zod3.z.number().optional(),
     updatedBy: import_zod3.z.number().optional(),
     dashboard: DSLRendererPropsSchema.optional(),
@@ -585,7 +593,8 @@ var ReportQueryFiltersSchema = import_zod3.z.object({
   projectId: import_zod3.z.string().optional(),
   createdBy: import_zod3.z.number().optional(),
   updatedBy: import_zod3.z.number().optional(),
-  name: import_zod3.z.string().optional()
+  name: import_zod3.z.string().optional(),
+  published: import_zod3.z.boolean().optional()
 });
 var ReportsRequestPayloadSchema = import_zod3.z.object({
   operation: import_zod3.z.enum(["create", "update", "delete", "getAll", "getOne", "query"]),
@@ -595,6 +604,7 @@ var ReportsRequestPayloadSchema = import_zod3.z.object({
     projectId: import_zod3.z.string().optional(),
     name: import_zod3.z.string().optional(),
     description: import_zod3.z.string().optional(),
+    published: import_zod3.z.boolean().optional(),
     createdBy: import_zod3.z.number().optional(),
     updatedBy: import_zod3.z.number().optional(),
     report: DSLRendererPropsSchema2.optional(),
@@ -3819,6 +3829,166 @@ var KB = {
 };
 var knowledge_base_default = KB;
 
+// src/utils/bm25l-reranker.ts
+var BM25L = class {
+  /**
+   * @param documents - Array of raw documents (strings)
+   * @param opts - Optional BM25L parameters
+   */
+  constructor(documents = [], opts = {}) {
+    if (!Array.isArray(documents)) {
+      throw new Error("BM25L: documents must be an array of strings.");
+    }
+    this.k1 = typeof opts.k1 === "number" ? opts.k1 : 1.5;
+    this.b = typeof opts.b === "number" ? opts.b : 0.75;
+    this.delta = typeof opts.delta === "number" ? opts.delta : 0.5;
+    this.documents = documents.map((d) => typeof d === "string" ? this.tokenize(d) : []);
+    this.docLengths = this.documents.map((doc) => doc.length);
+    this.avgDocLength = this.docLengths.reduce((a, b) => a + b, 0) / (this.docLengths.length || 1);
+    this.termDocFreq = {};
+    this.documents.forEach((doc) => {
+      const seen = /* @__PURE__ */ new Set();
+      doc.forEach((term) => {
+        if (!seen.has(term)) {
+          seen.add(term);
+          this.termDocFreq[term] = (this.termDocFreq[term] || 0) + 1;
+        }
+      });
+    });
+  }
+  /**
+   * Tokenize text into lowercase alphanumeric tokens
+   */
+  tokenize(text) {
+    if (typeof text !== "string") return [];
+    return text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
+  }
+  /**
+   * Compute IDF (Inverse Document Frequency) with smoothing
+   */
+  idf(term) {
+    const df = this.termDocFreq[term] || 0;
+    const N = this.documents.length || 1;
+    return Math.log(1 + (N - df + 0.5) / (df + 0.5));
+  }
+  /**
+   * Compute BM25L score for a single document
+   */
+  score(query, docIndex) {
+    if (typeof query !== "string") return 0;
+    if (docIndex < 0 || docIndex >= this.documents.length) return 0;
+    const tokens = this.tokenize(query);
+    if (tokens.length === 0) return 0;
+    const doc = this.documents[docIndex];
+    const docLength = this.docLengths[docIndex] || 1;
+    const freq = {};
+    for (const t of doc) {
+      freq[t] = (freq[t] || 0) + 1;
+    }
+    let sum = 0;
+    for (const term of tokens) {
+      const tf = freq[term] || 0;
+      if (tf === 0) continue;
+      const idfVal = this.idf(term);
+      let tfL = tf - this.b * (docLength / this.avgDocLength) + this.delta;
+      if (tfL < 0) tfL = 0;
+      sum += idfVal * (tfL / (this.k1 + tfL));
+    }
+    return sum;
+  }
+  /**
+   * Search and rank all documents
+   */
+  search(query) {
+    return this.documents.map((_, i) => ({
+      index: i,
+      score: this.score(query, i)
+    })).sort((a, b) => b.score - a.score);
+  }
+};
+function normalizeScores(scores) {
+  if (scores.length === 0) return [];
+  const min = Math.min(...scores);
+  const max = Math.max(...scores);
+  if (max === min) {
+    return scores.map(() => max === 0 ? 0 : 1);
+  }
+  return scores.map((score) => (score - min) / (max - min));
+}
+function hybridRerank(query, items, getDocument, getSemanticScore, options = {}) {
+  const {
+    semanticWeight = 0.7,
+    bm25Weight = 0.3,
+    minScore = 0,
+    k1 = 1.5,
+    b = 0.75,
+    delta = 0.5
+  } = options;
+  if (items.length === 0) return [];
+  const documents = items.map(getDocument);
+  const semanticScores = items.map(getSemanticScore);
+  const bm25 = new BM25L(documents, { k1, b, delta });
+  const bm25Scores = items.map((_, i) => bm25.score(query, i));
+  const normalizedSemantic = normalizeScores(semanticScores);
+  const normalizedBM25 = normalizeScores(bm25Scores);
+  const results = items.map((item, i) => {
+    const hybridScore = semanticWeight * normalizedSemantic[i] + bm25Weight * normalizedBM25[i];
+    return {
+      item,
+      originalIndex: i,
+      semanticScore: semanticScores[i],
+      bm25Score: bm25Scores[i],
+      hybridScore
+    };
+  });
+  return results.filter((r) => r.hybridScore >= minScore).sort((a, b2) => b2.hybridScore - a.hybridScore);
+}
+function rerankChromaResults(query, chromaResults, options = {}) {
+  const ids = chromaResults.ids[0] || [];
+  const documents = chromaResults.documents[0] || [];
+  const metadatas = chromaResults.metadatas[0] || [];
+  const distances = chromaResults.distances[0] || [];
+  if (ids.length === 0) return [];
+  const items = ids.map((id, i) => ({
+    id,
+    document: documents[i],
+    metadata: metadatas[i],
+    distance: distances[i]
+  }));
+  const reranked = hybridRerank(
+    query,
+    items,
+    (item) => item.document || "",
+    // Convert L2 distance to similarity score
+    (item) => 1 / (1 + item.distance),
+    options
+  );
+  return reranked.map((r) => ({
+    id: r.item.id,
+    document: r.item.document,
+    metadata: r.item.metadata,
+    distance: r.item.distance,
+    semanticScore: r.semanticScore,
+    bm25Score: r.bm25Score,
+    hybridScore: r.hybridScore
+  }));
+}
+function rerankConversationResults(query, results, options = {}) {
+  if (results.length === 0) return [];
+  const reranked = hybridRerank(
+    query,
+    results,
+    (item) => item.userPrompt || "",
+    (item) => item.similarity || 0,
+    options
+  );
+  return reranked.map((r) => ({
+    ...r.item,
+    hybridScore: r.hybridScore,
+    bm25Score: r.bm25Score
+  }));
+}
+
 // src/userResponse/conversation-search.ts
 var searchConversations = async ({
   userPrompt,
@@ -3865,8 +4035,93 @@ var searchConversations = async ({
     return null;
   }
 };
+var searchConversationsWithReranking = async (options) => {
+  const {
+    userPrompt,
+    collections,
+    userId,
+    similarityThreshold = 0.6,
+    rerankCandidates = 50,
+    // Fetch more candidates for better reranking
+    hybridOptions = {
+      semanticWeight: 0.7,
+      bm25Weight: 0.3
+    }
+  } = options;
+  try {
+    if (!collections || !collections["conversation-history"]) {
+      logger.info("[ConversationSearch] conversation-history collection not registered, skipping");
+      return null;
+    }
+    if (!collections["conversation-history"]["searchMultiple"]) {
+      logger.info("[ConversationSearch] searchMultiple not available, falling back to standard search");
+      return searchConversations({
+        userPrompt,
+        collections,
+        userId,
+        similarityThreshold
+      });
+    }
+    logger.info(`[ConversationSearch] Hybrid search for: "${userPrompt.substring(0, 50)}..."`);
+    logger.info(`[ConversationSearch] Fetching ${rerankCandidates} candidates for reranking`);
+    logger.info(`[ConversationSearch] Weights - Semantic: ${hybridOptions.semanticWeight}, BM25: ${hybridOptions.bm25Weight}`);
+    const results = await collections["conversation-history"]["searchMultiple"]({
+      userPrompt,
+      userId,
+      limit: rerankCandidates,
+      threshold: 0
+      // No threshold - get all candidates for reranking
+    });
+    if (!results || results.length === 0) {
+      logger.info("[ConversationSearch] No conversations found in database");
+      return null;
+    }
+    logger.info(`[ConversationSearch] Retrieved ${results.length} candidates for reranking`);
+    const candidatesForReranking = results.map((r) => ({
+      ...r,
+      userPrompt: r.metadata?.userPrompt || ""
+    }));
+    const reranked = rerankConversationResults(userPrompt, candidatesForReranking, hybridOptions);
+    if (reranked.length === 0) {
+      logger.info("[ConversationSearch] No results after reranking");
+      return null;
+    }
+    const best = reranked[0];
+    const hybridScore = best.hybridScore;
+    const semanticScore = best.similarity || 0;
+    const matchedUserPrompt = best.userPrompt || best.metadata?.userPrompt || "";
+    logger.info(`[ConversationSearch] Best match after reranking:`);
+    logger.info(`  - Hybrid score: ${(hybridScore * 100).toFixed(2)}%`);
+    logger.info(`  - Semantic score: ${(semanticScore * 100).toFixed(2)}%`);
+    logger.info(`  - BM25L score: ${best.bm25Score.toFixed(4)}`);
+    logger.info(`  - Matched prompt: "${matchedUserPrompt}"`);
+    logger.info(`  - Query prompt: "${userPrompt}"`);
+    if (semanticScore < similarityThreshold) {
+      logger.info(
+        `[ConversationSearch] Semantic score ${(semanticScore * 100).toFixed(2)}% below threshold ${(similarityThreshold * 100).toFixed(2)}% - rejecting match`
+      );
+      return null;
+    }
+    logger.info(
+      `[ConversationSearch] \u2713 Found match with semantic score ${(semanticScore * 100).toFixed(2)}%`
+    );
+    logger.info(`  - Returning cached result for: "${matchedUserPrompt}"`);
+    return {
+      uiBlock: best.uiBlock,
+      similarity: semanticScore,
+      hybridScore,
+      bm25Score: best.bm25Score,
+      metadata: best.metadata
+    };
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    logger.warn(`[ConversationSearch] Error in hybrid search: ${errorMsg}`);
+    return null;
+  }
+};
 var ConversationSearch = {
-  searchConversations
+  searchConversations,
+  searchConversationsWithReranking
 };
 var conversation_search_default = ConversationSearch;
 
@@ -3915,7 +4170,7 @@ var BaseLLM = class {
    * @param componentStreamCallback - Optional callback to stream primary KPI component as soon as it's identified
    * @returns Object containing matched components, layout title/description, and follow-up actions
    */
-  async matchComponentsFromAnalysis(analysisContent, components, apiKey, logCollector, componentStreamCallback) {
+  async matchComponentsFromAnalysis(analysisContent, components, apiKey, logCollector, componentStreamCallback, deferredTools, executedTools) {
     try {
       logger.debug(`[${this.getProviderName()}] Starting component matching from text response`);
       let availableComponentsText = "No components available";
@@ -3931,12 +4186,44 @@ var BaseLLM = class {
    Props Structure: ${propsPreview}`;
         }).join("\n\n");
       }
+      let deferredToolsText = "No deferred external tools for this request.";
+      if (deferredTools && deferredTools.length > 0) {
+        logger.info(`[${this.getProviderName()}] Passing ${deferredTools.length} deferred tools to component matching`);
+        deferredToolsText = "The following external tools need user input via a Form component.\n**IMPORTANT: Use these EXACT values when generating Form externalTool prop.**\n\n" + deferredTools.map((tool, idx) => {
+          return `${idx + 1}. **${tool.name}**
+   toolId: "${tool.id}" (USE THIS EXACT VALUE - do not modify!)
+   toolName: "${tool.name}"
+   parameters: ${JSON.stringify(tool.params || {})}
+   requiredFields:
+${JSON.stringify(tool.requiredFields || [], null, 2)}`;
+        }).join("\n\n");
+      }
+      let executedToolsText = "No external tools were executed for data fetching.";
+      if (executedTools && executedTools.length > 0) {
+        logger.info(`[${this.getProviderName()}] Passing ${executedTools.length} executed tools to component matching`);
+        executedToolsText = "The following external tools were executed to fetch data.\n**IMPORTANT: For components displaying this data, use externalTool prop instead of query.**\n**IMPORTANT: Use the result data to populate deferred tool parameters when applicable.**\n\n" + executedTools.map((tool, idx) => {
+          let resultPreview = "No result data";
+          if (tool.result) {
+            const resultStr = typeof tool.result === "string" ? tool.result : JSON.stringify(tool.result, null, 2);
+            resultPreview = resultStr.length > 2e3 ? resultStr.substring(0, 2e3) + "\n... (truncated)" : resultStr;
+          }
+          return `${idx + 1}. **${tool.name}**
+   toolId: "${tool.id}" (USE THIS EXACT VALUE for externalTool.toolId)
+   toolName: "${tool.name}" (USE THIS EXACT VALUE for externalTool.toolName)
+   parameters: ${JSON.stringify(tool.params || {})} (USE THESE for externalTool.parameters)
+   result: ${resultPreview}`;
+        }).join("\n\n");
+      }
       const schemaDoc = schema.generateSchemaDocumentation();
       logger.file("\n=============================\nText analysis response:", analysisContent);
+      logger.file("\n=============================\nDeferred tools:", deferredToolsText);
+      logger.file("\n=============================\nExecuted tools:", executedToolsText);
       const prompts = await promptLoader.loadPrompts("match-text-components", {
         ANALYSIS_CONTENT: analysisContent,
         AVAILABLE_COMPONENTS: availableComponentsText,
-        SCHEMA_DOC: schemaDoc
+        SCHEMA_DOC: schemaDoc,
+        DEFERRED_TOOLS: deferredToolsText,
+        EXECUTED_TOOLS: executedToolsText
       });
       logger.debug(`[${this.getProviderName()}] Loaded match-text-components prompts`);
       logger.file("\n=============================\nmatch text components system prompt:", prompts.system);
@@ -4056,9 +4343,6 @@ var BaseLLM = class {
       matchedComponents.forEach((comp, idx) => {
         logger.info(`[${this.getProviderName()}]   ${idx + 1}. ${comp.componentType} (${comp.componentName}) - ${comp.originalSuggestion || "N/A"}`);
       });
-      if (suggestedComponents.length !== matchedComponents.length) {
-        logger.warn(`[${this.getProviderName()}] \u26A0\uFE0F MISMATCH: Text suggested ${suggestedComponents.length} components, but LLM matched ${matchedComponents.length}`);
-      }
       logger.file("\n=============================\nFull LLM response:", JSON.stringify(result, null, 2));
       const rawActions = result.actions || [];
       const actions = convertQuestionsToActions(rawActions);
@@ -4133,6 +4417,7 @@ var BaseLLM = class {
    */
   async classifyQuestionCategory(userPrompt, apiKey, logCollector, conversationHistory, externalTools) {
     try {
+      const schemaDoc = schema.generateSchemaDocumentation();
       const availableToolsDoc = externalTools && externalTools.length > 0 ? externalTools.map((tool) => {
         const paramsStr = Object.entries(tool.params || {}).map(([key, type]) => `${key}: ${type}`).join(", ");
         return `- **${tool.name}** (id: ${tool.id})
@@ -4142,7 +4427,8 @@ var BaseLLM = class {
       const prompts = await promptLoader.loadPrompts("category-classification", {
         USER_PROMPT: userPrompt,
         CONVERSATION_HISTORY: conversationHistory || "No previous conversation",
-        AVAILABLE_TOOLS: availableToolsDoc
+        AVAILABLE_TOOLS: availableToolsDoc,
+        SCHEMA_DOC: schemaDoc || "No database schema available"
       });
       const result = await LLM.stream(
         {
@@ -4280,21 +4566,44 @@ var BaseLLM = class {
       let availableToolsDoc = "No external tools are available for this request.";
       if (externalTools && externalTools.length > 0) {
         logger.info(`[${this.getProviderName()}] External tools available: ${externalTools.map((t) => t.name).join(", ")}`);
-        availableToolsDoc = "\u26A0\uFE0F **EXECUTE THESE TOOLS IMMEDIATELY** \u26A0\uFE0F\n\nThe following external tools have been identified as necessary for this request. You MUST call them:\n\n" + externalTools.map((tool, idx) => {
-          const paramsText = Object.entries(tool.params || {}).map(([key, value]) => {
-            const valueType = typeof value;
-            if (valueType === "string" && ["string", "number", "integer", "boolean", "array", "object"].includes(String(value).toLowerCase())) {
-              return `- ${key}: ${value}`;
-            } else {
-              return `- ${key}: ${JSON.stringify(value)} (default value - use this)`;
+        const immediateTools = externalTools.filter((t) => t.executionType === "immediate" || t.executionType === "deferred" && t.userProvidedData);
+        const deferredTools = externalTools.filter((t) => t.executionType === "deferred" && !t.userProvidedData);
+        let toolsDocParts = [];
+        if (immediateTools.length > 0) {
+          const immediateDoc = "## IMMEDIATE EXECUTION TOOLS\nExecute these tools right away:\n\n" + immediateTools.map((tool, idx) => {
+            const paramsText = Object.entries(tool.params || {}).map(([key, value]) => {
+              const valueType = typeof value;
+              if (valueType === "string" && ["string", "number", "integer", "boolean", "array", "object"].includes(String(value).toLowerCase())) {
+                return `- ${key}: ${value}`;
+              } else {
+                return `- ${key}: ${JSON.stringify(value)} (default value - use this)`;
+              }
+            }).join("\n  ");
+            let userDataText = "";
+            if (tool.userProvidedData) {
+              userDataText = "\n  **User Provided Data** (use these values):\n  " + Object.entries(tool.userProvidedData).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`).join("\n  ");
             }
-          }).join("\n  ");
-          return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+            return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+  Execution Type: IMMEDIATE
   Description: ${tool.description}
-  **ACTION REQUIRED**: Call this tool with the parameters below
   Parameters:
-  ${paramsText}`;
-        }).join("\n\n");
+  ${paramsText}${userDataText}`;
+          }).join("\n\n");
+          toolsDocParts.push(immediateDoc);
+        }
+        if (deferredTools.length > 0) {
+          const deferredDoc = "## DEFERRED TOOLS (DO NOT EXECUTE)\nThese tools need user input. A Form component will be generated to collect the data.\n**DO NOT call these tools.** Instead, acknowledge the request and inform user that a form will be shown.\n\n" + deferredTools.map((tool, idx) => {
+            const requiredFieldsText = (tool.requiredFields || []).map((f) => `- ${f.label || f.name} (${f.type})${f.required ? " *required*" : ""}`).join("\n  ");
+            return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+  Execution Type: DEFERRED (needs form input)
+  Description: ${tool.description}
+  Reason: ${tool.executionReason || "Write operation requires user confirmation"}
+  Required Fields:
+  ${requiredFieldsText || "(fields will be determined by form)"}`;
+          }).join("\n\n");
+          toolsDocParts.push(deferredDoc);
+        }
+        availableToolsDoc = toolsDocParts.join("\n\n---\n\n");
       }
       const schemaDoc = schema.generateSchemaDocumentation();
       const knowledgeBaseContext = await knowledge_base_default.getKnowledgeBase({
@@ -4335,8 +4644,12 @@ var BaseLLM = class {
         }
       }];
       if (externalTools && externalTools.length > 0) {
-        externalTools.forEach((tool) => {
-          logger.info(`[${this.getProviderName()}] Processing external tool:`, JSON.stringify(tool, null, 2));
+        const executableTools = externalTools.filter(
+          (t) => t.executionType === "immediate" || t.executionType === "deferred" && t.userProvidedData
+        );
+        logger.info(`[${this.getProviderName()}] Executable tools: ${executableTools.length} of ${externalTools.length} total`);
+        executableTools.forEach((tool) => {
+          logger.info(`[${this.getProviderName()}] Processing executable tool:`, JSON.stringify(tool, null, 2));
           const properties = {};
           const required = [];
           Object.entries(tool.params || {}).forEach(([key, typeOrValue]) => {
@@ -4405,13 +4718,14 @@ var BaseLLM = class {
             input_schema: inputSchema
           });
         });
-        logger.info(`[${this.getProviderName()}] Added ${externalTools.length} external tools to tool calling capability`);
+        logger.info(`[${this.getProviderName()}] Added ${executableTools.length} executable tools to tool calling capability (${externalTools.length - executableTools.length} deferred tools await form input)`);
         logger.info(`[${this.getProviderName()}] Complete tools array:`, JSON.stringify(tools, null, 2));
       }
       const queryAttempts = /* @__PURE__ */ new Map();
       const MAX_QUERY_ATTEMPTS = 6;
       const toolAttempts = /* @__PURE__ */ new Map();
       const MAX_TOOL_ATTEMPTS = 3;
+      const executedToolsList = [];
       let maxAttemptsReached = false;
       let fullStreamedText = "";
       const wrappedStreamCallback = streamCallback ? (chunk) => {
@@ -4594,6 +4908,17 @@ Please try rephrasing your request or contact support.
               const result2 = await externalTool.fn(toolInput);
               logger.info(`[${this.getProviderName()}] External tool ${externalTool.name} executed successfully`);
               logCollector?.info(`\u2713 ${externalTool.name} executed successfully`);
+              if (!executedToolsList.find((t) => t.id === externalTool.id)) {
+                executedToolsList.push({
+                  id: externalTool.id,
+                  name: externalTool.name,
+                  params: toolInput,
+                  // The actual parameters used in this execution
+                  result: result2
+                  // Store the actual result data for populating deferred tool params
+                });
+                logger.info(`[${this.getProviderName()}] Tracked executed tool: ${externalTool.name} with params: ${JSON.stringify(toolInput)}`);
+              }
               if (wrappedStreamCallback) {
                 wrappedStreamCallback(`\u2705 **${externalTool.name} completed successfully**
 
@@ -4683,12 +5008,31 @@ ${errorMsg}
           wrappedStreamCallback(answerMarker);
           logger.info(`[${this.getProviderName()}] Streamed answer component to frontend: ${component.name} (${component.type})`);
         } : void 0;
+        const deferredTools = externalTools?.filter((t) => {
+          if (t.executionType === "deferred" && !t.userProvidedData) return true;
+          if (category === "data_modification" && !t.userProvidedData) {
+            const name = (t.name || t.id || "").toLowerCase();
+            const isWriteOperation = /create|add|insert|new|update|edit|modify|delete|remove|send/.test(name);
+            if (isWriteOperation) {
+              logger.info(`[${this.getProviderName()}] Inferred deferred execution for tool: ${t.name}`);
+              return true;
+            }
+          }
+          return false;
+        }) || [];
+        if (deferredTools.length > 0) {
+          logger.info(`[${this.getProviderName()}] Passing ${deferredTools.length} deferred tools for Form generation`);
+        }
+        logger.info(`[${this.getProviderName()}] passing deferred tools to the matching function: ${JSON.stringify(deferredTools, null, 2)}`);
+        logger.info(`[${this.getProviderName()}] passing executed tools to the matching function: ${JSON.stringify(executedToolsList, null, 2)}`);
         const matchResult = await this.matchComponentsFromAnalysis(
           textResponse,
           components,
           apiKey,
           logCollector,
-          componentStreamCallback
+          componentStreamCallback,
+          deferredTools,
+          executedToolsList
         );
         matchedComponents = matchResult.components;
         layoutTitle = matchResult.layoutTitle;
@@ -4757,23 +5101,20 @@ ${errorMsg}
    */
   async handleUserRequest(userPrompt, components, apiKey, logCollector, conversationHistory, responseMode = "text", streamCallback, collections, externalTools, userId) {
     const startTime = Date.now();
-    logger.info(`[${this.getProviderName()}] handleUserRequest called with responseMode: ${responseMode}`);
+    logger.info(`[${this.getProviderName()}] handleUserRequest called for user prompt: ${userPrompt}`);
     logCollector?.info(`Starting request processing with mode: ${responseMode}`);
     try {
       logger.info(`[${this.getProviderName()}] Step 1: Searching previous conversations...`);
       logCollector?.info("Step 1: Searching for similar previous conversations...");
-      const conversationMatch = await conversation_search_default.searchConversations({
+      const conversationMatch = await conversation_search_default.searchConversationsWithReranking({
         userPrompt,
         collections,
         userId,
         similarityThreshold: 0.6
         // 60% threshold
       });
-      logger.info("conversationMatch:", conversationMatch);
       if (conversationMatch) {
-        logger.info(
-          `[${this.getProviderName()}] \u2713 Found matching conversation with ${(conversationMatch.similarity * 100).toFixed(2)}% similarity`
-        );
+        logger.info(`[${this.getProviderName()}] \u2713 Found matching conversation with ${(conversationMatch.similarity * 100).toFixed(2)}% similarity`);
         logCollector?.info(
           `\u2713 Found similar conversation (${(conversationMatch.similarity * 100).toFixed(2)}% match)`
         );
@@ -4781,7 +5122,6 @@ ${errorMsg}
         const isValidComponent = rawComponent && typeof rawComponent === "object" && Object.keys(rawComponent).length > 0;
         const component = isValidComponent ? rawComponent : null;
         const cachedTextResponse = conversationMatch.uiBlock?.analysis || conversationMatch.uiBlock?.textResponse || conversationMatch.uiBlock?.text || "";
-        logger.debug(`[${this.getProviderName()}] Cached component: ${component ? "present" : "null"}, cachedTextResponse: ${cachedTextResponse ? cachedTextResponse.substring(0, 50) + "..." : "empty"}`);
         if (this.containsFormComponent(component)) {
           logger.info(`[${this.getProviderName()}] Skipping cached result - Form components contain stale defaultValues, fetching fresh data`);
           logCollector?.info("Skipping cache for form - fetching current values from database...");
@@ -4879,22 +5219,31 @@ ${errorMsg}
       if (categoryClassification.externalTools && categoryClassification.externalTools.length > 0) {
         logger.info(`[${this.getProviderName()}] Identified ${categoryClassification.externalTools.length} external tools needed`);
         logCollector?.info(`Identified external tools: ${categoryClassification.externalTools.map((t) => t.name || t.type).join(", ")}`);
-        toolsToUse = categoryClassification.externalTools?.map((t) => ({
-          id: t.type,
-          name: t.name,
-          description: t.description,
-          params: t.parameters || {},
-          fn: (() => {
-            const realTool = externalTools?.find((tool) => tool.id === t.type);
-            if (realTool) {
-              logger.info(`[${this.getProviderName()}] Using real tool implementation for ${t.type}`);
-              return realTool.fn;
-            } else {
-              logger.warn(`[${this.getProviderName()}] Tool ${t.type} not found in registered tools`);
-              return async () => ({ success: false, message: `Tool ${t.name || t.type} not registered` });
-            }
-          })()
-        })) || [];
+        logger.info(`[${this.getProviderName()}] Raw external tools from classification: ${JSON.stringify(categoryClassification.externalTools, null, 2)}`);
+        toolsToUse = categoryClassification.externalTools?.map((t) => {
+          const realTool = externalTools?.find((tool) => tool.id === t.type);
+          logger.info(`[${this.getProviderName()}] Tool ${t.name}: executionType=${t.executionType}, userProvidedData=${t.userProvidedData ? "present" : "null"}`);
+          return {
+            id: t.type,
+            name: t.name,
+            description: t.description,
+            params: t.parameters || {},
+            // NEW: Include execution type info from category classification
+            executionType: t.executionType || "immediate",
+            executionReason: t.executionReason || "",
+            requiredFields: t.requiredFields || [],
+            userProvidedData: t.userProvidedData || null,
+            fn: (() => {
+              if (realTool) {
+                logger.info(`[${this.getProviderName()}] Using real tool implementation for ${t.type}`);
+                return realTool.fn;
+              } else {
+                logger.warn(`[${this.getProviderName()}] Tool ${t.type} not found in registered tools`);
+                return async () => ({ success: false, message: `Tool ${t.name || t.type} not registered` });
+              }
+            })()
+          };
+        }) || [];
       }
       if (categoryClassification.category === "general") {
         logger.info(`[${this.getProviderName()}] Routing to general conversation (no database operations)`);
@@ -5585,7 +5934,6 @@ var get_user_request = async (data, components, sendMessage, anthropicApiKey, gr
   }
   logCollector.info(`Starting user prompt request with ${components.length} components`);
   const conversationHistory = thread.getConversationContext(CONTEXT_CONFIG.MAX_CONVERSATION_CONTEXT_BLOCKS, existingUiBlockId);
-  logger.info("conversationHistory", conversationHistory);
   const responseMode = payload.responseMode || "component";
   logger.info("responseMode", responseMode);
   let streamCallback;
@@ -6195,6 +6543,7 @@ async function handleUsersRequest(data, collections, sendMessage) {
     const password = requestData?.password;
     const fullname = requestData?.fullname;
     const role = requestData?.role;
+    const userInfo = requestData?.userInfo;
     const filters = requestData?.filters;
     const limit = requestData?.limit;
     const sort = requestData?.sort;
@@ -6209,10 +6558,10 @@ async function handleUsersRequest(data, collections, sendMessage) {
     const userManager = getUserManager();
     switch (operation) {
       case "create":
-        await handleCreate(id, { username, email, password, fullname, role }, executeCollection, userManager, sendMessage, from.id);
+        await handleCreate(id, { username, email, password, fullname, role, userInfo }, executeCollection, userManager, sendMessage, from.id);
         break;
       case "update":
-        await handleUpdate(id, numericId, { username, email, password, fullname, role }, executeCollection, userManager, sendMessage, from.id);
+        await handleUpdate(id, numericId, { username, email, password, fullname, role, userInfo }, executeCollection, userManager, sendMessage, from.id);
         break;
       case "delete":
         await handleDelete(id, numericId, username, executeCollection, userManager, sendMessage, from.id);
@@ -6241,7 +6590,7 @@ async function handleUsersRequest(data, collections, sendMessage) {
   }
 }
 async function handleCreate(id, userData, executeCollection, userManager, sendMessage, clientId) {
-  const { username, email, password, fullname, role } = userData;
+  const { username, email, password, fullname, role, userInfo } = userData;
   if (!username || username.trim().length === 0) {
     sendResponse3(id, {
       success: false,
@@ -6272,7 +6621,8 @@ async function handleCreate(id, userData, executeCollection, userManager, sendMe
       email: email || void 0,
       password,
       fullname: fullname || void 0,
-      role: role || void 0
+      role: role || void 0,
+      userInfo: userInfo || void 0
     });
     if (result && result.success) {
       logger.info(`[DB] User created successfully: ${username}`);
@@ -6340,7 +6690,7 @@ async function handleCreate(id, userData, executeCollection, userManager, sendMe
   }
 }
 async function handleUpdate(id, numericId, userData, executeCollection, userManager, sendMessage, clientId) {
-  const { username, email, password, fullname, role } = userData;
+  const { username, email, password, fullname, role, userInfo } = userData;
   if (!numericId && !username) {
     sendResponse3(id, {
       success: false,
@@ -6356,7 +6706,8 @@ async function handleUpdate(id, numericId, userData, executeCollection, userMana
         email,
         password,
         fullname,
-        role
+        role,
+        userInfo
       });
       if (result && result.success) {
         logger.info(`[DB] User updated successfully, ID: ${numericId}`);
@@ -6716,6 +7067,7 @@ async function handleDashboardsRequest(data, collections, sendMessage) {
     const projectId = requestData?.projectId;
     const name = requestData?.name;
     const description = requestData?.description;
+    const published = requestData?.published;
     const createdBy = requestData?.createdBy;
     const updatedBy = requestData?.updatedBy;
     const numericId = requestData?.id;
@@ -6733,10 +7085,10 @@ async function handleDashboardsRequest(data, collections, sendMessage) {
     const dashboardManager2 = getDashboardManager();
     switch (operation) {
       case "create":
-        await handleCreate2(id, dashboardId, dashboard, projectId, name, description, createdBy, executeCollection, dashboardManager2, sendMessage, from.id);
+        await handleCreate2(id, dashboardId, dashboard, projectId, name, description, published, createdBy, executeCollection, dashboardManager2, sendMessage, from.id);
         break;
       case "update":
-        await handleUpdate2(id, numericId, dashboardId, dashboard, name, description, updatedBy, executeCollection, dashboardManager2, sendMessage, from.id);
+        await handleUpdate2(id, numericId, dashboardId, dashboard, name, description, published, updatedBy, executeCollection, dashboardManager2, sendMessage, from.id);
         break;
       case "delete":
         await handleDelete2(id, numericId, dashboardId, executeCollection, dashboardManager2, sendMessage, from.id);
@@ -6764,7 +7116,7 @@ async function handleDashboardsRequest(data, collections, sendMessage) {
     }, sendMessage);
   }
 }
-async function handleCreate2(id, dashboardId, dashboard, projectId, name, description, createdBy, executeCollection, dashboardManager2, sendMessage, clientId) {
+async function handleCreate2(id, dashboardId, dashboard, projectId, name, description, published, createdBy, executeCollection, dashboardManager2, sendMessage, clientId) {
   if (!dashboardId || dashboardId.trim().length === 0) {
     sendResponse4(id, {
       success: false,
@@ -6786,6 +7138,7 @@ async function handleCreate2(id, dashboardId, dashboard, projectId, name, descri
       name: name || "",
       description,
       dashboard,
+      published,
       createdBy
     });
     if (result && result.success) {
@@ -6825,7 +7178,7 @@ async function handleCreate2(id, dashboardId, dashboard, projectId, name, descri
     }, sendMessage, clientId);
   }
 }
-async function handleUpdate2(id, numericId, dashboardId, dashboard, name, description, updatedBy, executeCollection, dashboardManager2, sendMessage, clientId) {
+async function handleUpdate2(id, numericId, dashboardId, dashboard, name, description, published, updatedBy, executeCollection, dashboardManager2, sendMessage, clientId) {
   if (!numericId) {
     sendResponse4(id, {
       success: false,
@@ -6839,6 +7192,7 @@ async function handleUpdate2(id, numericId, dashboardId, dashboard, name, descri
       name,
       description,
       dashboard,
+      published,
       updatedBy
     });
     if (result && result.success) {
@@ -7081,6 +7435,7 @@ async function handleReportsRequest(data, collections, sendMessage) {
     const projectId = requestData?.projectId;
     const name = requestData?.name;
     const description = requestData?.description;
+    const published = requestData?.published;
     const createdBy = requestData?.createdBy;
     const updatedBy = requestData?.updatedBy;
     const numericId = requestData?.id;
@@ -7098,10 +7453,10 @@ async function handleReportsRequest(data, collections, sendMessage) {
     const reportManager2 = getReportManager();
     switch (operation) {
       case "create":
-        await handleCreate3(id, reportId, report, projectId, name, description, createdBy, executeCollection, reportManager2, sendMessage, from.id);
+        await handleCreate3(id, reportId, report, projectId, name, description, published, createdBy, executeCollection, reportManager2, sendMessage, from.id);
         break;
       case "update":
-        await handleUpdate3(id, numericId, reportId, report, name, description, updatedBy, executeCollection, reportManager2, sendMessage, from.id);
+        await handleUpdate3(id, numericId, reportId, report, name, description, published, updatedBy, executeCollection, reportManager2, sendMessage, from.id);
         break;
       case "delete":
         await handleDelete3(id, numericId, reportId, executeCollection, reportManager2, sendMessage, from.id);
@@ -7129,7 +7484,7 @@ async function handleReportsRequest(data, collections, sendMessage) {
     }, sendMessage);
   }
 }
-async function handleCreate3(id, reportId, report, projectId, name, description, createdBy, executeCollection, reportManager2, sendMessage, clientId) {
+async function handleCreate3(id, reportId, report, projectId, name, description, published, createdBy, executeCollection, reportManager2, sendMessage, clientId) {
   if (!reportId || reportId.trim().length === 0) {
     sendResponse5(id, {
       success: false,
@@ -7151,6 +7506,7 @@ async function handleCreate3(id, reportId, report, projectId, name, description,
       name: name || "",
       description,
       report,
+      published,
       createdBy
     });
     if (result && result.success) {
@@ -7190,7 +7546,7 @@ async function handleCreate3(id, reportId, report, projectId, name, description,
     }, sendMessage, clientId);
   }
 }
-async function handleUpdate3(id, numericId, reportId, report, name, description, updatedBy, executeCollection, reportManager2, sendMessage, clientId) {
+async function handleUpdate3(id, numericId, reportId, report, name, description, published, updatedBy, executeCollection, reportManager2, sendMessage, clientId) {
   if (!numericId) {
     sendResponse5(id, {
       success: false,
@@ -7204,6 +7560,7 @@ async function handleUpdate3(id, numericId, reportId, report, name, description,
       name,
       description,
       report,
+      published,
       updatedBy
     });
     if (result && result.success) {
@@ -9523,6 +9880,7 @@ var SuperatomSDK = class {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  BM25L,
   CONTEXT_CONFIG,
   CleanupService,
   LLM,
@@ -9534,6 +9892,9 @@ var SuperatomSDK = class {
   UIBlock,
   UILogCollector,
   UserManager,
-  logger
+  hybridRerank,
+  logger,
+  rerankChromaResults,
+  rerankConversationResults
 });
 //# sourceMappingURL=index.js.map