@superatomai/sdk-node 0.0.23 → 0.0.25

package/dist/index.js

@@ -164,6 +164,7 @@ var init_utils = __esm({
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  BM25L: () => BM25L,
   CONTEXT_CONFIG: () => CONTEXT_CONFIG,
   CleanupService: () => CleanupService,
   LLM: () => LLM,
@@ -175,7 +176,10 @@ __export(index_exports, {
   UIBlock: () => UIBlock,
   UILogCollector: () => UILogCollector,
   UserManager: () => UserManager,
-  logger: () => logger
+  hybridRerank: () => hybridRerank,
+  logger: () => logger,
+  rerankChromaResults: () => rerankChromaResults,
+  rerankConversationResults: () => rerankConversationResults
 });
 module.exports = __toCommonJS(index_exports);
 
@@ -702,6 +706,36 @@ var BookmarksRequestMessageSchema = import_zod3.z.object({
   type: import_zod3.z.literal("BOOKMARKS"),
   payload: BookmarksRequestPayloadSchema
 });
+var KbNodesQueryFiltersSchema = import_zod3.z.object({
+  query: import_zod3.z.string().optional(),
+  category: import_zod3.z.string().optional(),
+  tags: import_zod3.z.array(import_zod3.z.string()).optional(),
+  createdBy: import_zod3.z.number().optional()
+});
+var KbNodesRequestPayloadSchema = import_zod3.z.object({
+  operation: import_zod3.z.enum(["create", "update", "delete", "getAll", "getOne", "search", "getByCategory", "getByUser", "getCategories", "getTags"]),
+  data: import_zod3.z.object({
+    id: import_zod3.z.number().optional(),
+    title: import_zod3.z.string().optional(),
+    content: import_zod3.z.string().optional(),
+    category: import_zod3.z.string().optional(),
+    tags: import_zod3.z.array(import_zod3.z.string()).optional(),
+    createdBy: import_zod3.z.number().optional(),
+    updatedBy: import_zod3.z.number().optional(),
+    userId: import_zod3.z.number().optional(),
+    // Query/search operation fields
+    query: import_zod3.z.string().optional(),
+    filters: KbNodesQueryFiltersSchema.optional(),
+    limit: import_zod3.z.number().optional(),
+    offset: import_zod3.z.number().optional()
+  }).optional()
+});
+var KbNodesRequestMessageSchema = import_zod3.z.object({
+  id: import_zod3.z.string(),
+  from: MessageParticipantSchema,
+  type: import_zod3.z.literal("KB_NODES"),
+  payload: KbNodesRequestPayloadSchema
+});
 
 // src/utils/logger.ts
 var import_fs = __toESM(require("fs"));
@@ -3789,6 +3823,166 @@ var KB = {
 };
 var knowledge_base_default = KB;
 
+// src/utils/bm25l-reranker.ts
+var BM25L = class {
+  /**
+   * @param documents - Array of raw documents (strings)
+   * @param opts - Optional BM25L parameters
+   */
+  constructor(documents = [], opts = {}) {
+    if (!Array.isArray(documents)) {
+      throw new Error("BM25L: documents must be an array of strings.");
+    }
+    this.k1 = typeof opts.k1 === "number" ? opts.k1 : 1.5;
+    this.b = typeof opts.b === "number" ? opts.b : 0.75;
+    this.delta = typeof opts.delta === "number" ? opts.delta : 0.5;
+    this.documents = documents.map((d) => typeof d === "string" ? this.tokenize(d) : []);
+    this.docLengths = this.documents.map((doc) => doc.length);
+    this.avgDocLength = this.docLengths.reduce((a, b) => a + b, 0) / (this.docLengths.length || 1);
+    this.termDocFreq = {};
+    this.documents.forEach((doc) => {
+      const seen = /* @__PURE__ */ new Set();
+      doc.forEach((term) => {
+        if (!seen.has(term)) {
+          seen.add(term);
+          this.termDocFreq[term] = (this.termDocFreq[term] || 0) + 1;
+        }
+      });
+    });
+  }
+  /**
+   * Tokenize text into lowercase alphanumeric tokens
+   */
+  tokenize(text) {
+    if (typeof text !== "string") return [];
+    return text.toLowerCase().replace(/[^a-z0-9\s]/g, " ").split(/\s+/).filter(Boolean);
+  }
+  /**
+   * Compute IDF (Inverse Document Frequency) with smoothing
+   */
+  idf(term) {
+    const df = this.termDocFreq[term] || 0;
+    const N = this.documents.length || 1;
+    return Math.log(1 + (N - df + 0.5) / (df + 0.5));
+  }
+  /**
+   * Compute BM25L score for a single document
+   */
+  score(query, docIndex) {
+    if (typeof query !== "string") return 0;
+    if (docIndex < 0 || docIndex >= this.documents.length) return 0;
+    const tokens = this.tokenize(query);
+    if (tokens.length === 0) return 0;
+    const doc = this.documents[docIndex];
+    const docLength = this.docLengths[docIndex] || 1;
+    const freq = {};
+    for (const t of doc) {
+      freq[t] = (freq[t] || 0) + 1;
+    }
+    let sum = 0;
+    for (const term of tokens) {
+      const tf = freq[term] || 0;
+      if (tf === 0) continue;
+      const idfVal = this.idf(term);
+      let tfL = tf - this.b * (docLength / this.avgDocLength) + this.delta;
+      if (tfL < 0) tfL = 0;
+      sum += idfVal * (tfL / (this.k1 + tfL));
+    }
+    return sum;
+  }
+  /**
+   * Search and rank all documents
+   */
+  search(query) {
+    return this.documents.map((_, i) => ({
+      index: i,
+      score: this.score(query, i)
+    })).sort((a, b) => b.score - a.score);
+  }
+};
+function normalizeScores(scores) {
+  if (scores.length === 0) return [];
+  const min = Math.min(...scores);
+  const max = Math.max(...scores);
+  if (max === min) {
+    return scores.map(() => max === 0 ? 0 : 1);
+  }
+  return scores.map((score) => (score - min) / (max - min));
+}
+function hybridRerank(query, items, getDocument, getSemanticScore, options = {}) {
+  const {
+    semanticWeight = 0.7,
+    bm25Weight = 0.3,
+    minScore = 0,
+    k1 = 1.5,
+    b = 0.75,
+    delta = 0.5
+  } = options;
+  if (items.length === 0) return [];
+  const documents = items.map(getDocument);
+  const semanticScores = items.map(getSemanticScore);
+  const bm25 = new BM25L(documents, { k1, b, delta });
+  const bm25Scores = items.map((_, i) => bm25.score(query, i));
+  const normalizedSemantic = normalizeScores(semanticScores);
+  const normalizedBM25 = normalizeScores(bm25Scores);
+  const results = items.map((item, i) => {
+    const hybridScore = semanticWeight * normalizedSemantic[i] + bm25Weight * normalizedBM25[i];
+    return {
+      item,
+      originalIndex: i,
+      semanticScore: semanticScores[i],
+      bm25Score: bm25Scores[i],
+      hybridScore
+    };
+  });
+  return results.filter((r) => r.hybridScore >= minScore).sort((a, b2) => b2.hybridScore - a.hybridScore);
+}
+function rerankChromaResults(query, chromaResults, options = {}) {
+  const ids = chromaResults.ids[0] || [];
+  const documents = chromaResults.documents[0] || [];
+  const metadatas = chromaResults.metadatas[0] || [];
+  const distances = chromaResults.distances[0] || [];
+  if (ids.length === 0) return [];
+  const items = ids.map((id, i) => ({
+    id,
+    document: documents[i],
+    metadata: metadatas[i],
+    distance: distances[i]
+  }));
+  const reranked = hybridRerank(
+    query,
+    items,
+    (item) => item.document || "",
+    // Convert L2 distance to similarity score
+    (item) => 1 / (1 + item.distance),
+    options
+  );
+  return reranked.map((r) => ({
+    id: r.item.id,
+    document: r.item.document,
+    metadata: r.item.metadata,
+    distance: r.item.distance,
+    semanticScore: r.semanticScore,
+    bm25Score: r.bm25Score,
+    hybridScore: r.hybridScore
+  }));
+}
+function rerankConversationResults(query, results, options = {}) {
+  if (results.length === 0) return [];
+  const reranked = hybridRerank(
+    query,
+    results,
+    (item) => item.userPrompt || "",
+    (item) => item.similarity || 0,
+    options
+  );
+  return reranked.map((r) => ({
+    ...r.item,
+    hybridScore: r.hybridScore,
+    bm25Score: r.bm25Score
+  }));
+}
+
 // src/userResponse/conversation-search.ts
 var searchConversations = async ({
   userPrompt,
@@ -3835,8 +4029,93 @@ var searchConversations = async ({
     return null;
   }
 };
+var searchConversationsWithReranking = async (options) => {
+  const {
+    userPrompt,
+    collections,
+    userId,
+    similarityThreshold = 0.6,
+    rerankCandidates = 50,
+    // Fetch more candidates for better reranking
+    hybridOptions = {
+      semanticWeight: 0.7,
+      bm25Weight: 0.3
+    }
+  } = options;
+  try {
+    if (!collections || !collections["conversation-history"]) {
+      logger.info("[ConversationSearch] conversation-history collection not registered, skipping");
+      return null;
+    }
+    if (!collections["conversation-history"]["searchMultiple"]) {
+      logger.info("[ConversationSearch] searchMultiple not available, falling back to standard search");
+      return searchConversations({
+        userPrompt,
+        collections,
+        userId,
+        similarityThreshold
+      });
+    }
+    logger.info(`[ConversationSearch] Hybrid search for: "${userPrompt.substring(0, 50)}..."`);
+    logger.info(`[ConversationSearch] Fetching ${rerankCandidates} candidates for reranking`);
+    logger.info(`[ConversationSearch] Weights - Semantic: ${hybridOptions.semanticWeight}, BM25: ${hybridOptions.bm25Weight}`);
+    const results = await collections["conversation-history"]["searchMultiple"]({
+      userPrompt,
+      userId,
+      limit: rerankCandidates,
+      threshold: 0
+      // No threshold - get all candidates for reranking
+    });
+    if (!results || results.length === 0) {
+      logger.info("[ConversationSearch] No conversations found in database");
+      return null;
+    }
+    logger.info(`[ConversationSearch] Retrieved ${results.length} candidates for reranking`);
+    const candidatesForReranking = results.map((r) => ({
+      ...r,
+      userPrompt: r.metadata?.userPrompt || ""
+    }));
+    const reranked = rerankConversationResults(userPrompt, candidatesForReranking, hybridOptions);
+    if (reranked.length === 0) {
+      logger.info("[ConversationSearch] No results after reranking");
+      return null;
+    }
+    const best = reranked[0];
+    const hybridScore = best.hybridScore;
+    const semanticScore = best.similarity || 0;
+    const matchedUserPrompt = best.userPrompt || best.metadata?.userPrompt || "";
+    logger.info(`[ConversationSearch] Best match after reranking:`);
+    logger.info(`  - Hybrid score: ${(hybridScore * 100).toFixed(2)}%`);
+    logger.info(`  - Semantic score: ${(semanticScore * 100).toFixed(2)}%`);
+    logger.info(`  - BM25L score: ${best.bm25Score.toFixed(4)}`);
+    logger.info(`  - Matched prompt: "${matchedUserPrompt}"`);
+    logger.info(`  - Query prompt: "${userPrompt}"`);
+    if (semanticScore < similarityThreshold) {
+      logger.info(
+        `[ConversationSearch] Semantic score ${(semanticScore * 100).toFixed(2)}% below threshold ${(similarityThreshold * 100).toFixed(2)}% - rejecting match`
+      );
+      return null;
+    }
+    logger.info(
+      `[ConversationSearch] \u2713 Found match with semantic score ${(semanticScore * 100).toFixed(2)}%`
+    );
+    logger.info(`  - Returning cached result for: "${matchedUserPrompt}"`);
+    return {
+      uiBlock: best.uiBlock,
+      similarity: semanticScore,
+      hybridScore,
+      bm25Score: best.bm25Score,
+      metadata: best.metadata
+    };
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    logger.warn(`[ConversationSearch] Error in hybrid search: ${errorMsg}`);
+    return null;
+  }
+};
 var ConversationSearch = {
-  searchConversations
+  searchConversations,
+  searchConversationsWithReranking
 };
 var conversation_search_default = ConversationSearch;
 
@@ -3885,7 +4164,7 @@ var BaseLLM = class {
    * @param componentStreamCallback - Optional callback to stream primary KPI component as soon as it's identified
    * @returns Object containing matched components, layout title/description, and follow-up actions
    */
-  async matchComponentsFromAnalysis(analysisContent, components, apiKey, logCollector, componentStreamCallback) {
+  async matchComponentsFromAnalysis(analysisContent, components, apiKey, logCollector, componentStreamCallback, deferredTools, executedTools) {
     try {
       logger.debug(`[${this.getProviderName()}] Starting component matching from text response`);
       let availableComponentsText = "No components available";
@@ -3901,12 +4180,44 @@ var BaseLLM = class {
    Props Structure: ${propsPreview}`;
         }).join("\n\n");
       }
+      let deferredToolsText = "No deferred external tools for this request.";
+      if (deferredTools && deferredTools.length > 0) {
+        logger.info(`[${this.getProviderName()}] Passing ${deferredTools.length} deferred tools to component matching`);
+        deferredToolsText = "The following external tools need user input via a Form component.\n**IMPORTANT: Use these EXACT values when generating Form externalTool prop.**\n\n" + deferredTools.map((tool, idx) => {
+          return `${idx + 1}. **${tool.name}**
+   toolId: "${tool.id}" (USE THIS EXACT VALUE - do not modify!)
+   toolName: "${tool.name}"
+   parameters: ${JSON.stringify(tool.params || {})}
+   requiredFields:
+${JSON.stringify(tool.requiredFields || [], null, 2)}`;
+        }).join("\n\n");
+      }
+      let executedToolsText = "No external tools were executed for data fetching.";
+      if (executedTools && executedTools.length > 0) {
+        logger.info(`[${this.getProviderName()}] Passing ${executedTools.length} executed tools to component matching`);
+        executedToolsText = "The following external tools were executed to fetch data.\n**IMPORTANT: For components displaying this data, use externalTool prop instead of query.**\n**IMPORTANT: Use the result data to populate deferred tool parameters when applicable.**\n\n" + executedTools.map((tool, idx) => {
+          let resultPreview = "No result data";
+          if (tool.result) {
+            const resultStr = typeof tool.result === "string" ? tool.result : JSON.stringify(tool.result, null, 2);
+            resultPreview = resultStr.length > 2e3 ? resultStr.substring(0, 2e3) + "\n... (truncated)" : resultStr;
+          }
+          return `${idx + 1}. **${tool.name}**
+   toolId: "${tool.id}" (USE THIS EXACT VALUE for externalTool.toolId)
+   toolName: "${tool.name}" (USE THIS EXACT VALUE for externalTool.toolName)
+   parameters: ${JSON.stringify(tool.params || {})} (USE THESE for externalTool.parameters)
+   result: ${resultPreview}`;
+        }).join("\n\n");
+      }
       const schemaDoc = schema.generateSchemaDocumentation();
       logger.file("\n=============================\nText analysis response:", analysisContent);
+      logger.file("\n=============================\nDeferred tools:", deferredToolsText);
+      logger.file("\n=============================\nExecuted tools:", executedToolsText);
       const prompts = await promptLoader.loadPrompts("match-text-components", {
         ANALYSIS_CONTENT: analysisContent,
         AVAILABLE_COMPONENTS: availableComponentsText,
-        SCHEMA_DOC: schemaDoc
+        SCHEMA_DOC: schemaDoc,
+        DEFERRED_TOOLS: deferredToolsText,
+        EXECUTED_TOOLS: executedToolsText
       });
       logger.debug(`[${this.getProviderName()}] Loaded match-text-components prompts`);
       logger.file("\n=============================\nmatch text components system prompt:", prompts.system);
@@ -4026,9 +4337,6 @@ var BaseLLM = class {
       matchedComponents.forEach((comp, idx) => {
         logger.info(`[${this.getProviderName()}]   ${idx + 1}. ${comp.componentType} (${comp.componentName}) - ${comp.originalSuggestion || "N/A"}`);
       });
-      if (suggestedComponents.length !== matchedComponents.length) {
-        logger.warn(`[${this.getProviderName()}] \u26A0\uFE0F MISMATCH: Text suggested ${suggestedComponents.length} components, but LLM matched ${matchedComponents.length}`);
-      }
       logger.file("\n=============================\nFull LLM response:", JSON.stringify(result, null, 2));
       const rawActions = result.actions || [];
       const actions = convertQuestionsToActions(rawActions);
@@ -4250,21 +4558,44 @@ var BaseLLM = class {
       let availableToolsDoc = "No external tools are available for this request.";
       if (externalTools && externalTools.length > 0) {
         logger.info(`[${this.getProviderName()}] External tools available: ${externalTools.map((t) => t.name).join(", ")}`);
-        availableToolsDoc = "\u26A0\uFE0F **EXECUTE THESE TOOLS IMMEDIATELY** \u26A0\uFE0F\n\nThe following external tools have been identified as necessary for this request. You MUST call them:\n\n" + externalTools.map((tool, idx) => {
-          const paramsText = Object.entries(tool.params || {}).map(([key, value]) => {
-            const valueType = typeof value;
-            if (valueType === "string" && ["string", "number", "integer", "boolean", "array", "object"].includes(String(value).toLowerCase())) {
-              return `- ${key}: ${value}`;
-            } else {
-              return `- ${key}: ${JSON.stringify(value)} (default value - use this)`;
+        const immediateTools = externalTools.filter((t) => t.executionType === "immediate" || t.executionType === "deferred" && t.userProvidedData);
+        const deferredTools = externalTools.filter((t) => t.executionType === "deferred" && !t.userProvidedData);
+        let toolsDocParts = [];
+        if (immediateTools.length > 0) {
+          const immediateDoc = "## IMMEDIATE EXECUTION TOOLS\nExecute these tools right away:\n\n" + immediateTools.map((tool, idx) => {
+            const paramsText = Object.entries(tool.params || {}).map(([key, value]) => {
+              const valueType = typeof value;
+              if (valueType === "string" && ["string", "number", "integer", "boolean", "array", "object"].includes(String(value).toLowerCase())) {
+                return `- ${key}: ${value}`;
+              } else {
+                return `- ${key}: ${JSON.stringify(value)} (default value - use this)`;
+              }
+            }).join("\n  ");
+            let userDataText = "";
+            if (tool.userProvidedData) {
+              userDataText = "\n  **User Provided Data** (use these values):\n  " + Object.entries(tool.userProvidedData).map(([key, value]) => `- ${key}: ${JSON.stringify(value)}`).join("\n  ");
             }
-          }).join("\n  ");
-          return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+            return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+  Execution Type: IMMEDIATE
   Description: ${tool.description}
-  **ACTION REQUIRED**: Call this tool with the parameters below
   Parameters:
-  ${paramsText}`;
-        }).join("\n\n");
+  ${paramsText}${userDataText}`;
+          }).join("\n\n");
+          toolsDocParts.push(immediateDoc);
+        }
+        if (deferredTools.length > 0) {
+          const deferredDoc = "## DEFERRED TOOLS (DO NOT EXECUTE)\nThese tools need user input. A Form component will be generated to collect the data.\n**DO NOT call these tools.** Instead, acknowledge the request and inform user that a form will be shown.\n\n" + deferredTools.map((tool, idx) => {
+            const requiredFieldsText = (tool.requiredFields || []).map((f) => `- ${f.label || f.name} (${f.type})${f.required ? " *required*" : ""}`).join("\n  ");
+            return `${idx + 1}. **${tool.name}** (ID: ${tool.id})
+  Execution Type: DEFERRED (needs form input)
+  Description: ${tool.description}
+  Reason: ${tool.executionReason || "Write operation requires user confirmation"}
+  Required Fields:
+  ${requiredFieldsText || "(fields will be determined by form)"}`;
+          }).join("\n\n");
+          toolsDocParts.push(deferredDoc);
+        }
+        availableToolsDoc = toolsDocParts.join("\n\n---\n\n");
       }
       const schemaDoc = schema.generateSchemaDocumentation();
       const knowledgeBaseContext = await knowledge_base_default.getKnowledgeBase({
@@ -4305,8 +4636,12 @@ var BaseLLM = class {
         }
       }];
       if (externalTools && externalTools.length > 0) {
-        externalTools.forEach((tool) => {
-          logger.info(`[${this.getProviderName()}] Processing external tool:`, JSON.stringify(tool, null, 2));
+        const executableTools = externalTools.filter(
+          (t) => t.executionType === "immediate" || t.executionType === "deferred" && t.userProvidedData
+        );
+        logger.info(`[${this.getProviderName()}] Executable tools: ${executableTools.length} of ${externalTools.length} total`);
+        executableTools.forEach((tool) => {
+          logger.info(`[${this.getProviderName()}] Processing executable tool:`, JSON.stringify(tool, null, 2));
           const properties = {};
           const required = [];
           Object.entries(tool.params || {}).forEach(([key, typeOrValue]) => {
@@ -4375,13 +4710,14 @@ var BaseLLM = class {
             input_schema: inputSchema
           });
         });
-        logger.info(`[${this.getProviderName()}] Added ${externalTools.length} external tools to tool calling capability`);
+        logger.info(`[${this.getProviderName()}] Added ${executableTools.length} executable tools to tool calling capability (${externalTools.length - executableTools.length} deferred tools await form input)`);
         logger.info(`[${this.getProviderName()}] Complete tools array:`, JSON.stringify(tools, null, 2));
       }
       const queryAttempts = /* @__PURE__ */ new Map();
       const MAX_QUERY_ATTEMPTS = 6;
       const toolAttempts = /* @__PURE__ */ new Map();
       const MAX_TOOL_ATTEMPTS = 3;
+      const executedToolsList = [];
       let maxAttemptsReached = false;
       let fullStreamedText = "";
       const wrappedStreamCallback = streamCallback ? (chunk) => {
@@ -4564,6 +4900,17 @@ Please try rephrasing your request or contact support.
               const result2 = await externalTool.fn(toolInput);
               logger.info(`[${this.getProviderName()}] External tool ${externalTool.name} executed successfully`);
               logCollector?.info(`\u2713 ${externalTool.name} executed successfully`);
+              if (!executedToolsList.find((t) => t.id === externalTool.id)) {
+                executedToolsList.push({
+                  id: externalTool.id,
+                  name: externalTool.name,
+                  params: toolInput,
+                  // The actual parameters used in this execution
+                  result: result2
+                  // Store the actual result data for populating deferred tool params
+                });
+                logger.info(`[${this.getProviderName()}] Tracked executed tool: ${externalTool.name} with params: ${JSON.stringify(toolInput)}`);
+              }
               if (wrappedStreamCallback) {
                 wrappedStreamCallback(`\u2705 **${externalTool.name} completed successfully**
 
@@ -4653,12 +5000,31 @@ ${errorMsg}
           wrappedStreamCallback(answerMarker);
           logger.info(`[${this.getProviderName()}] Streamed answer component to frontend: ${component.name} (${component.type})`);
         } : void 0;
+        const deferredTools = externalTools?.filter((t) => {
+          if (t.executionType === "deferred" && !t.userProvidedData) return true;
+          if (category === "data_modification" && !t.userProvidedData) {
+            const name = (t.name || t.id || "").toLowerCase();
+            const isWriteOperation = /create|add|insert|new|update|edit|modify|delete|remove|send/.test(name);
+            if (isWriteOperation) {
+              logger.info(`[${this.getProviderName()}] Inferred deferred execution for tool: ${t.name}`);
+              return true;
+            }
+          }
+          return false;
+        }) || [];
+        if (deferredTools.length > 0) {
+          logger.info(`[${this.getProviderName()}] Passing ${deferredTools.length} deferred tools for Form generation`);
+        }
+        logger.info(`[${this.getProviderName()}] passing deferred tools to the matching function: ${JSON.stringify(deferredTools, null, 2)}`);
+        logger.info(`[${this.getProviderName()}] passing executed tools to the matching function: ${JSON.stringify(executedToolsList, null, 2)}`);
         const matchResult = await this.matchComponentsFromAnalysis(
           textResponse,
           components,
           apiKey,
           logCollector,
-          componentStreamCallback
+          componentStreamCallback,
+          deferredTools,
+          executedToolsList
         );
         matchedComponents = matchResult.components;
         layoutTitle = matchResult.layoutTitle;
@@ -4727,23 +5093,20 @@ ${errorMsg}
    */
   async handleUserRequest(userPrompt, components, apiKey, logCollector, conversationHistory, responseMode = "text", streamCallback, collections, externalTools, userId) {
     const startTime = Date.now();
-    logger.info(`[${this.getProviderName()}] handleUserRequest called with responseMode: ${responseMode}`);
+    logger.info(`[${this.getProviderName()}] handleUserRequest called for user prompt: ${userPrompt}`);
     logCollector?.info(`Starting request processing with mode: ${responseMode}`);
     try {
       logger.info(`[${this.getProviderName()}] Step 1: Searching previous conversations...`);
       logCollector?.info("Step 1: Searching for similar previous conversations...");
-      const conversationMatch = await conversation_search_default.searchConversations({
+      const conversationMatch = await conversation_search_default.searchConversationsWithReranking({
         userPrompt,
         collections,
         userId,
         similarityThreshold: 0.6
         // 60% threshold
       });
-      logger.info("conversationMatch:", conversationMatch);
       if (conversationMatch) {
-        logger.info(
-          `[${this.getProviderName()}] \u2713 Found matching conversation with ${(conversationMatch.similarity * 100).toFixed(2)}% similarity`
-        );
+        logger.info(`[${this.getProviderName()}] \u2713 Found matching conversation with ${(conversationMatch.similarity * 100).toFixed(2)}% similarity`);
         logCollector?.info(
           `\u2713 Found similar conversation (${(conversationMatch.similarity * 100).toFixed(2)}% match)`
         );
@@ -4751,7 +5114,6 @@ ${errorMsg}
         const isValidComponent = rawComponent && typeof rawComponent === "object" && Object.keys(rawComponent).length > 0;
         const component = isValidComponent ? rawComponent : null;
         const cachedTextResponse = conversationMatch.uiBlock?.analysis || conversationMatch.uiBlock?.textResponse || conversationMatch.uiBlock?.text || "";
-        logger.debug(`[${this.getProviderName()}] Cached component: ${component ? "present" : "null"}, cachedTextResponse: ${cachedTextResponse ? cachedTextResponse.substring(0, 50) + "..." : "empty"}`);
         if (this.containsFormComponent(component)) {
           logger.info(`[${this.getProviderName()}] Skipping cached result - Form components contain stale defaultValues, fetching fresh data`);
           logCollector?.info("Skipping cache for form - fetching current values from database...");
@@ -4849,22 +5211,31 @@ ${errorMsg}
       if (categoryClassification.externalTools && categoryClassification.externalTools.length > 0) {
         logger.info(`[${this.getProviderName()}] Identified ${categoryClassification.externalTools.length} external tools needed`);
         logCollector?.info(`Identified external tools: ${categoryClassification.externalTools.map((t) => t.name || t.type).join(", ")}`);
-        toolsToUse = categoryClassification.externalTools?.map((t) => ({
-          id: t.type,
-          name: t.name,
-          description: t.description,
-          params: t.parameters || {},
-          fn: (() => {
-            const realTool = externalTools?.find((tool) => tool.id === t.type);
-            if (realTool) {
-              logger.info(`[${this.getProviderName()}] Using real tool implementation for ${t.type}`);
-              return realTool.fn;
-            } else {
-              logger.warn(`[${this.getProviderName()}] Tool ${t.type} not found in registered tools`);
-              return async () => ({ success: false, message: `Tool ${t.name || t.type} not registered` });
-            }
-          })()
-        })) || [];
+        logger.info(`[${this.getProviderName()}] Raw external tools from classification: ${JSON.stringify(categoryClassification.externalTools, null, 2)}`);
+        toolsToUse = categoryClassification.externalTools?.map((t) => {
+          const realTool = externalTools?.find((tool) => tool.id === t.type);
+          logger.info(`[${this.getProviderName()}] Tool ${t.name}: executionType=${t.executionType}, userProvidedData=${t.userProvidedData ? "present" : "null"}`);
+          return {
+            id: t.type,
+            name: t.name,
+            description: t.description,
+            params: t.parameters || {},
+            // NEW: Include execution type info from category classification
+            executionType: t.executionType || "immediate",
+            executionReason: t.executionReason || "",
+            requiredFields: t.requiredFields || [],
+            userProvidedData: t.userProvidedData || null,
+            fn: (() => {
+              if (realTool) {
+                logger.info(`[${this.getProviderName()}] Using real tool implementation for ${t.type}`);
+                return realTool.fn;
+              } else {
+                logger.warn(`[${this.getProviderName()}] Tool ${t.type} not found in registered tools`);
+                return async () => ({ success: false, message: `Tool ${t.name || t.type} not registered` });
+              }
+            })()
+          };
+        }) || [];
       }
       if (categoryClassification.category === "general") {
         logger.info(`[${this.getProviderName()}] Routing to general conversation (no database operations)`);
@@ -5555,7 +5926,6 @@ var get_user_request = async (data, components, sendMessage, anthropicApiKey, gr
   }
   logCollector.info(`Starting user prompt request with ${components.length} components`);
   const conversationHistory = thread.getConversationContext(CONTEXT_CONFIG.MAX_CONVERSATION_CONTEXT_BLOCKS, existingUiBlockId);
-  logger.info("conversationHistory", conversationHistory);
   const responseMode = payload.responseMode || "component";
   logger.info("responseMode", responseMode);
   let streamCallback;
@@ -7885,6 +8255,458 @@ function sendResponse7(id, res, sendMessage, clientId) {
   sendMessage(response);
 }
 
+// src/handlers/kb-nodes.ts
+async function handleKbNodesRequest(data, collections, sendMessage) {
+  const executeCollection = async (collection, op, params) => {
+    const handler = collections[collection]?.[op];
+    if (!handler) {
+      throw new Error(`Collection '${collection}' or operation '${op}' not found`);
+    }
+    return await handler(params);
+  };
+  try {
+    const request = KbNodesRequestMessageSchema.parse(data);
+    const { id, payload, from } = request;
+    const { operation, data: requestData } = payload;
+    const nodeId = requestData?.id;
+    const title = requestData?.title;
+    const content = requestData?.content;
+    const category = requestData?.category;
+    const tags = requestData?.tags;
+    const createdBy = requestData?.createdBy;
+    const updatedBy = requestData?.updatedBy;
+    const userId = requestData?.userId;
+    const query = requestData?.query;
+    const filters = requestData?.filters;
+    const limit = requestData?.limit;
+    const offset = requestData?.offset;
+    switch (operation) {
+      case "create":
+        await handleCreate6(id, { title, content, category, tags, createdBy }, executeCollection, sendMessage, from.id);
+        break;
+      case "update":
+        await handleUpdate6(id, nodeId, { title, content, category, tags, updatedBy }, executeCollection, sendMessage, from.id);
+        break;
+      case "delete":
+        await handleDelete6(id, nodeId, executeCollection, sendMessage, from.id);
+        break;
+      case "getAll":
+        await handleGetAll6(id, limit, offset, executeCollection, sendMessage, from.id);
+        break;
+      case "getOne":
+        await handleGetOne6(id, nodeId, executeCollection, sendMessage, from.id);
+        break;
+      case "search":
+        await handleSearch(id, { query, category, tags, createdBy, limit, offset }, executeCollection, sendMessage, from.id);
+        break;
+      case "getByCategory":
+        await handleGetByCategory(id, category, limit, offset, executeCollection, sendMessage, from.id);
+        break;
+      case "getByUser":
+        await handleGetByUser(id, userId, limit, offset, executeCollection, sendMessage, from.id);
+        break;
+      case "getCategories":
+        await handleGetCategories(id, executeCollection, sendMessage, from.id);
+        break;
+      case "getTags":
+        await handleGetTags(id, executeCollection, sendMessage, from.id);
+        break;
+      default:
+        sendResponse8(id, {
+          success: false,
+          error: `Unknown operation: ${operation}`
+        }, sendMessage, from.id);
+    }
+  } catch (error) {
+    logger.error("Failed to handle KB nodes request:", error);
+    sendResponse8(null, {
+      success: false,
+      error: error instanceof Error ? error.message : "Unknown error occurred"
+    }, sendMessage);
+  }
+}
+async function handleCreate6(id, nodeData, executeCollection, sendMessage, clientId) {
+  const { title, content, category, tags, createdBy } = nodeData;
+  if (!title || title.trim().length === 0) {
+    sendResponse8(id, {
+      success: false,
+      error: "Title is required and cannot be empty"
+    }, sendMessage, clientId);
+    return;
+  }
+  if (!content || content.trim().length === 0) {
+    sendResponse8(id, {
+      success: false,
+      error: "Content is required and cannot be empty"
+    }, sendMessage, clientId);
+    return;
+  }
+  if (!createdBy) {
+    sendResponse8(id, {
+      success: false,
+      error: "createdBy (user ID) is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "create", {
+      title,
+      content,
+      category: category || void 0,
+      tags: tags || void 0,
+      createdBy
+    });
+    if (result && result.success) {
+      logger.info(`[DB] KB node created successfully: ${title}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          ...result.data,
+          message: `Knowledge node '${title}' created successfully`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to create knowledge node"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to create KB node:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to create knowledge node"
+    }, sendMessage, clientId);
+  }
+}
+async function handleUpdate6(id, nodeId, nodeData, executeCollection, sendMessage, clientId) {
+  const { title, content, category, tags, updatedBy } = nodeData;
+  if (!nodeId) {
+    sendResponse8(id, {
+      success: false,
+      error: "Knowledge node ID is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  if (!updatedBy) {
+    sendResponse8(id, {
+      success: false,
+      error: "updatedBy (user ID) is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "update", {
+      id: nodeId,
+      title,
+      content,
+      category,
+      tags,
+      updatedBy
+    });
+    if (result && result.success) {
+      logger.info(`[DB] KB node updated successfully, ID: ${nodeId}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          ...result.data,
+          message: `Knowledge node updated successfully`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to update knowledge node"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to update KB node:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to update knowledge node"
+    }, sendMessage, clientId);
+  }
+}
+async function handleDelete6(id, nodeId, executeCollection, sendMessage, clientId) {
+  if (!nodeId) {
+    sendResponse8(id, {
+      success: false,
+      error: "Knowledge node ID is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "delete", { id: nodeId });
+    if (result && result.success) {
+      logger.info(`[DB] KB node deleted successfully, ID: ${nodeId}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          id: nodeId,
+          ...result.data,
+          message: `Knowledge node deleted successfully`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to delete knowledge node"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to delete KB node:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to delete knowledge node"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetAll6(id, limit, offset, executeCollection, sendMessage, clientId) {
+  try {
+    const result = await executeCollection("kbNodes", "getAll", {
+      limit: limit || 100,
+      offset: offset || 0
+    });
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved ${result.count} KB nodes`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          nodes: result.data,
+          count: result.count,
+          message: `Retrieved ${result.count} knowledge nodes`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve knowledge nodes"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get all KB nodes:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve knowledge nodes"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetOne6(id, nodeId, executeCollection, sendMessage, clientId) {
+  if (!nodeId) {
+    sendResponse8(id, {
+      success: false,
+      error: "Knowledge node ID is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "getOne", { id: nodeId });
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved KB node ID: ${nodeId}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          node: result.data,
+          message: `Retrieved knowledge node`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve knowledge node"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get KB node:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve knowledge node"
+    }, sendMessage, clientId);
+  }
+}
+async function handleSearch(id, searchParams, executeCollection, sendMessage, clientId) {
+  const { query, category, tags, createdBy, limit, offset } = searchParams;
+  try {
+    const result = await executeCollection("kbNodes", "search", {
+      query,
+      category,
+      tags,
+      createdBy,
+      limit: limit || 50,
+      offset: offset || 0
+    });
+    if (result && result.success) {
+      logger.info(`[DB] Search returned ${result.count} KB nodes`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          nodes: result.data,
+          count: result.count,
+          message: `Search returned ${result.count} knowledge nodes`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to search knowledge nodes"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to search KB nodes:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to search knowledge nodes"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetByCategory(id, category, limit, offset, executeCollection, sendMessage, clientId) {
+  if (!category) {
+    sendResponse8(id, {
+      success: false,
+      error: "Category is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "getByCategory", {
+      category,
+      limit: limit || 50,
+      offset: offset || 0
+    });
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved ${result.count} KB nodes for category: ${category}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          nodes: result.data,
+          count: result.count,
+          category,
+          message: `Retrieved ${result.count} knowledge nodes for category '${category}'`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve knowledge nodes by category"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get KB nodes by category:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve knowledge nodes by category"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetByUser(id, userId, limit, offset, executeCollection, sendMessage, clientId) {
+  if (!userId) {
+    sendResponse8(id, {
+      success: false,
+      error: "User ID is required"
+    }, sendMessage, clientId);
+    return;
+  }
+  try {
+    const result = await executeCollection("kbNodes", "getByUser", {
+      userId,
+      limit: limit || 50,
+      offset: offset || 0
+    });
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved ${result.count} KB nodes for user: ${userId}`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          nodes: result.data,
+          count: result.count,
+          userId,
+          message: `Retrieved ${result.count} knowledge nodes for user ${userId}`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve knowledge nodes by user"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get KB nodes by user:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve knowledge nodes by user"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetCategories(id, executeCollection, sendMessage, clientId) {
+  try {
+    const result = await executeCollection("kbNodes", "getCategories", {});
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved ${result.count} categories`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          categories: result.data,
+          count: result.count,
+          message: `Retrieved ${result.count} categories`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve categories"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get categories:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve categories"
+    }, sendMessage, clientId);
+  }
+}
+async function handleGetTags(id, executeCollection, sendMessage, clientId) {
+  try {
+    const result = await executeCollection("kbNodes", "getTags", {});
+    if (result && result.success) {
+      logger.info(`[DB] Retrieved ${result.count} tags`);
+      sendResponse8(id, {
+        success: true,
+        data: {
+          tags: result.data,
+          count: result.count,
+          message: `Retrieved ${result.count} tags`
+        }
+      }, sendMessage, clientId);
+      return;
+    }
+    sendResponse8(id, {
+      success: false,
+      error: "Failed to retrieve tags"
+    }, sendMessage, clientId);
+  } catch (error) {
+    logger.error("[DB] Failed to get tags:", error);
+    sendResponse8(id, {
+      success: false,
+      error: error instanceof Error ? error.message : "Failed to retrieve tags"
+    }, sendMessage, clientId);
+  }
+}
+function sendResponse8(id, res, sendMessage, clientId) {
+  const response = {
+    id: id || "unknown",
+    type: "KB_NODES_RES",
+    from: { type: "data-agent" },
+    to: {
+      type: "admin",
+      id: clientId
+    },
+    payload: {
+      ...res
+    }
+  };
+  sendMessage(response);
+}
+
 // src/auth/user-manager.ts
 var import_fs4 = __toESM(require("fs"));
 var import_path3 = __toESM(require("path"));
@@ -8914,6 +9736,11 @@ var SuperatomSDK = class {
             logger.error("Failed to handle bookmarks request:", error);
           });
           break;
+        case "KB_NODES":
+          handleKbNodesRequest(parsed, this.collections, (msg) => this.send(msg)).catch((error) => {
+            logger.error("Failed to handle KB nodes request:", error);
+          });
+          break;
         default:
           const handler = this.messageTypeHandlers.get(message.type);
           if (handler) {
@@ -9036,6 +9863,7 @@ var SuperatomSDK = class {
 };
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  BM25L,
   CONTEXT_CONFIG,
   CleanupService,
   LLM,
@@ -9047,6 +9875,9 @@ var SuperatomSDK = class {
   UIBlock,
   UILogCollector,
   UserManager,
-  logger
+  hybridRerank,
+  logger,
+  rerankChromaResults,
+  rerankConversationResults
 });
 //# sourceMappingURL=index.js.map