@superatomai/sdk-node 0.0.45-mds → 0.0.46-mds

package/dist/index.mjs

@@ -1678,6 +1678,21 @@ var QueryCache = class {
 };
 var queryCache = new QueryCache();
 
+// src/utils/surrogate.ts
+var LONE_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
+function stripLoneSurrogates(value) {
+  if (typeof value !== "string") return value;
+  if (!/[\uD800-\uDFFF]/.test(value)) return value;
+  return value.replace(LONE_SURROGATE_RE, "\uFFFD");
+}
+function safeTruncate(text, maxUnits) {
+  if (typeof text !== "string" || text.length <= maxUnits || maxUnits < 0) return text;
+  let end = maxUnits;
+  const lastCode = text.charCodeAt(end - 1);
+  if (lastCode >= 55296 && lastCode <= 56319) end -= 1;
+  return text.slice(0, end);
+}
+
 // src/userResponse/llm-result-truncator.ts
 var DEFAULT_MAX_ROWS = 10;
 var DEFAULT_MAX_CHARS_PER_FIELD = 500;
@@ -1720,12 +1735,12 @@ function isDateString(value) {
 }
 function truncateTextField(value, maxLength) {
   if (value.length <= maxLength) {
-    return { text: value, wasTruncated: false };
+    return { text: stripLoneSurrogates(value), wasTruncated: false };
   }
-  const truncated = value.substring(0, maxLength);
-  const remaining = value.length - maxLength;
+  const truncated = safeTruncate(value, maxLength);
+  const remaining = value.length - truncated.length;
   return {
-    text: `${truncated}... (${remaining} more chars)`,
+    text: `${stripLoneSurrogates(truncated)}... (${remaining} more chars)`,
     wasTruncated: true
   };
 }
@@ -5582,6 +5597,7 @@ var LLM = class {
   /* Get a complete text response from an LLM (Anthropic or Groq) */
   static async text(messages, options = {}) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicText(messages, modelName, options);
     } else if (provider === "groq") {
@@ -5597,6 +5613,7 @@ var LLM = class {
   /* Stream response from an LLM (Anthropic or Groq) */
   static async stream(messages, options = {}, json) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicStream(messages, modelName, options, json);
     } else if (provider === "groq") {
@@ -5612,6 +5629,7 @@ var LLM = class {
   /* Stream response with tool calling support (Anthropic and Gemini) */
   static async streamWithTools(messages, tools, toolHandler, options = {}, maxIterations = 3) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicStreamWithTools(messages, tools, toolHandler, modelName, options, maxIterations);
     } else if (provider === "gemini") {
@@ -5637,6 +5655,26 @@ var LLM = class {
     }
     return sys;
   }
+  /**
+   * Strip unpaired UTF-16 surrogates from every text field of a message set.
+   *
+   * A lone surrogate (from mid-pair string slicing or corrupt source data)
+   * serializes to a bare `\udXXX` escape that strict JSON parsers — including
+   * the one on Anthropic's API — reject with "no low surrogate in string",
+   * failing the whole request. Sanitizing here, at the single boundary every
+   * provider call flows through, guarantees no request can carry one.
+   */
+  static _sanitizeMessages(messages) {
+    const sys = typeof messages.sys === "string" ? stripLoneSurrogates(messages.sys) : messages.sys.map(
+      (block) => block?.type === "text" && typeof block.text === "string" ? { ...block, text: stripLoneSurrogates(block.text) } : block
+    );
+    return {
+      ...messages,
+      sys,
+      user: stripLoneSurrogates(messages.user),
+      ...messages.prefill !== void 0 && { prefill: stripLoneSurrogates(messages.prefill) }
+    };
+  }
   /**
    * Log cache usage metrics from Anthropic API response
    * Shows cache hits, costs, and savings
@@ -6016,12 +6054,14 @@ var LLM = class {
             let resultContent = typeof result === "string" ? result : JSON.stringify(result);
             const MAX_RESULT_LENGTH = 5e4;
             if (resultContent.length > MAX_RESULT_LENGTH) {
-              resultContent = resultContent.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
+              resultContent = safeTruncate(resultContent, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
             }
             return {
               type: "tool_result",
               tool_use_id: toolUse.id,
-              content: resultContent
+              // Final safety net: tool results carry source data and are built
+              // mid-loop (after entry-point sanitize), so strip lone surrogates here.
+              content: stripLoneSurrogates(resultContent)
             };
           } catch (error) {
             return {
@@ -6536,11 +6576,12 @@ var LLM = class {
           let resultContent = typeof result2 === "string" ? result2 : JSON.stringify(result2);
           const MAX_RESULT_LENGTH = 5e4;
           if (resultContent.length > MAX_RESULT_LENGTH) {
-            resultContent = resultContent.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
+            resultContent = safeTruncate(resultContent, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
           }
           return {
             name: fc.name,
-            response: { result: resultContent }
+            // Final safety net: strip lone surrogates from source-data results.
+            response: { result: stripLoneSurrogates(resultContent) }
           };
         } catch (error) {
           return {
@@ -6813,12 +6854,12 @@ var LLM = class {
           result = typeof toolResult === "string" ? toolResult : JSON.stringify(toolResult);
           const MAX_RESULT_LENGTH = 5e4;
           if (result.length > MAX_RESULT_LENGTH) {
-            result = result.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + result.length + " total]";
+            result = safeTruncate(result, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + result.length + " total]";
           }
         } catch (error) {
           result = JSON.stringify({ error: error instanceof Error ? error.message : String(error) });
         }
-        return { role: "tool", tool_call_id: tc.id, content: result };
+        return { role: "tool", tool_call_id: tc.id, content: stripLoneSurrogates(result) };
       }));
       toolCallResults.forEach((r) => conversationMessages.push(r));
     }
@@ -7922,7 +7963,7 @@ Execution time: ${metadata.executionTimeMs}ms
       const truncatedRow = {};
       for (const [key, value] of Object.entries(row)) {
         if (typeof value === "string" && value.length > 200) {
-          truncatedRow[key] = value.substring(0, 200) + "...";
+          truncatedRow[key] = safeTruncate(value, 200) + "...";
         } else {
           truncatedRow[key] = value;
         }
@@ -15059,7 +15100,7 @@ Fixed SQL query:`;
 }
 
 // src/dashComp/create-filter.ts
-async function createFilterWithLLM(prompt, components, existingComponents, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, tools, dashCompModels, collections) {
+async function createFilterWithLLM(prompt, components, existingComponents, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, tools, dashCompModels, collections, userId) {
   const errors = [];
   try {
     const filterComponents = components.filter((c) => c.type.startsWith("Filter"));
@@ -15079,6 +15120,25 @@ async function createFilterWithLLM(prompt, components, existingComponents, anthr
       schemaDoc = schema.generateSchemaDocumentation();
     }
     const databaseRules = await promptLoader.loadDatabaseRules();
+    let globalKnowledgeBase = "No global knowledge base available.";
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      globalKnowledgeBase = kbResult.globalContext || globalKnowledgeBase;
+      const dynamicParts = [];
+      if (kbResult.userContext) {
+        dynamicParts.push("## User-Specific Knowledge Base\n" + kbResult.userContext);
+      }
+      if (kbResult.queryContext) {
+        dynamicParts.push("## Relevant Knowledge Base (Query-Matched)\n" + kbResult.queryContext);
+      }
+      knowledgeBaseContext = dynamicParts.join("\n\n") || knowledgeBaseContext;
+    }
     const prompts = await promptLoader.loadPrompts("dash-filter-picker", {
       USER_PROMPT: prompt,
       AVAILABLE_COMPONENTS: formatComponentsForPrompt(filterComponents),
@@ -15086,8 +15146,12 @@ async function createFilterWithLLM(prompt, components, existingComponents, anthr
       SCHEMA_DOC: schemaDoc || "No database schema available",
       DATABASE_RULES: databaseRules,
       AVAILABLE_TOOLS: formatToolsForPrompt(tools),
-      CURRENT_DATETIME: getCurrentDateTimeForPrompt()
+      CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
+      GLOBAL_KNOWLEDGE_BASE: globalKnowledgeBase,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext
     });
+    logger.logLLMPrompt("dashFilterPicker", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("dashFilterPicker", "user", prompts.user);
     logger.debug("[DASH_COMP_REQ:FILTER] Loaded dash-filter-picker prompts");
     const { apiKey, model } = getApiKeyAndModel(
       anthropicApiKey,
@@ -15461,7 +15525,8 @@ var processDashCompRequest = async (data, components, _sendMessage, anthropicApi
       llmProviders,
       tools,
       dashCompModels,
-      collections
+      collections,
+      userId
     );
   } else {
     llmResponse = await pickComponentWithLLM(
@@ -15615,7 +15680,7 @@ function sendReportCompResponse(id, res, sendMessage, clientId) {
 }
 
 // src/reportComp/generate-report.ts
-async function generateReportComponents(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, modelConfig, conversationHistory) {
+async function generateReportComponents(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, modelConfig, conversationHistory, userId) {
   const errors = [];
   const availableComponentsText = formatComponentsForPrompt2(components);
   const availableToolsText = formatToolsForPrompt2(tools);
@@ -15631,6 +15696,25 @@ async function generateReportComponents(prompt, components, anthropicApiKey, gro
       schemaDoc = schema.generateSchemaDocumentation();
     }
     const databaseRules = await promptLoader.loadDatabaseRules();
+    let globalKnowledgeBase = "No global knowledge base available.";
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      globalKnowledgeBase = kbResult.globalContext || globalKnowledgeBase;
+      const dynamicParts = [];
+      if (kbResult.userContext) {
+        dynamicParts.push("## User-Specific Knowledge Base\n" + kbResult.userContext);
+      }
+      if (kbResult.queryContext) {
+        dynamicParts.push("## Relevant Knowledge Base (Query-Matched)\n" + kbResult.queryContext);
+      }
+      knowledgeBaseContext = dynamicParts.join("\n\n") || knowledgeBaseContext;
+    }
     const prompts = await promptLoader.loadPrompts("report-comp-picker", {
       USER_PROMPT: prompt,
       AVAILABLE_COMPONENTS: availableComponentsText,
@@ -15638,8 +15722,12 @@ async function generateReportComponents(prompt, components, anthropicApiKey, gro
       DATABASE_RULES: databaseRules,
       AVAILABLE_TOOLS: availableToolsText,
       CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
-      CONVERSATION_HISTORY: conversationHistory || "No previous conversation"
+      CONVERSATION_HISTORY: conversationHistory || "No previous conversation",
+      GLOBAL_KNOWLEDGE_BASE: globalKnowledgeBase,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext
     });
+    logger.logLLMPrompt("reportCompPicker", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("reportCompPicker", "user", prompts.user);
     logger.debug("[REPORT_COMP_REQ] Loaded report-comp-picker prompts with schema and tools");
     const { apiKey, model } = getApiKeyAndModel2(
       anthropicApiKey,
@@ -15864,13 +15952,21 @@ async function validateAllExternalToolQueries(components, collections, tools, mo
           data: {}
         });
         if (result?.success !== false && !result?.error) {
-          const resultData = result?.data?.data ?? result?.data ?? [];
-          const dataArray = Array.isArray(resultData) ? resultData : [resultData];
+          const toolResult = result?.data ?? result;
+          const valueKey = comp.props?.config?.valueKey;
+          const isKpi = comp.type === "KPICard" || comp.name === "DynamicKPICard";
+          let dataArray;
+          if (isKpi && valueKey && toolResult && typeof toolResult === "object" && !Array.isArray(toolResult) && toolResult[valueKey] !== void 0) {
+            dataArray = [toolResult];
+          } else {
+            const resultData = toolResult?.data ?? toolResult ?? [];
+            dataArray = Array.isArray(resultData) ? resultData : [resultData];
+          }
           if (!comp.props.config) {
             comp.props.config = {};
           }
           comp.props.config.data = dataArray;
-          logger.info(`[REPORT_COMP_REQ] \u2713 ${comp.name} prefetched ${dataArray.length} rows (non-SQL tool)`);
+          logger.info(`[REPORT_COMP_REQ] \u2713 ${comp.name} prefetched ${dataArray.length} ${dataArray.length === 1 && isKpi ? "aggregate" : "rows"} (non-SQL tool)`);
         }
       } catch (err) {
         logger.warn(`[REPORT_COMP_REQ] \u26A0 ${comp.name} non-SQL prefetch failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -16010,7 +16106,8 @@ var processReportCompRequest = async (data, components, _sendMessage, anthropicA
     collections,
     tools,
     modelConfig,
-    conversationHistory
+    conversationHistory,
+    userId
   );
   if (llmResponse.success && reportId && prompt) {
     const comps = llmResponse.data?.components;