@superatomai/sdk-node 0.0.4-mds → 0.0.5-mds

package/dist/index.mjs

@@ -4029,6 +4029,38 @@ var PromptLoader = class {
     logger.warn(`Using default database rules for '${this.databaseType}' (file not found at ${rulesPath})`);
     return defaultRules;
   }
+  /**
+   * Load database-specific SQL rules for a given source type.
+   * Used by the agent architecture where multiple source types can coexist.
+   * @param sourceType - Source type from tool ID (e.g., 'postgres', 'mssql', 'mysql')
+   * @returns Database rules as a string
+   */
+  async loadDatabaseRulesForType(sourceType) {
+    const typeMap = {
+      "postgres": "postgresql",
+      "postgresql": "postgresql",
+      "mssql": "mssql",
+      "mysql": "postgresql"
+      // MySQL uses similar rules to PostgreSQL
+    };
+    const dbType = typeMap[sourceType] || "postgresql";
+    if (this.databaseRulesCache.has(dbType)) {
+      return this.databaseRulesCache.get(dbType);
+    }
+    const rulesPath = path2.join(this.promptsDir, "database-rules", `${dbType}.md`);
+    try {
+      if (fs3.existsSync(rulesPath)) {
+        const rules = fs3.readFileSync(rulesPath, "utf-8");
+        this.databaseRulesCache.set(dbType, rules);
+        return rules;
+      }
+    } catch (error) {
+      logger.warn(`Could not load database rules for '${dbType}' from file system: ${error}`);
+    }
+    const defaultRules = this.getDefaultDatabaseRulesForType(dbType);
+    this.databaseRulesCache.set(dbType, defaultRules);
+    return defaultRules;
+  }
   /**
    * Get default database rules as fallback
    * @returns Minimal database rules
@@ -4048,6 +4080,33 @@ var PromptLoader = class {
     }
     return `**Database Type: PostgreSQL**
 
+**SQL Query Rules:**
+- Use \`LIMIT N\` for row limiting (e.g., \`SELECT * FROM table LIMIT 32\`)
+- Use \`true\` / \`false\` for boolean values
+- Use \`||\` for string concatenation
+- Use \`NOW()\` for current timestamp
+- Use \`::TYPE\` or \`CAST()\` for type casting
+- Use \`RETURNING\` clause for mutations
+- NULL values: Use \`NULL\` keyword without quotes`;
+  }
+  /**
+   * Get default database rules for a specific type
+   */
+  getDefaultDatabaseRulesForType(dbType) {
+    if (dbType === "mssql") {
+      return `**Database Type: Microsoft SQL Server**
+
+**SQL Query Rules:**
+- Use \`TOP N\` for row limiting (e.g., \`SELECT TOP 32 * FROM table\`)
+- Use \`1\` for true, \`0\` for false (no native boolean)
+- Use \`+\` or \`CONCAT()\` for string concatenation
+- Use \`GETDATE()\` for current timestamp
+- Use \`CAST()\` or \`CONVERT()\` for type casting
+- Use \`OUTPUT INSERTED.*\` instead of \`RETURNING\`
+- NULL values: Use \`NULL\` keyword without quotes`;
+    }
+    return `**Database Type: PostgreSQL**
+
 **SQL Query Rules:**
 - Use \`LIMIT N\` for row limiting (e.g., \`SELECT * FROM table LIMIT 32\`)
 - Use \`true\` / \`false\` for boolean values
@@ -6613,9 +6672,11 @@ var SourceAgent = class {
     const { intent, aggregation = "raw" } = input;
     logger.info(`[SourceAgent:${this.tool.name}] Starting | intent: "${intent}" | aggregation: ${aggregation}`);
     if (this.streamBuffer.hasCallback()) {
+      const sourceType = this.extractSourceType();
+      const sourceLabel = sourceType !== "unknown" ? ` (${sourceType})` : "";
       this.streamBuffer.write(`
 
-\u{1F517} **Querying ${this.tool.name}...**
+\u{1F517} **Connecting to ${this.tool.name}**${sourceLabel}
 
 `);
       await streamDelay();
@@ -6635,7 +6696,7 @@ var SourceAgent = class {
         if (this.attempts > 1 && this.streamBuffer.hasCallback()) {
           this.streamBuffer.write(`
 
-\u{1F504} **Retrying ${this.tool.name} (attempt ${this.attempts}/${this.config.maxRetries})...**
+\u{1F504} **Query failed, retrying with corrected query** (attempt ${this.attempts}/${this.config.maxRetries})
 
 `);
           await streamDelay();
@@ -6645,10 +6706,29 @@ var SourceAgent = class {
           cappedInput.limit = this.config.maxRowsPerSource;
         }
         queryExecuted = cappedInput.sql || cappedInput.query || JSON.stringify(cappedInput);
+        if (this.streamBuffer.hasCallback() && queryExecuted) {
+          const queryDisplay = cappedInput.sql || cappedInput.query;
+          if (queryDisplay) {
+            this.streamBuffer.write(`\u{1F4DD} **Generated SQL query:**
+\`\`\`sql
+${queryDisplay}
+\`\`\`
+
+`);
+          } else {
+            this.streamBuffer.write(`\u{1F4DD} **Query parameters:**
+\`\`\`json
+${JSON.stringify(cappedInput, null, 2)}
+\`\`\`
+
+`);
+          }
+          await streamDelay();
+        }
         try {
           const result = await withProgressHeartbeat(
             () => this.tool.fn(cappedInput),
-            `Running ${this.tool.name}`,
+            `Executing query`,
             this.streamBuffer
           );
           if (result && result.error) {
@@ -6660,6 +6740,12 @@ Analyze the error and try again with a corrected query.`;
           }
           resultData = result.data || [];
           totalRowsMatched = result.metadata?.totalCount || result.count || resultData.length;
+          if (this.streamBuffer.hasCallback()) {
+            const totalInfo = totalRowsMatched > resultData.length ? ` (${totalRowsMatched} total matched)` : "";
+            this.streamBuffer.write(`\u2705 Retrieved ${resultData.length} rows${totalInfo}
+
+`);
+          }
           const formattedResult = formatToolResultForLLM(result, {
             toolName: this.tool.name,
             maxRows: 5,
@@ -6675,7 +6761,9 @@ Analyze the error and try again with a corrected query.`;
               _metadata: result.metadata,
               _sampleData: resultData.slice(0, 3)
             },
-            outputSchema: this.tool.outputSchema
+            outputSchema: this.tool.outputSchema,
+            sourceSchema: this.tool.description,
+            sourceType: this.extractSourceType()
           };
           const formatted = typeof formattedResult === "string" ? formattedResult : JSON.stringify(formattedResult);
           return `\u2705 Query executed successfully. ${resultData.length} rows returned (${totalRowsMatched} total matched). Data is ready \u2014 do NOT call the tool again.
@@ -7028,158 +7116,134 @@ var DEFAULT_AGENT_CONFIG = {
   maxIterations: 10
 };
 
-// src/userResponse/anthropic.ts
-import dotenv from "dotenv";
-
-// src/userResponse/schema.ts
-import path5 from "path";
-import fs6 from "fs";
-var Schema = class {
-  constructor(schemaFilePath) {
-    this.cachedSchema = null;
-    this.schemaFilePath = schemaFilePath || path5.join(process.cwd(), "../analysis/data/schema.json");
+// src/userResponse/utils/component-props-processor.ts
+function validateExternalTool(externalTool, executedTools, providerName) {
+  if (!externalTool) {
+    return { valid: true };
   }
-  /**
-   * Gets the database schema from the schema file
-   * @returns Parsed schema object or null if error occurs
-   */
-  getDatabaseSchema() {
-    try {
-      const dir = path5.dirname(this.schemaFilePath);
-      if (!fs6.existsSync(dir)) {
-        logger.info(`Creating directory structure: ${dir}`);
-        fs6.mkdirSync(dir, { recursive: true });
-      }
-      if (!fs6.existsSync(this.schemaFilePath)) {
-        logger.info(`Schema file does not exist at ${this.schemaFilePath}, creating with empty schema`);
-        const initialSchema = {
-          database: "",
-          schema: "",
-          description: "",
-          tables: [],
-          relationships: []
-        };
-        fs6.writeFileSync(this.schemaFilePath, JSON.stringify(initialSchema, null, 4));
-        this.cachedSchema = initialSchema;
-        return initialSchema;
-      }
-      const fileContent = fs6.readFileSync(this.schemaFilePath, "utf-8");
-      const schema2 = JSON.parse(fileContent);
-      this.cachedSchema = schema2;
-      return schema2;
-    } catch (error) {
-      logger.error("Error parsing schema file:", error);
-      return null;
-    }
+  const toolId = externalTool.toolId;
+  const validToolIds = (executedTools || []).map((t) => t.id);
+  const isValidTool = toolId && typeof toolId === "string" && validToolIds.includes(toolId);
+  if (!isValidTool) {
+    logger.warn(`[${providerName}] externalTool.toolId "${toolId}" not found in executed tools [${validToolIds.join(", ")}], setting to null`);
+    return { valid: false };
   }
-  /**
-   * Gets the cached schema or loads it if not cached
-   * @returns Cached schema or freshly loaded schema
-   */
-  getSchema() {
-    if (this.cachedSchema) {
-      return this.cachedSchema;
-    }
-    return this.getDatabaseSchema();
+  const executedTool = executedTools?.find((t) => t.id === toolId);
+  return { valid: true, executedTool };
+}
+function validateAndCleanQuery(query, config) {
+  if (!query) {
+    return { query: null, wasModified: false };
   }
-  /**
-   * Generates database schema documentation for LLM from Snowflake JSON schema
-   * @returns Formatted schema documentation string
-   */
-  generateSchemaDocumentation() {
-    const schema2 = this.getSchema();
-    if (!schema2) {
-      logger.warn("No database schema found.");
-      return "No database schema available.";
-    }
-    const tables = [];
-    tables.push(`Database: ${schema2.database}`);
-    tables.push(`Schema: ${schema2.schema}`);
-    tables.push(`Description: ${schema2.description}`);
-    tables.push("");
-    tables.push("=".repeat(80));
-    tables.push("");
-    for (const table of schema2.tables) {
-      const tableInfo = [];
-      tableInfo.push(`TABLE: ${table.fullName}`);
-      tableInfo.push(`Description: ${table.description}`);
-      tableInfo.push(`Row Count: ~${table.rowCount.toLocaleString()}`);
-      tableInfo.push("");
-      tableInfo.push("Columns:");
-      for (const column of table.columns) {
-        let columnLine = `  - ${column.name}: ${column.type}`;
-        if (column.isPrimaryKey) {
-          columnLine += " (PRIMARY KEY)";
-        }
-        if (column.isForeignKey && column.references) {
-          columnLine += ` (FK -> ${column.references.table}.${column.references.column})`;
-        }
-        if (!column.nullable) {
-          columnLine += " NOT NULL";
-        }
-        if (column.description) {
-          columnLine += ` - ${column.description}`;
-        }
-        tableInfo.push(columnLine);
-        if (column.sampleValues && column.sampleValues.length > 0) {
-          tableInfo.push(`    Sample values: [${column.sampleValues.join(", ")}]`);
-        }
-        if (column.statistics) {
-          const stats = column.statistics;
-          if (stats.min !== void 0 && stats.max !== void 0) {
-            tableInfo.push(`    Range: ${stats.min} to ${stats.max}`);
-          }
-          if (stats.distinct !== void 0) {
-            tableInfo.push(`    Distinct values: ${stats.distinct.toLocaleString()}`);
-          }
-        }
-      }
-      tableInfo.push("");
-      tables.push(tableInfo.join("\n"));
+  let wasModified = false;
+  let cleanedQuery = query;
+  const queryStr = typeof query === "string" ? query : query?.sql || "";
+  if (queryStr.includes("OPENJSON") || queryStr.includes("JSON_VALUE")) {
+    logger.warn(`[${config.providerName}] Query contains OPENJSON/JSON_VALUE (invalid - cannot parse tool result), setting query to null`);
+    return { query: null, wasModified: true };
+  }
+  const { query: fixedQuery, fixed, fixes } = validateAndFixSqlQuery(queryStr);
+  if (fixed) {
+    logger.warn(`[${config.providerName}] SQL fixes applied to component query: ${fixes.join("; ")}`);
+    wasModified = true;
+    if (typeof cleanedQuery === "string") {
+      cleanedQuery = fixedQuery;
+    } else if (cleanedQuery?.sql) {
+      cleanedQuery = { ...cleanedQuery, sql: fixedQuery };
     }
-    tables.push("=".repeat(80));
-    tables.push("");
-    tables.push("TABLE RELATIONSHIPS:");
-    tables.push("");
-    for (const rel of schema2.relationships) {
-      tables.push(`${rel.from} -> ${rel.to} (${rel.type}): ${rel.keys.join(" = ")}`);
+  }
+  if (typeof cleanedQuery === "string") {
+    const limitedQuery = ensureQueryLimit(cleanedQuery, config.defaultLimit, MAX_COMPONENT_QUERY_LIMIT);
+    if (limitedQuery !== cleanedQuery) wasModified = true;
+    cleanedQuery = limitedQuery;
+  } else if (cleanedQuery?.sql) {
+    const limitedSql = ensureQueryLimit(cleanedQuery.sql, config.defaultLimit, MAX_COMPONENT_QUERY_LIMIT);
+    if (limitedSql !== cleanedQuery.sql) wasModified = true;
+    cleanedQuery = { ...cleanedQuery, sql: limitedSql };
+  }
+  return { query: cleanedQuery, wasModified };
+}
+function processComponentProps(props, executedTools, config) {
+  let cleanedProps = { ...props };
+  if (cleanedProps.externalTool) {
+    const { valid } = validateExternalTool(
+      cleanedProps.externalTool,
+      executedTools,
+      config.providerName
+    );
+    if (!valid) {
+      cleanedProps.externalTool = null;
     }
-    return tables.join("\n");
   }
-  /**
-   * Clears the cached schema, forcing a reload on next access
-   */
-  clearCache() {
-    this.cachedSchema = null;
+  if (cleanedProps.query) {
+    const { query } = validateAndCleanQuery(cleanedProps.query, config);
+    cleanedProps.query = query;
   }
-  /**
-   * Sets a custom schema file path
-   * @param filePath - Path to the schema file
-   */
-  setSchemaPath(filePath) {
-    this.schemaFilePath = filePath;
-    this.clearCache();
+  if (cleanedProps.query && cleanedProps.externalTool) {
+    logger.info(`[${config.providerName}] Both query and externalTool exist, keeping both - frontend will decide`);
   }
-};
-var schema = new Schema();
+  return cleanedProps;
+}
 
-// src/userResponse/knowledge-base.ts
-var getKnowledgeBase = async ({
-  prompt,
-  collections,
-  topK = 1
-}) => {
-  try {
-    if (!collections || !collections["knowledge-base"] || !collections["knowledge-base"]["query"]) {
-      logger.warn("[KnowledgeBase] knowledge-base.query collection not registered, skipping");
-      return "";
-    }
-    const result = await collections["knowledge-base"]["query"]({
-      prompt,
-      topK
-    });
-    if (!result || !result.content) {
-      logger.warn("[KnowledgeBase] No knowledge base results returned");
+// src/userResponse/prompt-extractor.ts
+function extractPromptText(content) {
+  if (content === null || content === void 0) {
+    return "";
+  }
+  if (typeof content === "string") {
+    return content;
+  }
+  if (Array.isArray(content)) {
+    return content.map((item) => extractContentBlockText(item)).filter((text) => text.length > 0).join("\n\n---\n\n");
+  }
+  if (content && typeof content === "object") {
+    return extractObjectText(content);
+  }
+  return String(content);
+}
+function extractContentBlockText(item) {
+  if (typeof item === "string") {
+    return item;
+  }
+  if (item && typeof item === "object") {
+    const obj = item;
+    if (typeof obj.text === "string") {
+      return obj.text;
+    }
+    if (typeof obj.content === "string") {
+      return obj.content;
+    }
+    return JSON.stringify(item, null, 2);
+  }
+  return String(item);
+}
+function extractObjectText(obj) {
+  if (typeof obj.text === "string") {
+    return obj.text;
+  }
+  if (typeof obj.content === "string") {
+    return obj.content;
+  }
+  return JSON.stringify(obj, null, 2);
+}
+
+// src/userResponse/knowledge-base.ts
+var getKnowledgeBase = async ({
+  prompt,
+  collections,
+  topK = 1
+}) => {
+  try {
+    if (!collections || !collections["knowledge-base"] || !collections["knowledge-base"]["query"]) {
+      logger.warn("[KnowledgeBase] knowledge-base.query collection not registered, skipping");
+      return "";
+    }
+    const result = await collections["knowledge-base"]["query"]({
+      prompt,
+      topK
+    });
+    if (!result || !result.content) {
+      logger.warn("[KnowledgeBase] No knowledge base results returned");
       return "";
     }
     logger.info(`[KnowledgeBase] Retrieved knowledge base context (${result.content.length} chars)`);
@@ -7287,116 +7351,601 @@ var KB = {
 };
 var knowledge_base_default = KB;
 
-// src/userResponse/prompt-extractor.ts
-function extractPromptText(content) {
-  if (content === null || content === void 0) {
-    return "";
-  }
-  if (typeof content === "string") {
-    return content;
+// src/userResponse/agents/agent-component-generator.ts
+async function generateAgentComponents(params) {
+  const startTime = Date.now();
+  const {
+    analysisContent,
+    components,
+    userPrompt,
+    executedTools,
+    collections,
+    apiKey,
+    componentStreamCallback,
+    userId
+  } = params;
+  logger.info(`[AgentComponentGen] Starting | ${executedTools.length} executed tools | ${components.length} available components`);
+  try {
+    const availableComponentsText = formatAvailableComponents(components);
+    const executedToolsText = formatExecutedTools(executedTools);
+    const sourceTypes = [...new Set(executedTools.map((t) => t.sourceType).filter(Boolean))];
+    let databaseRules = "";
+    if (sourceTypes.length > 0) {
+      const rulesArr = await Promise.all(
+        sourceTypes.map((st) => promptLoader.loadDatabaseRulesForType(st))
+      );
+      databaseRules = rulesArr.join("\n\n");
+    } else {
+      databaseRules = await promptLoader.loadDatabaseRules();
+    }
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt: userPrompt || analysisContent,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      knowledgeBaseContext = kbResult.combinedContext || knowledgeBaseContext;
+    }
+    const prompts = await promptLoader.loadPrompts("match-text-components", {
+      USER_PROMPT: userPrompt || "",
+      ANALYSIS_CONTENT: analysisContent,
+      AVAILABLE_COMPONENTS: availableComponentsText,
+      SCHEMA_DOC: "Use column names from executed query results in EXECUTED_TOOLS.",
+      DATABASE_RULES: databaseRules,
+      DEFERRED_TOOLS: "No deferred external tools for this request.",
+      EXECUTED_TOOLS: executedToolsText,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext,
+      CURRENT_DATETIME: getCurrentDateTimeForPrompt()
+    });
+    logger.logLLMPrompt("agentComponentGen", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("agentComponentGen", "user", `Text Analysis:
+${analysisContent}
+
+Executed Tools:
+${executedToolsText}`);
+    let fullResponseText = "";
+    let answerComponentStreamed = false;
+    const partialCallback = componentStreamCallback ? (chunk) => {
+      fullResponseText += chunk;
+      if (!answerComponentStreamed && componentStreamCallback) {
+        const streamed = tryStreamAnswerComponent(
+          fullResponseText,
+          components,
+          executedTools,
+          collections,
+          apiKey,
+          componentStreamCallback
+        );
+        if (streamed) answerComponentStreamed = true;
+      }
+    } : void 0;
+    const result = await LLM.stream(
+      { sys: prompts.system, user: prompts.user },
+      {
+        model: "anthropic/claude-haiku-4-5-20251001",
+        maxTokens: MAX_TOKENS_COMPONENT_MATCHING,
+        temperature: 0,
+        apiKey,
+        partial: partialCallback
+      },
+      true
+      // Parse as JSON
+    );
+    const matchedComponents = result.matchedComponents || [];
+    const layoutTitle = result.layoutTitle || "Dashboard";
+    const layoutDescription = result.layoutDescription || "Multi-component dashboard";
+    if (result.hasAnswerComponent && result.answerComponent?.componentId) {
+      const answer = result.answerComponent;
+      const answerSql = answer.props?.externalTool?.parameters?.sql || "";
+      const answerTitle = answer.props?.title || "";
+      const answerType = answer.componentName || "";
+      const isDuplicate = matchedComponents.some((mc) => {
+        const mcSql = mc.props?.externalTool?.parameters?.sql || "";
+        const mcTitle = mc.props?.title || "";
+        const mcType = mc.componentName || "";
+        return mcType === answerType && (mcTitle === answerTitle || mcSql === answerSql);
+      });
+      if (!isDuplicate) {
+        matchedComponents.unshift(answer);
+      }
+    }
+    logger.info(`[AgentComponentGen] LLM returned ${matchedComponents.length} components`);
+    matchedComponents.forEach((comp, idx) => {
+      logger.info(`[AgentComponentGen]   ${idx + 1}. ${comp.componentType} (${comp.componentName})`);
+    });
+    logger.file("\n=============================\nFull LLM response:", JSON.stringify(result, null, 2));
+    const rawActions = result.actions || [];
+    const actions = convertQuestionsToActions(rawActions);
+    const finalComponents = matchedComponents.map((mc) => {
+      const originalComponent = components.find((c) => c.name === mc.componentName);
+      if (!originalComponent) {
+        logger.warn(`[AgentComponentGen] Component "${mc.componentName}" not found in available components`);
+        return null;
+      }
+      const cleanedProps = processComponentProps(
+        mc.props,
+        executedTools,
+        { providerName: "AgentComponentGen", defaultLimit: 10 }
+      );
+      return {
+        ...originalComponent,
+        props: { ...originalComponent.props, ...cleanedProps }
+      };
+    }).filter(Boolean);
+    const validatedComponents = await validateExternalToolQueries(
+      finalComponents,
+      collections,
+      executedTools,
+      apiKey
+    );
+    const elapsed = Date.now() - startTime;
+    logger.info(`[AgentComponentGen] Complete | ${validatedComponents.length}/${finalComponents.length} validated | ${elapsed}ms`);
+    return {
+      components: validatedComponents,
+      layoutTitle,
+      layoutDescription,
+      actions
+    };
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    logger.error(`[AgentComponentGen] Error: ${errorMsg}`);
+    return {
+      components: [],
+      layoutTitle: "Dashboard",
+      layoutDescription: "",
+      actions: []
+    };
   }
-  if (Array.isArray(content)) {
-    return content.map((item) => extractContentBlockText(item)).filter((text) => text.length > 0).join("\n\n---\n\n");
+}
+function formatAvailableComponents(components) {
+  if (!components || components.length === 0) return "No components available";
+  return components.map((comp, idx) => {
+    const keywords = comp.keywords ? comp.keywords.join(", ") : "";
+    const propsPreview = comp.props ? JSON.stringify(comp.props, null, 2) : "No props";
+    return `${idx + 1}. ID: ${comp.id}
+   Name: ${comp.name}
+   Type: ${comp.type}
+   Description: ${comp.description || "No description"}
+   Keywords: ${keywords}
+   Props Structure: ${propsPreview}`;
+  }).join("\n\n");
+}
+function formatExecutedTools(executedTools) {
+  if (!executedTools || executedTools.length === 0) {
+    return "No external tools were executed for data fetching.";
+  }
+  return "The following external tools were executed to fetch data.\n" + executedTools.map((tool, idx) => {
+    let outputSchemaText = "Not available";
+    let fieldNamesList = "";
+    const recordCount = tool.result?._totalRecords ?? "unknown";
+    let metadataText = "";
+    if (tool.result?._metadata && Object.keys(tool.result._metadata).length > 0) {
+      const metadataEntries = Object.entries(tool.result._metadata).map(([key, value]) => `${key}: ${value}`).join(", ");
+      metadataText = `
+   \u{1F4CB} METADATA: ${metadataEntries}`;
+    }
+    if (tool.outputSchema) {
+      const fields = tool.outputSchema.fields || [];
+      const numericFields = fields.filter((f) => f.type === "number").map((f) => f.name);
+      const stringFields = fields.filter((f) => f.type === "string").map((f) => f.name);
+      fieldNamesList = `
+   \u{1F4CA} NUMERIC FIELDS (use for yAxisKey, valueKey, aggregationField): ${numericFields.join(", ") || "none"}
+   \u{1F4DD} STRING FIELDS (use for xAxisKey, groupBy, nameKey): ${stringFields.join(", ") || "none"}`;
+      const fieldsText = fields.map(
+        (f) => `     "${f.name}" (${f.type}): ${f.description}`
+      ).join("\n");
+      outputSchemaText = `${tool.outputSchema.description}
+   Fields:
+${fieldsText}`;
+    }
+    return `${idx + 1}. **${tool.name}**
+   toolId: "${tool.id}"
+   toolName: "${tool.name}"
+   parameters: ${JSON.stringify(tool.params || {})}
+   recordCount: ${recordCount} rows returned${metadataText}
+   outputSchema: ${outputSchemaText}${fieldNamesList}`;
+  }).join("\n\n");
+}
+function tryStreamAnswerComponent(text, components, executedTools, collections, apiKey, callback) {
+  const hasMatch = text.match(/"hasAnswerComponent"\s*:\s*(true|false)/);
+  if (!hasMatch || hasMatch[1] !== "true") return false;
+  const startMatch = text.match(/"answerComponent"\s*:\s*\{/);
+  if (!startMatch) return false;
+  const startPos = startMatch.index + startMatch[0].length - 1;
+  let depth = 0;
+  let inString = false;
+  let escapeNext = false;
+  let endPos = -1;
+  for (let i = startPos; i < text.length; i++) {
+    const char = text[i];
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+    if (char === "\\") {
+      escapeNext = true;
+      continue;
+    }
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (!inString) {
+      if (char === "{") depth++;
+      else if (char === "}") {
+        depth--;
+        if (depth === 0) {
+          endPos = i + 1;
+          break;
+        }
+      }
+    }
   }
-  if (content && typeof content === "object") {
-    return extractObjectText(content);
+  if (endPos <= startPos) return false;
+  try {
+    const answerData = JSON.parse(text.substring(startPos, endPos));
+    if (!answerData?.componentId) return false;
+    const original = components.find((c) => c.id === answerData.componentId);
+    if (!original) return false;
+    const answerComponent = {
+      ...original,
+      props: { ...original.props, ...answerData.props }
+    };
+    const answerProps = answerComponent.props;
+    const sql = answerProps?.externalTool?.parameters?.sql;
+    if (sql && collections?.["external-tools"]?.["execute"]) {
+      const toolId = answerProps.externalTool.toolId;
+      const toolName = answerProps.externalTool.toolName;
+      (async () => {
+        try {
+          const result = await collections["external-tools"]["execute"]({
+            toolId,
+            toolName,
+            sql,
+            data: {}
+          });
+          if (result?.success === false || result?.error) {
+            logger.warn(`[AgentComponentGen] Answer component query failed: ${result?.error}`);
+            return;
+          }
+          queryCache.set(sql, result?.data ?? result);
+          callback(answerComponent);
+        } catch (err) {
+          const msg = err instanceof Error ? err.message : String(err);
+          logger.warn(`[AgentComponentGen] Answer component validation failed: ${msg}`);
+        }
+      })();
+    } else {
+      callback(answerComponent);
+    }
+    return true;
+  } catch {
+    return false;
   }
-  return String(content);
 }
-function extractContentBlockText(item) {
-  if (typeof item === "string") {
-    return item;
+async function validateExternalToolQueries(components, collections, executedTools, apiKey) {
+  if (!collections?.["external-tools"]?.["execute"]) {
+    logger.warn(`[AgentComponentGen] external-tools.execute not available, skipping validation`);
+    return components;
   }
-  if (item && typeof item === "object") {
-    const obj = item;
-    if (typeof obj.text === "string") {
-      return obj.text;
+  const validated = [];
+  const withSql = [];
+  const withoutSql = [];
+  for (const comp of components) {
+    const sql = comp.props?.externalTool?.parameters?.sql;
+    if (sql) {
+      withSql.push(comp);
+    } else {
+      withoutSql.push(comp);
     }
-    if (typeof obj.content === "string") {
-      return obj.content;
+  }
+  validated.push(...withoutSql);
+  if (withSql.length === 0) return validated;
+  const sqlGroups = /* @__PURE__ */ new Map();
+  for (const comp of withSql) {
+    const sql = comp.props.externalTool.parameters.sql;
+    const normalized = sql.replace(/\s+/g, " ").trim();
+    if (!sqlGroups.has(normalized)) {
+      sqlGroups.set(normalized, []);
     }
-    return JSON.stringify(item, null, 2);
+    sqlGroups.get(normalized).push(comp);
   }
-  return String(item);
-}
-function extractObjectText(obj) {
-  if (typeof obj.text === "string") {
-    return obj.text;
+  const uniqueQueries = Array.from(sqlGroups.entries());
+  logger.info(`[AgentComponentGen] Validating ${uniqueQueries.length} unique queries (${withSql.length} components)...`);
+  const results = await Promise.allSettled(
+    uniqueQueries.map(([_, comps]) => validateSingleExternalToolQuery(comps[0], collections, executedTools, apiKey))
+  );
+  for (let i = 0; i < results.length; i++) {
+    const result = results[i];
+    const [_, groupComps] = uniqueQueries[i];
+    if (result.status === "fulfilled" && result.value) {
+      validated.push(result.value);
+      for (let j = 1; j < groupComps.length; j++) {
+        const fixedSql = result.value.props?.externalTool?.parameters?.sql;
+        if (fixedSql) {
+          const siblingProps = groupComps[j].props;
+          validated.push({
+            ...groupComps[j],
+            props: {
+              ...groupComps[j].props,
+              externalTool: {
+                ...siblingProps.externalTool,
+                parameters: { ...siblingProps.externalTool.parameters, sql: fixedSql }
+              }
+            }
+          });
+        } else {
+          validated.push(groupComps[j]);
+        }
+      }
+    } else {
+      const reason = result.status === "rejected" ? result.reason : "validation failed";
+      for (const comp of groupComps) {
+        logger.warn(`[AgentComponentGen] Excluded ${comp.name}: ${reason}`);
+      }
+    }
   }
-  if (typeof obj.content === "string") {
-    return obj.content;
+  logger.info(`[AgentComponentGen] Validation complete: ${validated.length}/${components.length} components passed`);
+  return validated;
+}
+async function validateSingleExternalToolQuery(component, collections, executedTools, apiKey) {
+  const compProps = component.props;
+  const toolId = compProps?.externalTool?.toolId;
+  const toolName = compProps?.externalTool?.toolName;
+  let currentSql = compProps?.externalTool?.parameters?.sql;
+  if (!toolId || !currentSql) return component;
+  currentSql = ensureQueryLimit(currentSql, 10, MAX_COMPONENT_QUERY_LIMIT);
+  let attempts = 0;
+  while (attempts < MAX_QUERY_VALIDATION_RETRIES) {
+    attempts++;
+    try {
+      logger.info(`[AgentComponentGen] Validating ${component.name} (attempt ${attempts}/${MAX_QUERY_VALIDATION_RETRIES})`);
+      const result = await collections["external-tools"]["execute"]({
+        toolId,
+        toolName,
+        sql: currentSql,
+        data: {}
+      });
+      if (result?.success === false || result?.error) {
+        const errorMsg = result?.error || "Unknown error";
+        throw new Error(typeof errorMsg === "string" ? errorMsg : JSON.stringify(errorMsg));
+      }
+      const rawToolData = result?.data ?? result;
+      queryCache.set(currentSql, rawToolData);
+      logger.info(`[AgentComponentGen] \u2713 ${component.name} validated (attempt ${attempts})`);
+      return {
+        ...component,
+        props: {
+          ...component.props,
+          externalTool: {
+            ...compProps.externalTool,
+            parameters: {
+              ...compProps.externalTool.parameters,
+              sql: currentSql
+            }
+          }
+        }
+      };
+    } catch (error) {
+      const errorMsg = error instanceof Error ? error.message : String(error);
+      logger.warn(`[AgentComponentGen] \u2717 ${component.name} failed (attempt ${attempts}): ${errorMsg}`);
+      if (attempts >= MAX_QUERY_VALIDATION_RETRIES) {
+        logger.error(`[AgentComponentGen] Max retries reached for ${component.name}, excluding`);
+        return null;
+      }
+      try {
+        const fixedSql = await requestExternalToolQueryFix(
+          currentSql,
+          errorMsg,
+          component,
+          executedTools,
+          toolId,
+          apiKey
+        );
+        if (fixedSql && fixedSql !== currentSql) {
+          currentSql = ensureQueryLimit(fixedSql, 10, MAX_COMPONENT_QUERY_LIMIT);
+          logger.info(`[AgentComponentGen] LLM provided fix for ${component.name}, retrying...`);
+        } else {
+          logger.warn(`[AgentComponentGen] LLM returned same or empty query, stopping retries`);
+          return null;
+        }
+      } catch (fixError) {
+        const fixMsg = fixError instanceof Error ? fixError.message : String(fixError);
+        logger.error(`[AgentComponentGen] Failed to get LLM fix: ${fixMsg}`);
+        return null;
+      }
+    }
   }
-  return JSON.stringify(obj, null, 2);
+  return null;
 }
+async function requestExternalToolQueryFix(failedSql, errorMessage, component, executedTools, toolId, apiKey) {
+  const executedTool = executedTools.find((t) => t.id === toolId);
+  const sourceSchema = executedTool?.sourceSchema || "Schema not available";
+  const sourceType = executedTool?.sourceType || "postgresql";
+  const databaseRules = await promptLoader.loadDatabaseRulesForType(sourceType);
+  const prompt = `You are a SQL expert. Fix the following SQL query that failed execution.
 
-// src/userResponse/utils/component-props-processor.ts
-function validateExternalTool(externalTool, executedTools, providerName) {
-  if (!externalTool) {
-    return { valid: true };
-  }
-  const toolId = externalTool.toolId;
-  const validToolIds = (executedTools || []).map((t) => t.id);
-  const isValidTool = toolId && typeof toolId === "string" && validToolIds.includes(toolId);
-  if (!isValidTool) {
-    logger.warn(`[${providerName}] externalTool.toolId "${toolId}" not found in executed tools [${validToolIds.join(", ")}], setting to null`);
-    return { valid: false };
-  }
-  const executedTool = executedTools?.find((t) => t.id === toolId);
-  return { valid: true, executedTool };
+## Database Schema
+${sourceSchema}
+
+## Database-Specific SQL Rules
+${databaseRules}
+
+## Component Context
+- Component Name: ${component.name}
+- Component Type: ${component.type}
+- Title: ${component.props?.title || "N/A"}
+
+## Failed Query
+\`\`\`sql
+${failedSql}
+\`\`\`
+
+## Error Message
+${errorMessage}
+
+## Instructions
+1. Analyze the error message and identify what caused the query to fail
+2. Fix the query to resolve the error while preserving the original intent
+3. Ensure the fixed query follows the database-specific SQL rules above
+4. Return ONLY the fixed SQL query, no explanations or markdown
+
+Fixed SQL query:`;
+  const response = await LLM.text(
+    {
+      sys: "You are a SQL expert. Return only the fixed SQL query with no additional text, explanations, or markdown formatting.",
+      user: prompt
+    },
+    {
+      model: "anthropic/claude-haiku-4-5-20251001",
+      maxTokens: 2048,
+      temperature: 0,
+      apiKey
+    }
+  );
+  let fixedQuery = response.trim();
+  fixedQuery = fixedQuery.replace(/^```sql\s*/i, "").replace(/\s*```$/i, "");
+  fixedQuery = fixedQuery.replace(/^```\s*/i, "").replace(/\s*```$/i, "");
+  const { query: validatedQuery } = validateAndFixSqlQuery(fixedQuery);
+  return validatedQuery;
 }
-function validateAndCleanQuery(query, config) {
-  if (!query) {
-    return { query: null, wasModified: false };
-  }
-  let wasModified = false;
-  let cleanedQuery = query;
-  const queryStr = typeof query === "string" ? query : query?.sql || "";
-  if (queryStr.includes("OPENJSON") || queryStr.includes("JSON_VALUE")) {
-    logger.warn(`[${config.providerName}] Query contains OPENJSON/JSON_VALUE (invalid - cannot parse tool result), setting query to null`);
-    return { query: null, wasModified: true };
+
+// src/userResponse/anthropic.ts
+import dotenv from "dotenv";
+
+// src/userResponse/schema.ts
+import path5 from "path";
+import fs6 from "fs";
+var Schema = class {
+  constructor(schemaFilePath) {
+    this.cachedSchema = null;
+    this.schemaFilePath = schemaFilePath || path5.join(process.cwd(), "../analysis/data/schema.json");
   }
-  const { query: fixedQuery, fixed, fixes } = validateAndFixSqlQuery(queryStr);
-  if (fixed) {
-    logger.warn(`[${config.providerName}] SQL fixes applied to component query: ${fixes.join("; ")}`);
-    wasModified = true;
-    if (typeof cleanedQuery === "string") {
-      cleanedQuery = fixedQuery;
-    } else if (cleanedQuery?.sql) {
-      cleanedQuery = { ...cleanedQuery, sql: fixedQuery };
+  /**
+   * Gets the database schema from the schema file
+   * @returns Parsed schema object or null if error occurs
+   */
+  getDatabaseSchema() {
+    try {
+      const dir = path5.dirname(this.schemaFilePath);
+      if (!fs6.existsSync(dir)) {
+        logger.info(`Creating directory structure: ${dir}`);
+        fs6.mkdirSync(dir, { recursive: true });
+      }
+      if (!fs6.existsSync(this.schemaFilePath)) {
+        logger.info(`Schema file does not exist at ${this.schemaFilePath}, creating with empty schema`);
+        const initialSchema = {
+          database: "",
+          schema: "",
+          description: "",
+          tables: [],
+          relationships: []
+        };
+        fs6.writeFileSync(this.schemaFilePath, JSON.stringify(initialSchema, null, 4));
+        this.cachedSchema = initialSchema;
+        return initialSchema;
+      }
+      const fileContent = fs6.readFileSync(this.schemaFilePath, "utf-8");
+      const schema2 = JSON.parse(fileContent);
+      this.cachedSchema = schema2;
+      return schema2;
+    } catch (error) {
+      logger.error("Error parsing schema file:", error);
+      return null;
     }
   }
-  if (typeof cleanedQuery === "string") {
-    const limitedQuery = ensureQueryLimit(cleanedQuery, config.defaultLimit, MAX_COMPONENT_QUERY_LIMIT);
-    if (limitedQuery !== cleanedQuery) wasModified = true;
-    cleanedQuery = limitedQuery;
-  } else if (cleanedQuery?.sql) {
-    const limitedSql = ensureQueryLimit(cleanedQuery.sql, config.defaultLimit, MAX_COMPONENT_QUERY_LIMIT);
-    if (limitedSql !== cleanedQuery.sql) wasModified = true;
-    cleanedQuery = { ...cleanedQuery, sql: limitedSql };
+  /**
+   * Gets the cached schema or loads it if not cached
+   * @returns Cached schema or freshly loaded schema
+   */
+  getSchema() {
+    if (this.cachedSchema) {
+      return this.cachedSchema;
+    }
+    return this.getDatabaseSchema();
   }
-  return { query: cleanedQuery, wasModified };
-}
-function processComponentProps(props, executedTools, config) {
-  let cleanedProps = { ...props };
-  if (cleanedProps.externalTool) {
-    const { valid } = validateExternalTool(
-      cleanedProps.externalTool,
-      executedTools,
-      config.providerName
-    );
-    if (!valid) {
-      cleanedProps.externalTool = null;
+  /**
+   * Generates database schema documentation for LLM from Snowflake JSON schema
+   * @returns Formatted schema documentation string
+   */
+  generateSchemaDocumentation() {
+    const schema2 = this.getSchema();
+    if (!schema2) {
+      logger.warn("No database schema found.");
+      return "No database schema available.";
+    }
+    const tables = [];
+    tables.push(`Database: ${schema2.database}`);
+    tables.push(`Schema: ${schema2.schema}`);
+    tables.push(`Description: ${schema2.description}`);
+    tables.push("");
+    tables.push("=".repeat(80));
+    tables.push("");
+    for (const table of schema2.tables) {
+      const tableInfo = [];
+      tableInfo.push(`TABLE: ${table.fullName}`);
+      tableInfo.push(`Description: ${table.description}`);
+      tableInfo.push(`Row Count: ~${table.rowCount.toLocaleString()}`);
+      tableInfo.push("");
+      tableInfo.push("Columns:");
+      for (const column of table.columns) {
+        let columnLine = `  - ${column.name}: ${column.type}`;
+        if (column.isPrimaryKey) {
+          columnLine += " (PRIMARY KEY)";
+        }
+        if (column.isForeignKey && column.references) {
+          columnLine += ` (FK -> ${column.references.table}.${column.references.column})`;
+        }
+        if (!column.nullable) {
+          columnLine += " NOT NULL";
+        }
+        if (column.description) {
+          columnLine += ` - ${column.description}`;
+        }
+        tableInfo.push(columnLine);
+        if (column.sampleValues && column.sampleValues.length > 0) {
+          tableInfo.push(`    Sample values: [${column.sampleValues.join(", ")}]`);
+        }
+        if (column.statistics) {
+          const stats = column.statistics;
+          if (stats.min !== void 0 && stats.max !== void 0) {
+            tableInfo.push(`    Range: ${stats.min} to ${stats.max}`);
+          }
+          if (stats.distinct !== void 0) {
+            tableInfo.push(`    Distinct values: ${stats.distinct.toLocaleString()}`);
+          }
+        }
+      }
+      tableInfo.push("");
+      tables.push(tableInfo.join("\n"));
     }
+    tables.push("=".repeat(80));
+    tables.push("");
+    tables.push("TABLE RELATIONSHIPS:");
+    tables.push("");
+    for (const rel of schema2.relationships) {
+      tables.push(`${rel.from} -> ${rel.to} (${rel.type}): ${rel.keys.join(" = ")}`);
+    }
+    return tables.join("\n");
   }
-  if (cleanedProps.query) {
-    const { query } = validateAndCleanQuery(cleanedProps.query, config);
-    cleanedProps.query = query;
+  /**
+   * Clears the cached schema, forcing a reload on next access
+   */
+  clearCache() {
+    this.cachedSchema = null;
   }
-  if (cleanedProps.query && cleanedProps.externalTool) {
-    logger.info(`[${config.providerName}] Both query and externalTool exist, keeping both - frontend will decide`);
+  /**
+   * Sets a custom schema file path
+   * @param filePath - Path to the schema file
+   */
+  setSchemaPath(filePath) {
+    this.schemaFilePath = filePath;
+    this.clearCache();
   }
-  return cleanedProps;
-}
+};
+var schema = new Schema();
 
 // src/userResponse/services/query-execution-service.ts
 var QueryExecutionService = class {
@@ -9329,7 +9878,7 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
     let layoutDescription = "Multi-component dashboard";
     let actions = [];
     if (!hasExecutedTools) {
-      logger.info(`[AgentFlow] No tools executed \u2014 general question, skipping component generation`);
+      logger.info(`[AgentFlow] No tools executed \u2014 general question, wrapping in DynamicMarkdownBlock`);
       const nextQuestions = await llmInstance.generateNextQuestions(
         prompt,
         null,
@@ -9339,6 +9888,18 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
         textResponse
       );
       actions = convertQuestionsToActions(nextQuestions);
+      const markdownContent = textResponse.replace(/<DashboardComponents>[\s\S]*?<\/DashboardComponents>/g, "").trim();
+      matchedComponents = [{
+        id: "dynamic-markdown-block",
+        name: "DynamicMarkdownBlock",
+        type: "MarkdownBlock",
+        description: "Text response rendered as markdown",
+        props: {
+          content: markdownContent
+        }
+      }];
+      layoutTitle = "";
+      layoutDescription = "";
     } else if (components && components.length > 0) {
       logger.info(`[AgentFlow] ${agentResponse.executedTools.length} tools executed \u2014 generating components`);
       if (streamBuffer.hasCallback()) {
@@ -9353,18 +9914,16 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
         /<DataTable>[\s\S]*?<\/DataTable>/g,
         "<DataTable>[Data preview removed - for table components, REUSE the exact SQL query shown above (the one that returned these results). Do NOT write a new query or embed data in props.]</DataTable>"
       );
-      const matchResult = await llmInstance.matchComponentsFromAnalysis(
-        sanitizedTextResponse,
+      const matchResult = await generateAgentComponents({
+        analysisContent: sanitizedTextResponse,
         components,
-        prompt,
+        userPrompt: prompt,
+        executedTools: agentResponse.executedTools,
+        collections,
         apiKey,
         componentStreamCallback,
-        [],
-        // deferredTools — MainAgent handles tool execution
-        agentResponse.executedTools,
-        collections,
         userId
-      );
+      });
       matchedComponents = matchResult.components;
       layoutTitle = matchResult.layoutTitle;
       layoutDescription = matchResult.layoutDescription;
@@ -9372,9 +9931,11 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
     }
     const securedComponents = matchedComponents.map((comp) => {
       const props = { ...comp.props };
-      if (props.externalTool?.parameters?.sql) {
-        const { sql, ...restParams } = props.externalTool.parameters;
-        const queryId = queryCache.storeQuery(sql);
+      const sqlValue = props.externalTool?.parameters?.sql || props.externalTool?.parameters?.query;
+      if (sqlValue) {
+        const { sql, query, ...restParams } = props.externalTool.parameters;
+        const cachedData = queryCache.get(sqlValue);
+        const queryId = queryCache.storeQuery(sqlValue, cachedData);
         props.externalTool = {
           ...props.externalTool,
           parameters: { queryId, ...restParams }
@@ -9493,18 +10054,10 @@ async function saveConversation(params) {
   }
   try {
     logger.info(`[CONVERSATION_SAVER] Saving conversation for userId: ${userId}, uiBlockId: ${uiBlockId}, threadId: ${threadId}`);
-    const userIdNumber = Number(userId);
-    if (isNaN(userIdNumber)) {
-      logger.warn(`[CONVERSATION_SAVER] Invalid userId: ${userId} (not a valid number)`);
-      return {
-        success: false,
-        error: `Invalid userId: ${userId} (not a valid number)`
-      };
-    }
     const dbUIBlock = transformUIBlockForDB(uiblock, userPrompt, uiBlockId);
     logger.debug(`[CONVERSATION_SAVER] Transformed UIBlock for DB: ${JSON.stringify(dbUIBlock)}`);
     const saveResult = await collections["user-conversations"]["create"]({
-      userId: userIdNumber,
+      userId,
       userPrompt,
       uiblock: dbUIBlock,
       threadId
@@ -9525,7 +10078,7 @@ async function saveConversation(params) {
           userPrompt,
           uiBlock: dbUIBlock,
           // Use the transformed UIBlock
-          userId: userIdNumber
+          userId
         });
         if (embedResult?.success) {
           logger.info("[CONVERSATION_SAVER] Successfully created embedding");