@superatomai/sdk-node 0.0.1-s → 0.0.2-dsp

package/dist/index.mjs

@@ -326,6 +326,24 @@ var ComponentListResponseMessageSchema = z3.object({
   type: z3.literal("COMPONENT_LIST_RES"),
   payload: ComponentListResponsePayloadSchema
 });
+var WorkflowDescriptorSchema = z3.object({
+  id: z3.string(),
+  name: z3.string(),
+  description: z3.string(),
+  whenToUse: z3.string(),
+  propsSchema: z3.record(z3.string()),
+  defaultProps: z3.record(z3.unknown()).optional()
+});
+var WorkflowsSchema = z3.array(WorkflowDescriptorSchema);
+var WorkflowListResponsePayloadSchema = z3.object({
+  workflows: z3.array(WorkflowDescriptorSchema)
+});
+var WorkflowListResponseMessageSchema = z3.object({
+  id: z3.string(),
+  from: MessageParticipantSchema,
+  type: z3.literal("WORKFLOW_LIST_RES"),
+  payload: WorkflowListResponsePayloadSchema
+});
 var OutputFieldSchema = z3.object({
   name: z3.string(),
   // Field name (column name in the result)
@@ -349,8 +367,10 @@ var ToolSchema = z3.object({
   fullSchema: z3.string().optional(),
   params: z3.record(z3.string()),
   fn: z3.function().args(z3.any()).returns(z3.any()),
-  outputSchema: OutputSchema.optional()
+  outputSchema: OutputSchema.optional(),
   // Optional: describes the data structure returned by this tool
+  /** Cache policy. `false` = never cache (live data, write ops). Mirrors HTTP `Cache-Control: no-store`. */
+  cache: z3.union([z3.literal(false), z3.object({ ttlMs: z3.number().optional() })]).optional()
 });
 var UserQueryFiltersSchema = z3.object({
   username: z3.string().optional(),
@@ -1465,7 +1485,7 @@ var QueryCache = class {
     this.cache = /* @__PURE__ */ new Map();
     this.ttlMs = 10 * 60 * 1e3;
     // Default: 10 minutes
-    this.maxCacheSize = 500;
+    this.maxCacheSize = 5e3;
     // Max data cache entries
     this.cleanupInterval = null;
     // Encryption for queryId tokens
@@ -1492,9 +1512,13 @@ var QueryCache = class {
     return this.ttlMs / 60 / 1e3;
   }
   /**
-   * Store query result in data cache
+   * Store query result in data cache.
+   * If the key already exists, it's removed first so the re-insert places it
+   * at the back of the iteration order (LRU). Eviction only fires when adding
+   * a genuinely new key past the size limit.
    */
   set(query, data) {
+    this.cache.delete(query);
     if (this.cache.size >= this.maxCacheSize) {
       const oldestKey = this.cache.keys().next().value;
       if (oldestKey) this.cache.delete(oldestKey);
@@ -1507,7 +1531,9 @@ var QueryCache = class {
     logger.debug(`[QueryCache] Stored result for query (${query.substring(0, 50)}...)`);
   }
   /**
-   * Get cached result if exists and not expired
+   * Get cached result if exists and not expired.
+   * On hit, re-inserts the entry so it moves to the back of the Map's
+   * iteration order — turning FIFO eviction into true LRU.
    */
   get(query) {
     const entry = this.cache.get(query);
@@ -1516,6 +1542,8 @@ var QueryCache = class {
       this.cache.delete(query);
       return null;
     }
+    this.cache.delete(query);
+    this.cache.set(query, entry);
     logger.info(`[QueryCache] Cache HIT for query (${query.substring(0, 50)}...)`);
     return entry.data;
   }
@@ -1657,6 +1685,21 @@ var QueryCache = class {
 };
 var queryCache = new QueryCache();
 
+// src/utils/surrogate.ts
+var LONE_SURROGATE_RE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
+function stripLoneSurrogates(value) {
+  if (typeof value !== "string") return value;
+  if (!/[\uD800-\uDFFF]/.test(value)) return value;
+  return value.replace(LONE_SURROGATE_RE, "\uFFFD");
+}
+function safeTruncate(text, maxUnits) {
+  if (typeof text !== "string" || text.length <= maxUnits || maxUnits < 0) return text;
+  let end = maxUnits;
+  const lastCode = text.charCodeAt(end - 1);
+  if (lastCode >= 55296 && lastCode <= 56319) end -= 1;
+  return text.slice(0, end);
+}
+
 // src/userResponse/llm-result-truncator.ts
 var DEFAULT_MAX_ROWS = 10;
 var DEFAULT_MAX_CHARS_PER_FIELD = 500;
@@ -1699,12 +1742,12 @@ function isDateString(value) {
 }
 function truncateTextField(value, maxLength) {
   if (value.length <= maxLength) {
-    return { text: value, wasTruncated: false };
+    return { text: stripLoneSurrogates(value), wasTruncated: false };
   }
-  const truncated = value.substring(0, maxLength);
-  const remaining = value.length - maxLength;
+  const truncated = safeTruncate(value, maxLength);
+  const remaining = value.length - truncated.length;
   return {
-    text: `${truncated}... (${remaining} more chars)`,
+    text: `${stripLoneSurrogates(truncated)}... (${remaining} more chars)`,
     wasTruncated: true
   };
 }
@@ -2002,6 +2045,21 @@ function formatResultAsString(formattedResult) {
   return JSON.stringify(formattedResult, null, 2);
 }
 
+// src/utils/cache-key.ts
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") {
+    return JSON.stringify(value);
+  }
+  if (Array.isArray(value)) {
+    return "[" + value.map(stableStringify).join(",") + "]";
+  }
+  const keys = Object.keys(value).sort();
+  return "{" + keys.map((k) => JSON.stringify(k) + ":" + stableStringify(value[k])).join(",") + "}";
+}
+function buildDirectToolCacheKey(toolId, parameters) {
+  return `et-direct:${toolId}:${stableStringify(parameters || {})}`;
+}
+
 // src/handlers/data-request.ts
 function getQueryCacheKey(query) {
   if (typeof query === "string") {
@@ -2015,7 +2073,7 @@ function getQueryCacheKey(query) {
   }
   return "";
 }
-function getCacheKey(collection, op, params) {
+function getCacheKey(collection, op, params, tools) {
   if (collection === "database" && op === "execute" && params?.sql) {
     return getQueryCacheKey(params.sql);
   }
@@ -2025,6 +2083,13 @@ function getCacheKey(collection, op, params) {
     const paramsKey = params.params ? JSON.stringify(params.params) : "";
     return sqlKey ? `et:${toolId}:${sqlKey}:${paramsKey}` : "";
   }
+  if (collection === "external-tools" && op === "execute" && params?.toolId && !params?.sql) {
+    const tool = tools?.find((t) => t.id === params.toolId);
+    if (tool?.cache === false) return "";
+    const { toolId, toolName, ...rest } = params;
+    const paramsKey = stableStringify(rest);
+    return `et-direct:${toolId}:${paramsKey}`;
+  }
   if (collection === "external-tools" && op === "executeByQueryId" && params?.queryId) {
     const toolId = params.toolId || "";
     const filterKey = params.filterParams ? JSON.stringify(params.filterParams) : "";
@@ -2033,7 +2098,7 @@ function getCacheKey(collection, op, params) {
   }
   return "";
 }
-async function handleDataRequest(data, collections, sendMessage) {
+async function handleDataRequest(data, collections, sendMessage, tools) {
   let requestId;
   let collection;
   let op;
@@ -2059,7 +2124,7 @@ async function handleDataRequest(data, collections, sendMessage) {
     const startTime = performance.now();
     let result;
     let fromCache = false;
-    const cacheKey = getCacheKey(collection, op, params);
+    const cacheKey = getCacheKey(collection, op, params, tools);
     if (cacheKey) {
       const cachedResult = queryCache.get(cacheKey);
       if (cachedResult !== null) {
@@ -3724,6 +3789,12 @@ You MUST respond with ONLY a valid JSON object (no markdown, no code blocks):
 ---
 
 ## CONTEXT
+
+### Global Knowledge Base
+{{GLOBAL_KNOWLEDGE_BASE}}
+
+### Knowledge Base Context
+{{KNOWLEDGE_BASE_CONTEXT}}
 `,
     user: `{{USER_PROMPT}}`
   },
@@ -4276,13 +4347,11 @@ var PromptLoader = class {
    * @returns Processed string
    */
   replaceVariables(template, variables) {
-    let content = template;
-    for (const [key, value] of Object.entries(variables)) {
-      const pattern = new RegExp(`{{${key}}}`, "g");
-      const replacementValue = typeof value === "string" ? value : JSON.stringify(value);
-      content = content.replace(pattern, replacementValue);
-    }
-    return content;
+    return template.replace(/\{\{(\w+)\}\}/g, (match, key) => {
+      if (!(key in variables)) return match;
+      const value = variables[key];
+      return typeof value === "string" ? value : JSON.stringify(value);
+    });
   }
   /**
    * Load both system and user prompts from cache and replace variables
@@ -4305,8 +4374,6 @@ var PromptLoader = class {
       const [staticPart, contextPart] = template.system.split(contextMarker);
       const processedStatic = this.replaceVariables(staticPart, variables);
       const processedContext = this.replaceVariables(contextMarker + contextPart, variables);
-      const staticLength = processedStatic.length;
-      const contextLength = processedContext.length;
       return {
         system: [
           {
@@ -5236,9 +5303,36 @@ var searchConversationsWithReranking = async (options) => {
     return null;
   }
 };
+var findExactMatch = async ({
+  userPrompt,
+  collections,
+  userId
+}) => {
+  try {
+    if (!collections || !collections["conversation-history"] || !collections["conversation-history"]["exactMatch"]) {
+      logger.info("[ConversationSearch] conversation-history.exactMatch collection not registered, skipping");
+      return null;
+    }
+    const result = await collections["conversation-history"]["exactMatch"]({
+      userPrompt,
+      userId
+    });
+    if (!result || !result.uiBlock) {
+      logger.info("[ConversationSearch] No exact match found");
+      return null;
+    }
+    logger.info(`[ConversationSearch] \u2713 Exact prompt match found for "${userPrompt.substring(0, 50)}..."`);
+    return result;
+  } catch (error) {
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    logger.warn(`[ConversationSearch] Error in exact match lookup: ${errorMsg}`);
+    return null;
+  }
+};
 var ConversationSearch = {
   searchConversations,
-  searchConversationsWithReranking
+  searchConversationsWithReranking,
+  findExactMatch
 };
 var conversation_search_default = ConversationSearch;
 
@@ -5256,16 +5350,20 @@ var MAX_TOKENS_CLASSIFICATION = 1500;
 var MAX_TOKENS_ADAPTATION = 8192;
 var MAX_TOKENS_TEXT_RESPONSE = 4e3;
 var MAX_TOKENS_NEXT_QUESTIONS = 1200;
-var DEFAULT_MAX_ROWS_FOR_LLM = 10;
+var MAX_ROWS_FETCHED = 50;
+var MAX_ROWS_TO_LLM = 10;
+var LLM_SAMPLE_ROWS = 10;
+var MAX_ROWS_RENDERED = 250;
 var DEFAULT_MAX_CHARS_PER_FIELD2 = 500;
-var STREAM_PREVIEW_MAX_ROWS = 10;
 var STREAM_PREVIEW_MAX_CHARS = 200;
-var TOOL_TRACKING_MAX_ROWS = 5;
 var TOOL_TRACKING_MAX_CHARS = 200;
-var TOOL_TRACKING_SAMPLE_ROWS = 3;
+var DEFAULT_MAX_ROWS_FOR_LLM = LLM_SAMPLE_ROWS;
+var STREAM_PREVIEW_MAX_ROWS = LLM_SAMPLE_ROWS;
+var TOOL_TRACKING_SAMPLE_ROWS = LLM_SAMPLE_ROWS;
+var TOOL_TRACKING_MAX_ROWS = 5;
 var DEFAULT_QUERY_LIMIT = 24;
-var MAX_AGENT_QUERY_LIMIT = 10;
-var MAX_COMPONENT_QUERY_LIMIT = 100;
+var MAX_AGENT_QUERY_LIMIT = LLM_SAMPLE_ROWS;
+var MAX_COMPONENT_QUERY_LIMIT = MAX_ROWS_RENDERED;
 var EXACT_MATCH_SIMILARITY_THRESHOLD = 0.99;
 var DEFAULT_CONVERSATION_SIMILARITY_THRESHOLD = 0.8;
 var MAX_TOOL_CALLING_ITERATIONS = 20;
@@ -5762,6 +5860,7 @@ var LLM = class {
   /* Get a complete text response from an LLM (Anthropic or Groq) */
   static async text(messages, options = {}) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicText(messages, modelName, options);
     } else if (provider === "groq") {
@@ -5777,6 +5876,7 @@ var LLM = class {
   /* Stream response from an LLM (Anthropic or Groq) */
   static async stream(messages, options = {}, json) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicStream(messages, modelName, options, json);
     } else if (provider === "groq") {
@@ -5792,6 +5892,7 @@ var LLM = class {
   /* Stream response with tool calling support (Anthropic and Gemini) */
   static async streamWithTools(messages, tools, toolHandler, options = {}, maxIterations = 3) {
     const [provider, modelName] = this._parseModel(options.model);
+    messages = this._sanitizeMessages(messages);
     if (provider === "anthropic") {
       return this._anthropicStreamWithTools(messages, tools, toolHandler, modelName, options, maxIterations);
     } else if (provider === "gemini") {
@@ -5817,6 +5918,26 @@ var LLM = class {
     }
     return sys;
   }
+  /**
+   * Strip unpaired UTF-16 surrogates from every text field of a message set.
+   *
+   * A lone surrogate (from mid-pair string slicing or corrupt source data)
+   * serializes to a bare `\udXXX` escape that strict JSON parsers — including
+   * the one on Anthropic's API — reject with "no low surrogate in string",
+   * failing the whole request. Sanitizing here, at the single boundary every
+   * provider call flows through, guarantees no request can carry one.
+   */
+  static _sanitizeMessages(messages) {
+    const sys = typeof messages.sys === "string" ? stripLoneSurrogates(messages.sys) : messages.sys.map(
+      (block) => block?.type === "text" && typeof block.text === "string" ? { ...block, text: stripLoneSurrogates(block.text) } : block
+    );
+    return {
+      ...messages,
+      sys,
+      user: stripLoneSurrogates(messages.user),
+      ...messages.prefill !== void 0 && { prefill: stripLoneSurrogates(messages.prefill) }
+    };
+  }
   /**
    * Log cache usage metrics from Anthropic API response
    * Shows cache hits, costs, and savings
@@ -6196,12 +6317,14 @@ var LLM = class {
             let resultContent = typeof result === "string" ? result : JSON.stringify(result);
             const MAX_RESULT_LENGTH = 5e4;
             if (resultContent.length > MAX_RESULT_LENGTH) {
-              resultContent = resultContent.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
+              resultContent = safeTruncate(resultContent, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
             }
             return {
               type: "tool_result",
               tool_use_id: toolUse.id,
-              content: resultContent
+              // Final safety net: tool results carry source data and are built
+              // mid-loop (after entry-point sanitize), so strip lone surrogates here.
+              content: stripLoneSurrogates(resultContent)
             };
           } catch (error) {
             return {
@@ -6716,11 +6839,12 @@ var LLM = class {
           let resultContent = typeof result2 === "string" ? result2 : JSON.stringify(result2);
           const MAX_RESULT_LENGTH = 5e4;
           if (resultContent.length > MAX_RESULT_LENGTH) {
-            resultContent = resultContent.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
+            resultContent = safeTruncate(resultContent, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + resultContent.length + " total]";
           }
           return {
             name: fc.name,
-            response: { result: resultContent }
+            // Final safety net: strip lone surrogates from source-data results.
+            response: { result: stripLoneSurrogates(resultContent) }
           };
         } catch (error) {
           return {
@@ -6993,12 +7117,12 @@ var LLM = class {
           result = typeof toolResult === "string" ? toolResult : JSON.stringify(toolResult);
           const MAX_RESULT_LENGTH = 5e4;
           if (result.length > MAX_RESULT_LENGTH) {
-            result = result.substring(0, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + result.length + " total]";
+            result = safeTruncate(result, MAX_RESULT_LENGTH) + "\n\n... [Result truncated - showing first 50000 characters of " + result.length + " total]";
           }
         } catch (error) {
           result = JSON.stringify({ error: error instanceof Error ? error.message : String(error) });
         }
-        return { role: "tool", tool_call_id: tc.id, content: result };
+        return { role: "tool", tool_call_id: tc.id, content: stripLoneSurrogates(result) };
       }));
       toolCallResults.forEach((r) => conversationMessages.push(r));
     }
@@ -7137,6 +7261,130 @@ function extractObjectText(obj) {
   return JSON.stringify(obj, null, 2);
 }
 
+// src/userResponse/agents/data-summary.ts
+var MAIN_AGENT_COMPLETE_ROWS = MAX_ROWS_TO_LLM;
+function summarizeRows(rows) {
+  if (!rows.length) return { totalRows: 0, columns: {} };
+  const MAX_CATEGORIES = 20;
+  const keys = Object.keys(rows[0]);
+  const columns = {};
+  const fmtDate = (ms) => {
+    const d = new Date(ms);
+    const p = (n) => String(n).padStart(2, "0");
+    return `${d.getFullYear()}-${p(d.getMonth() + 1)}-${p(d.getDate())}`;
+  };
+  for (const key of keys) {
+    let nonNull = 0, zeros = 0, isNumeric = true, isDateLike = true, sum = 0;
+    let min, max;
+    let dMin, dMax;
+    const counts = /* @__PURE__ */ new Map();
+    let overflow = false;
+    for (const row of rows) {
+      const v = row[key];
+      if (v === null || v === void 0 || v === "") continue;
+      nonNull++;
+      const num = typeof v === "number" ? v : typeof v === "string" && v.trim() !== "" && !isNaN(Number(v)) ? Number(v) : NaN;
+      if (!Number.isNaN(num) && typeof v !== "boolean") {
+        if (num === 0) zeros++;
+        min = min === void 0 ? num : Math.min(min, num);
+        max = max === void 0 ? num : Math.max(max, num);
+        sum += num;
+      } else {
+        isNumeric = false;
+      }
+      const t = v instanceof Date ? v.getTime() : typeof v === "string" ? Date.parse(v) : NaN;
+      if (!Number.isNaN(t) && typeof v !== "number") {
+        dMin = dMin === void 0 ? t : Math.min(dMin, t);
+        dMax = dMax === void 0 ? t : Math.max(dMax, t);
+      } else {
+        isDateLike = false;
+      }
+      if (!overflow) {
+        const sv = String(v);
+        counts.set(sv, (counts.get(sv) || 0) + 1);
+        if (counts.size > MAX_CATEGORIES * 5) overflow = true;
+      }
+    }
+    const nulls = rows.length - nonNull;
+    if (nonNull === 0) {
+      columns[key] = { type: "empty", nulls };
+      continue;
+    }
+    if (isNumeric) {
+      columns[key] = {
+        type: "number",
+        nonNull,
+        nulls,
+        zeros,
+        min,
+        max,
+        avg: Math.round(sum / nonNull * 100) / 100
+      };
+    } else if (isDateLike) {
+      columns[key] = {
+        type: "date",
+        nonNull,
+        min: fmtDate(dMin),
+        max: fmtDate(dMax),
+        distinct: overflow ? `${MAX_CATEGORIES * 5}+` : counts.size
+      };
+    } else if (!overflow && counts.size <= MAX_CATEGORIES) {
+      const dist = {};
+      for (const [k, c] of [...counts.entries()].sort((a, b) => b[1] - a[1])) dist[k] = c;
+      columns[key] = { type: "category", distinct: counts.size, counts: dist };
+    } else {
+      columns[key] = {
+        type: "text",
+        distinct: overflow ? `${MAX_CATEGORIES * 5}+` : counts.size,
+        examples: [...counts.keys()].slice(0, 3)
+      };
+    }
+  }
+  const dims = keys.filter((k) => columns[k]?.type === "category" && columns[k].distinct <= 6).sort((a, b) => columns[a].distinct - columns[b].distinct);
+  const measures = keys.filter((k) => columns[k]?.type === "number");
+  let groups;
+  if (dims.length && measures.length) {
+    const MAX_GROUPS = 30;
+    let useDims = dims.slice(0, 2);
+    const keyOf = (row) => useDims.map((d) => `${d}=${row[d]}`).join(" | ");
+    if (new Set(rows.map(keyOf)).size > MAX_GROUPS) useDims = dims.slice(0, 1);
+    const agg = /* @__PURE__ */ new Map();
+    for (const row of rows) {
+      const key = useDims.map((d) => `${d}=${row[d]}`).join(" | ");
+      let g = agg.get(key);
+      if (!g) {
+        g = { count: 0, sums: {}, ns: {} };
+        agg.set(key, g);
+      }
+      g.count++;
+      for (const m of measures) {
+        const v = row[m];
+        if (typeof v === "number" && !Number.isNaN(v)) {
+          g.sums[m] = (g.sums[m] || 0) + v;
+          g.ns[m] = (g.ns[m] || 0) + 1;
+        }
+      }
+    }
+    if (agg.size <= MAX_GROUPS) {
+      const isCountLike = (name) => /count|\bobs\b|observation|qty|quantity|\bnum|total|evidential|volume|units/i.test(name);
+      groups = {
+        by: useDims,
+        rows: [...agg.entries()].map(([key, g]) => {
+          const avg = {};
+          const sum = {};
+          for (const m of measures) {
+            if (!g.ns[m]) continue;
+            avg[m] = Math.round(g.sums[m] / g.ns[m] * 100) / 100;
+            if (isCountLike(m)) sum[m] = Math.round(g.sums[m] * 100) / 100;
+          }
+          return Object.keys(sum).length ? { key, count: g.count, avg, sum } : { key, count: g.count, avg };
+        })
+      };
+    }
+  }
+  return groups ? { totalRows: rows.length, columns, groups } : { totalRows: rows.length, columns };
+}
+
 // src/userResponse/agents/agent-prompt-builder.ts
 function buildSourceSummaries(externalTools) {
   return externalTools.map((tool) => {
@@ -7356,8 +7604,15 @@ ${result}`;
           await streamDelay();
         }
         const cappedInput = { ...toolInput };
-        if (cappedInput.limit === void 0 || cappedInput.limit > this.config.maxRowsPerSource) {
-          cappedInput.limit = this.config.maxRowsPerSource;
+        if (cappedInput.limit === void 0 || cappedInput.limit > this.config.maxRowsFetched) {
+          cappedInput.limit = this.config.maxRowsFetched;
+        }
+        const _st = this.extractSourceType();
+        if (typeof cappedInput.sql === "string") {
+          cappedInput.sql = ensureQueryLimit(cappedInput.sql, this.config.maxRowsFetched, this.config.maxRowsFetched, _st);
+        }
+        if (typeof cappedInput.query === "string") {
+          cappedInput.query = ensureQueryLimit(cappedInput.query, this.config.maxRowsFetched, this.config.maxRowsFetched, _st);
         }
         queryExecuted = cappedInput.sql || cappedInput.query || JSON.stringify(cappedInput);
         if (this.streamBuffer.hasCallback() && queryExecuted) {
@@ -7408,8 +7663,11 @@ Analyze the error and try again with a corrected query.`;
 `);
             if (resultData.length > 0) {
               try {
-                const previewData = resultData.slice(0, this.config.maxRowsPerSource);
-                this.streamBuffer.write(`<DataTable>${JSON.stringify(previewData)}</DataTable>
+                const preview = formatQueryResultForLLM(resultData, {
+                  maxRows: STREAM_PREVIEW_MAX_ROWS,
+                  maxCharsPerField: STREAM_PREVIEW_MAX_CHARS
+                });
+                this.streamBuffer.write(`<DataTable>${JSON.stringify(preview.data)}</DataTable>
 
 `);
               } catch {
@@ -7432,7 +7690,12 @@ Analyze the error and try again with a corrected query.`;
               _totalRecords: totalRowsMatched,
               _recordsShown: resultData.length,
               _metadata: result.metadata,
-              _sampleData: resultData.slice(0, 3)
+              _sampleData: resultData.slice(0, TOOL_TRACKING_SAMPLE_ROWS),
+              // For the main agent: a bounded summary over the FULL fetched
+              // result (complete structure regardless of size) + a complete
+              // slice for small results (so lookups arrive whole, not biased).
+              _summary: summarizeRows(resultData),
+              _mainAgentRows: resultData.slice(0, MAIN_AGENT_COMPLETE_ROWS)
             },
             outputSchema: this.tool.outputSchema,
             sourceSchema: this.tool.description,
@@ -7440,7 +7703,7 @@ Analyze the error and try again with a corrected query.`;
           };
           allExecutedTools.push(executedTool);
           const formatted = typeof formattedResult === "string" ? formattedResult : JSON.stringify(formattedResult);
-          const followUpNote = successfulQueries < 2 ? "You may make ONE follow-up query if this data is incomplete for the intent. Otherwise, STOP." : "Maximum queries reached. Use the data you have.";
+          const followUpNote = successfulQueries < 2 ? "STOP now and return this data \u2014 it satisfies the intent in the common case. Make a second query ONLY if a column or row the intent explicitly requires is verifiably MISSING from these results (not merely to widen, re-sort, or double-check). A redundant follow-up wastes a full round-trip." : "Maximum queries reached. Use the data you have.";
           return `\u2705 Query executed successfully. ${resultData.length} rows returned (${totalRowsMatched} total matched). ${followUpNote}
 
 ${formatted}`;
@@ -7502,7 +7765,13 @@ Analyze the error and try again with a corrected query.`;
         sourceId: this.tool.id,
         sourceName: this.tool.name,
         success: true,
-        data: resultData,
+        // Don't retain the full fetched result — only the bounded slice the
+        // main agent can actually use. The complete-structure summary and the
+        // complete-small rows already live on each executedTool (`_summary` /
+        // `_mainAgentRows`); keeping the full resultData here would hold up to
+        // MAX_ROWS_FETCHED rows × every query in memory for the whole request
+        // for no functional gain.
+        data: resultData.slice(0, MAIN_AGENT_COMPLETE_ROWS),
         metadata: {
           totalRowsMatched,
           rowsReturned: resultData.length,
@@ -7553,13 +7822,13 @@ Analyze the error and try again with a corrected query.`;
     const fullSchema = this.options.preResolvedSchema || this.tool.fullSchema || this.tool.description || "No schema available";
     const databaseRules = await promptLoader.loadDatabaseRulesForType(sourceType);
     const rowLimitSyntax = {
-      "mssql": `Use SELECT TOP ${this.config.maxRowsPerSource} in every SELECT statement`,
-      "postgres": `Add LIMIT ${this.config.maxRowsPerSource} at the end of every query`,
-      "mysql": `Add LIMIT ${this.config.maxRowsPerSource} at the end of every query`,
-      "excel": `Add LIMIT ${this.config.maxRowsPerSource} at the end of every query`,
-      "csv": `Add LIMIT ${this.config.maxRowsPerSource} at the end of every query`
+      "mssql": `Use SELECT TOP ${this.config.maxRowsFetched} in every SELECT statement`,
+      "postgres": `Add LIMIT ${this.config.maxRowsFetched} at the end of every query`,
+      "mysql": `Add LIMIT ${this.config.maxRowsFetched} at the end of every query`,
+      "excel": `Add LIMIT ${this.config.maxRowsFetched} at the end of every query`,
+      "csv": `Add LIMIT ${this.config.maxRowsFetched} at the end of every query`
     };
-    const rowLimitInstruction = rowLimitSyntax[sourceType] || `Limit results to ${this.config.maxRowsPerSource} rows`;
+    const rowLimitInstruction = rowLimitSyntax[sourceType] || `Limit results to ${this.config.maxRowsFetched} rows`;
     const hasSchemaSearch = !this.options.skipSchemaSearch && !!this.tool.schemaSearchFn;
     const schemaTier = this.tool.schemaTier || "";
     let schemaSearchInstructions = "";
@@ -7598,8 +7867,13 @@ Even if a table appears in the detailed schema above, search_schema returns samp
       FULL_SCHEMA: fullSchema,
       DATABASE_RULES: databaseRules,
       ROW_LIMIT_SYNTAX: rowLimitInstruction,
+      // NOTE: MAX_ROWS here governs the SQL row cap (the {{MAX_ROWS}} in the
+      // "limit results to N rows" line), so it MUST match the fetch cap that
+      // ROW_LIMIT_SYNTAX states — not the smaller LLM-show cap — or the prompt
+      // contradicts itself ("max 10 rows" + "LIMIT 50") and the LLM picks its
+      // own number. See maxRowsFetched override below.
       SCHEMA_SEARCH_INSTRUCTIONS: schemaSearchInstructions,
-      MAX_ROWS: String(this.config.maxRowsPerSource),
+      MAX_ROWS: String(this.config.maxRowsFetched),
       AGGREGATION_MODE: aggregation,
       GLOBAL_KNOWLEDGE_BASE: this.config.globalKnowledgeBase || "No global knowledge base available.",
       CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
@@ -7721,13 +7995,27 @@ Even if a table appears in the detailed schema above, search_schema returns samp
 // src/userResponse/scripts/script-runner.ts
 import { spawn } from "child_process";
 import * as path5 from "path";
+import * as os from "os";
 
 // src/userResponse/scripts/script-ipc.ts
 function encodeMessage(msg) {
   return JSON.stringify(msg) + "\n";
 }
+var LineOverflowError = class extends Error {
+  constructor(buffered, limit) {
+    super(`IPC message exceeded ${limit} bytes without a newline (${buffered} buffered) \u2014 aborting to avoid unbounded memory.`);
+    this.buffered = buffered;
+    this.limit = limit;
+    this.name = "LineOverflowError";
+  }
+};
+var DEFAULT_MAX_IPC_BYTES = (() => {
+  const envVal = Number(process.env.SCRIPT_MAX_IPC_BYTES);
+  return Number.isFinite(envVal) && envVal > 0 ? Math.floor(envVal) : 64 * 1024 * 1024;
+})();
 var LineSplitter = class {
-  constructor() {
+  constructor(maxBytes = DEFAULT_MAX_IPC_BYTES) {
+    this.maxBytes = maxBytes;
     this.buffer = "";
   }
   push(chunk) {
@@ -7739,6 +8027,11 @@ var LineSplitter = class {
       this.buffer = this.buffer.slice(idx + 1);
       if (line.length > 0) lines.push(line);
     }
+    if (this.buffer.length > this.maxBytes) {
+      const buffered = this.buffer.length;
+      this.buffer = "";
+      throw new LineOverflowError(buffered, this.maxBytes);
+    }
     return lines;
   }
   /** Flush any remaining partial data (useful on stream close) */
@@ -7751,7 +8044,57 @@ var LineSplitter = class {
 };
 
 // src/userResponse/scripts/script-runner.ts
-var SCRIPT_TIMEOUT_MS = 3e5;
+var DEFAULT_SCRIPT_TIMEOUT_MS = 6e4;
+var SCRIPT_TIMEOUT_MS = (() => {
+  const envVal = Number(process.env.SCRIPT_TIMEOUT_MS);
+  return Number.isFinite(envVal) && envVal > 0 ? envVal : DEFAULT_SCRIPT_TIMEOUT_MS;
+})();
+var MAX_CONCURRENT_SCRIPTS = (() => {
+  const envVal = Number(process.env.SCRIPT_MAX_CONCURRENCY);
+  if (Number.isFinite(envVal) && envVal > 0) return Math.floor(envVal);
+  return Math.max(2, Math.min(8, (os.cpus()?.length || 4) - 1));
+})();
+var MAX_SCRIPT_QUEUE = (() => {
+  const envVal = Number(process.env.SCRIPT_MAX_QUEUE);
+  if (Number.isFinite(envVal) && envVal >= 0) return Math.floor(envVal);
+  return 100;
+})();
+var ScriptCapacityError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "ScriptCapacityError";
+  }
+};
+var Semaphore = class {
+  constructor(max, maxQueue) {
+    this.max = max;
+    this.maxQueue = maxQueue;
+    this.active = 0;
+    this.waiters = [];
+  }
+  async acquire() {
+    if (this.active < this.max) {
+      this.active++;
+      return;
+    }
+    if (this.waiters.length >= this.maxQueue) {
+      throw new ScriptCapacityError(
+        `Script execution at capacity (${this.max} running, ${this.waiters.length} queued). Try again shortly.`
+      );
+    }
+    return new Promise((resolve2) => this.waiters.push(resolve2));
+  }
+  release() {
+    const next = this.waiters.shift();
+    if (next) next();
+    else this.active--;
+  }
+  get queued() {
+    return this.waiters.length;
+  }
+};
+var scriptSemaphore = new Semaphore(MAX_CONCURRENT_SCRIPTS, MAX_SCRIPT_QUEUE);
+logger.info(`[ScriptRunner] Concurrency cap: ${MAX_CONCURRENT_SCRIPTS} concurrent, queue ${MAX_SCRIPT_QUEUE}`);
 var tsxBinaryPath = null;
 function resolveTsxBinary() {
   if (tsxBinaryPath) return tsxBinaryPath;
@@ -7774,6 +8117,22 @@ async function runScript(recipe, scriptPath, params, options) {
   const resolvedParams = coerceParams(recipe, applyDefaults(recipe, params));
   const paramSchema = buildParamSchema(recipe);
   logger.info(`[ScriptRunner] Executing "${recipe.name}" (${recipe.id}) with params: ${JSON.stringify(resolvedParams)}`);
+  try {
+    if (scriptSemaphore.queued > 0) {
+      logger.info(`[ScriptRunner] "${recipe.name}" queued (${scriptSemaphore.queued} ahead)`);
+    }
+    await scriptSemaphore.acquire();
+  } catch (capErr) {
+    const msg = capErr instanceof Error ? capErr.message : String(capErr);
+    logger.warn(`[ScriptRunner] "${recipe.name}" shed: ${msg}`);
+    return {
+      success: false,
+      data: [],
+      executedQueries: [],
+      error: msg,
+      executionTimeMs: Date.now() - startedAt
+    };
+  }
   try {
     const result = await executeInSubprocess(
       scriptPath,
@@ -7815,6 +8174,8 @@ async function runScript(recipe, scriptPath, params, options) {
       error: msg,
       executionTimeMs: totalMs
     };
+  } finally {
+    scriptSemaphore.release();
   }
 }
 function executeInSubprocess(scriptPath, params, paramSchema, externalTools, streamBuffer, timeoutMs) {
@@ -7827,6 +8188,15 @@ function executeInSubprocess(scriptPath, params, paramSchema, externalTools, str
       [tsxBin, bootstrap, scriptPath],
       {
         stdio: ["pipe", "pipe", "pipe"],
+        // `detached: true` makes the child a process-group LEADER. tsx 4.x
+        // re-execs node to run the user script, so the script is a GRANDCHILD;
+        // a plain `child.kill()` would hit only the tsx wrapper and orphan the
+        // grandchild (a runaway/looping script that outlives the timeout — #4).
+        // With its own group we can SIGKILL the whole tree via `-pid` in
+        // cleanup(). We deliberately do NOT unref() — the parent still tracks
+        // the child's exit and pipes normally. (POSIX; Windows falls back to
+        // a direct kill in cleanup().)
+        detached: process.platform !== "win32",
         env: {
           ...process.env,
           // Keep the child quiet about dotenv etc.
@@ -7834,22 +8204,23 @@ function executeInSubprocess(scriptPath, params, paramSchema, externalTools, str
         }
       }
     );
-    const toolsById = /* @__PURE__ */ new Map();
-    const toolsByName = /* @__PURE__ */ new Map();
-    for (const t of externalTools) {
-      toolsById.set(t.id, t);
-      toolsByName.set(t.name.toLowerCase(), t);
-    }
     let resolved = false;
     let executedQueries = [];
     let stderrBuffer = "";
     const cleanup = () => {
-      if (!child.killed) {
+      if (child.killed) return;
+      const pid = child.pid;
+      if (pid && process.platform !== "win32") {
         try {
-          child.kill("SIGKILL");
+          process.kill(-pid, "SIGKILL");
+          return;
         } catch {
         }
       }
+      try {
+        child.kill("SIGKILL");
+      } catch {
+      }
     };
     const finish = (r) => {
       if (resolved) return;
@@ -7869,7 +8240,19 @@ function executeInSubprocess(scriptPath, params, paramSchema, externalTools, str
     const stdoutSplitter = new LineSplitter();
     child.stdout.setEncoding("utf-8");
     child.stdout.on("data", (chunk) => {
-      for (const line of stdoutSplitter.push(chunk)) {
+      let lines;
+      try {
+        lines = stdoutSplitter.push(chunk);
+      } catch (err) {
+        finish({
+          kind: "err",
+          phase: "ipc",
+          message: err instanceof Error ? err.message : String(err),
+          executedQueries
+        });
+        return;
+      }
+      for (const line of lines) {
         let msg;
         try {
           msg = JSON.parse(line);
@@ -7913,9 +8296,6 @@ function executeInSubprocess(scriptPath, params, paramSchema, externalTools, str
     child.stdin.write(init);
     function handleChildMessage(msg) {
       switch (msg.type) {
-        case "query":
-          void handleQuery8(msg.id, msg.toolId, msg.sql);
-          return;
         case "stream":
           if (streamBuffer?.hasCallback()) {
             streamBuffer.write(msg.chunk);
@@ -7941,117 +8321,21 @@ ${msg.stack}` : ""),
           return;
       }
     }
-    async function handleQuery8(id, toolId, sql) {
-      const tool = resolveAuthorizedTool(toolId, toolsById, toolsByName);
-      if (!tool) {
-        sendToChild({
-          type: "query_error",
-          id,
-          error: `Tool "${toolId}" is not authorized for this request.`
-        });
-        return;
-      }
-      const startedAt = Date.now();
-      streamQueryStart(streamBuffer, tool.name, sql);
-      try {
-        const result = await tool.fn({ sql });
-        const elapsed = Date.now() - startedAt;
-        if (result && result.error) {
-          const errMsg = typeof result.error === "string" ? result.error : JSON.stringify(result.error);
-          streamQueryDone(streamBuffer, elapsed);
-          streamQueryError(streamBuffer, tool.name, errMsg);
-          sendToChild({ type: "query_error", id, error: errMsg });
-          return;
-        }
-        const data = result?.data || [];
-        const count = result?.count ?? data.length;
-        const totalCount = result?.metadata?.totalCount;
-        streamQueryDone(streamBuffer, elapsed);
-        streamQuerySuccess(streamBuffer, tool.name, data.length, totalCount ?? count);
-        streamDataPreview(streamBuffer, data);
-        sendToChild({
-          type: "query_result",
-          id,
-          data,
-          count,
-          metadata: { totalCount, executionTimeMs: elapsed }
-        });
-      } catch (err) {
-        const elapsed = Date.now() - startedAt;
-        const errMsg = err instanceof Error ? err.message : String(err);
-        streamQueryDone(streamBuffer, elapsed);
-        streamQueryError(streamBuffer, tool.name, errMsg);
-        sendToChild({ type: "query_error", id, error: errMsg });
-      }
-    }
-    function sendToChild(msg) {
-      try {
-        child.stdin.write(encodeMessage(msg));
-      } catch (err) {
-        logger.warn(`[ScriptRunner] Failed to write to child stdin: ${err.message}`);
-      }
-    }
   });
 }
-function resolveAuthorizedTool(toolId, byId, byName) {
-  const exact = byId.get(toolId);
-  if (exact) return exact;
-  const exactName = byName.get(toolId.toLowerCase());
-  if (exactName) return exactName;
-  for (const [id, tool] of byId) {
-    if (id.includes(toolId) || toolId.includes(id)) return tool;
-  }
-  const lower = toolId.toLowerCase();
-  for (const [name, tool] of byName) {
-    if (name.includes(lower) || lower.includes(name)) return tool;
-  }
-  return null;
-}
-function streamQueryStart(sb, sourceName, sql) {
-  if (!sb?.hasCallback()) return;
-  sb.write(`
-\u{1F4DD} **Querying ${sourceName}:**
-\`\`\`sql
-${sql}
-\`\`\`
-
-`);
-  sb.write(`__QUERY_TIMER_START_Executing query__`);
-}
-function streamQueryDone(sb, ms) {
-  if (!sb?.hasCallback()) return;
-  sb.write(`__QUERY_TIMER_DONE_${(ms / 1e3).toFixed(1)}__
-
-`);
-}
-function streamQuerySuccess(sb, sourceName, rows, total) {
-  if (!sb?.hasCallback()) return;
-  const totalInfo = total > rows ? ` of ${total} total` : "";
-  sb.write(`\u2705 **${rows} rows${totalInfo} from ${sourceName}**
-
-`);
-}
-function streamQueryError(sb, sourceName, msg) {
-  if (!sb?.hasCallback()) return;
-  sb.write(`\u274C **Query failed on ${sourceName}:** ${msg}
-
-`);
-}
-function streamDataPreview(sb, data) {
-  if (!sb?.hasCallback() || data.length === 0) return;
-  try {
-    sb.write(`<DataTable>${JSON.stringify(data.slice(0, 10))}</DataTable>
-
-`);
-  } catch {
-    sb.write(`_Data preview not available_
-
-`);
-  }
-}
 function withSynthesizedFinalData(queries, finalData, finalCount) {
   if (!finalData || finalData.length === 0) return queries;
-  if (!queries || queries.length === 0) return queries;
+  if (!queries || queries.length === 0) {
+    return [{
+      sourceId: "computed:_final",
+      sourceName: "Script final dataset",
+      sql: "-- script final returned data (runTool / pure JS, no ctx.query)",
+      data: finalData,
+      count: finalCount,
+      executionTimeMs: 0,
+      virtual: true
+    }];
+  }
   const firstSqlQuery = queries.find((q) => !q.virtual);
   const rawSample = firstSqlQuery?.data?.[0];
   const finalSample = finalData[0];
@@ -8191,10 +8475,11 @@ var EXECUTE_SCRIPT_TOOL_DEF = {
 };
 var MAX_SCRIPT_ATTEMPTS = 3;
 var MainAgent = class {
-  constructor(externalTools, config, scriptStore, turnId, streamBuffer) {
+  constructor(externalTools, config, scriptStore, turnId, streamBuffer, workflows = []) {
     this.createdFromPrompt = "";
     this.scriptState = { recipeId: null, attempts: 0, lastSuccessfulResult: null };
     this.externalTools = externalTools;
+    this.workflows = workflows;
     this.config = config;
     this.streamBuffer = streamBuffer || new StreamBuffer();
     this.scriptStore = scriptStore ?? null;
@@ -8220,22 +8505,31 @@ var MainAgent = class {
     this.createdFromPrompt = userPrompt;
     const sourceTools = this.externalTools.filter((t) => t.toolType !== "direct");
     const directTools = this.externalTools.filter((t) => t.toolType === "direct");
-    logger.info(`[MainAgent] ${sourceTools.length} source tool(s), ${directTools.length} direct tool(s)`);
+    logger.info(`[MainAgent] ${sourceTools.length} source tool(s), ${directTools.length} direct tool(s), ${this.workflows.length} workflow(s)`);
     const summaries = buildSourceSummaries(sourceTools);
-    const systemPrompt = await this.buildSystemPrompt(summaries, directTools, conversationHistory);
+    const systemPrompt = await this.buildSystemPrompt(summaries, directTools, this.workflows, conversationHistory);
     logger.logLLMPrompt("mainAgent", "system", extractPromptText(systemPrompt));
     logger.logLLMPrompt("mainAgent", "user", userPrompt);
     const sourceToolDefs = this.buildSourceToolDefinitions(summaries);
     const directToolDefs = this.buildDirectToolDefinitions(directTools);
+    const workflowToolDefs = this.buildWorkflowToolDefinitions(this.workflows);
     const tools = [
       ...sourceToolDefs,
       ...directToolDefs,
+      ...workflowToolDefs,
       ...this.scriptingEnabled ? [WRITE_SCRIPT_TOOL_DEF, EXECUTE_SCRIPT_TOOL_DEF] : []
     ];
     const sourceResults = [];
     const executedTools = [];
     let sourceCallCounter = 0;
+    let selectedWorkflow;
     const toolHandler = async (toolName, toolInput) => {
+      const workflow = this.workflows.find((w) => w.id === toolName);
+      if (workflow) {
+        return this.handleWorkflow(workflow, toolInput, (w) => {
+          selectedWorkflow = w;
+        });
+      }
       if (toolName === "write_script") {
         if (!this.scriptingEnabled) return "Scripting is not enabled for this request.";
         return this.handleWriteScript(toolInput);
@@ -8300,24 +8594,34 @@ var MainAgent = class {
       this.config.maxIterations
     );
     const totalTime = Date.now() - startTime;
-    logger.info(`[MainAgent] Complete | ${sourceResults.length} source queries, ${executedTools.length} successful | ${totalTime}ms`);
-    const savedScript = this.buildSavedScript();
+    logger.info(
+      `[MainAgent] Complete | ${sourceResults.length} source queries, ${executedTools.length} successful${selectedWorkflow ? ` | workflow="${selectedWorkflow.name}"` : ""} | ${totalTime}ms`
+    );
+    const savedScript = await this.buildSavedScript();
     if (savedScript) {
       logger.info(`[MainAgent] Script authored: "${savedScript.name}" (${savedScript.parameters.length} params, ${this.scriptState.attempts} attempt${this.scriptState.attempts === 1 ? "" : "s"})`);
     } else if (sourceResults.some((r) => r.success)) {
       logger.warn(`[MainAgent] Source query succeeded but no script was authored \u2014 LLM skipped write_script/execute_script. Prompt policy may need tightening.`);
     }
+    if (!savedScript && this.scriptStore && this.scriptState.recipeId) {
+      try {
+        await this.scriptStore.discardDraft(this.scriptState.recipeId);
+      } catch (err) {
+        logger.warn(`[MainAgent] Failed to discard failed draft ${this.scriptState.recipeId}: ${err}`);
+      }
+    }
     return {
       text,
       executedTools,
       sourceResults,
-      savedScript
+      savedScript,
+      workflow: selectedWorkflow
     };
   }
   // ============================================
   // Script-authoring tool handlers
   // ============================================
-  handleWriteScript(toolInput) {
+  async handleWriteScript(toolInput) {
     const scriptBody = typeof toolInput?.scriptBody === "string" ? toolInput.scriptBody.trim() : "";
     if (!scriptBody) {
       return "write_script requires a non-empty `scriptBody` starting with `export async function getData(ctx, params)`.";
@@ -8326,7 +8630,7 @@ var MainAgent = class {
     const intentDescription = toolInput.description || "";
     const tags = Array.isArray(toolInput.tags) ? toolInput.tags : [];
     const parameters = Array.isArray(toolInput.parameters) ? this.normalizeParameterList(toolInput.parameters) : [];
-    const draft = this.scriptStore.saveDraft({
+    const draft = await this.scriptStore.saveDraft({
       recipeId: this.scriptState.recipeId ?? void 0,
       turnId: this.turnId,
       name,
@@ -8347,7 +8651,7 @@ var MainAgent = class {
     if (!this.scriptState.recipeId) {
       return "No draft found. Call write_script first with the script you want to verify.";
     }
-    const draftRecipe = this.scriptStore.get(this.scriptState.recipeId);
+    const draftRecipe = await this.scriptStore.get(this.scriptState.recipeId);
     if (!draftRecipe) {
       logger.error(`[MainAgent] execute_script: draft "${this.scriptState.recipeId}" missing from store`);
       return "Draft was lost from the store. Call write_script again.";
@@ -8365,10 +8669,17 @@ var MainAgent = class {
     );
     if (result.success) {
       this.scriptState.lastSuccessfulResult = result;
+      const totalRows = result.data.length;
       const summary2 = {
         ok: true,
         executionTimeMs: result.executionTimeMs,
+        totalRows,
+        // dataSummary is computed over ALL rows — use it for any range/count/
+        // min/max/total/"all-or-none" claim. sampleRows is just the first few
+        // rows (head of result, NOT representative) for illustrating shape.
+        dataSummary: summarizeRows(result.data),
         sampleRows: result.data.slice(0, 5),
+        analysisGuidance: `Showing ${Math.min(totalRows, 5)} of ${totalRows} rows. The sample is the HEAD of the result and is NOT representative \u2014 do not generalize from it. Base every number, range, count, and "all/none" statement on dataSummary, which is computed over all ${totalRows} rows. The full result is rendered for the user by the on-screen component.`,
         executedQueries: result.executedQueries.map((q) => ({
           sourceId: q.sourceId,
           sourceName: q.sourceName,
@@ -8376,10 +8687,10 @@ var MainAgent = class {
           sample: q.data.slice(0, 2)
         }))
       };
-      logger.info(`[MainAgent] execute_script: ok \u2014 ${result.data.length} rows in ${result.executionTimeMs}ms (attempt ${this.scriptState.attempts})`);
+      logger.info(`[MainAgent] execute_script: ok \u2014 ${totalRows} rows in ${result.executionTimeMs}ms (attempt ${this.scriptState.attempts})`);
       return JSON.stringify(summary2, null, 2);
     }
-    this.scriptStore.recordDraftError(draftRecipe.id, {
+    await this.scriptStore.recordDraftError(draftRecipe.id, {
       phase: result.errorPhase ?? "runtime",
       message: result.error ?? "Unknown error",
       attempt: this.scriptState.attempts
@@ -8405,11 +8716,11 @@ var MainAgent = class {
    * `ScriptStore.promoteToVerified()`. Only returned when a verified
    * successful execution is on record.
    */
-  buildSavedScript() {
+  async buildSavedScript() {
     const { recipeId, lastSuccessfulResult } = this.scriptState;
     if (!recipeId || !lastSuccessfulResult) return void 0;
     if (!this.scriptStore) return void 0;
-    const draft = this.scriptStore.get(recipeId);
+    const draft = await this.scriptStore.get(recipeId);
     if (!draft) return void 0;
     const executedQueries = lastSuccessfulResult.executedQueries.map((q) => ({
       sourceId: q.sourceId,
@@ -8536,6 +8847,7 @@ ${lines.join("\n")}`;
         maxRows: 5,
         maxCharsPerField: 200
       });
+      const sampleData = Array.isArray(resultData) ? resultData.slice(0, TOOL_TRACKING_SAMPLE_ROWS) : Array.isArray(formattedResult.data) ? formattedResult.data.slice(0, TOOL_TRACKING_SAMPLE_ROWS) : [];
       executedTools.push({
         id: tool.id,
         name: tool.name,
@@ -8544,7 +8856,7 @@ ${lines.join("\n")}`;
           _totalRecords: result.totalItems || result.count || rowCount,
           _recordsShown: rowCount,
           _metadata: result.metadata,
-          _sampleData: Array.isArray(resultData) ? resultData.slice(0, 3) : [resultData]
+          _sampleData: sampleData
         },
         outputSchema: tool.outputSchema
       });
@@ -8570,9 +8882,10 @@ ${formatted}`;
   // System Prompt
   // ============================================
   /**
-   * Build the main agent's system prompt with source summaries and direct tool descriptions.
+   * Build the main agent's system prompt with source summaries, direct tool descriptions,
+   * and workflow component descriptions.
    */
-  async buildSystemPrompt(summaries, directTools, conversationHistory) {
+  async buildSystemPrompt(summaries, directTools, workflows, conversationHistory) {
     const summariesText = formatSummariesForPrompt(summaries);
     const maxSourceCalls = Math.max(2, this.config.maxIterations - 2);
     let directToolsText = "";
@@ -8584,10 +8897,28 @@ ${formatted}`;
   ${t.description || "No description"}${paramList ? "\n  Parameters:\n" + paramList : ""}`;
       }).join("\n\n");
     }
+    let workflowsText = "";
+    if (workflows.length > 0) {
+      workflowsText = workflows.map((w, idx) => {
+        const propLines = Object.entries(w.propsSchema || {}).map(([k, v]) => `    - ${k}: ${v}`).join("\n");
+        return [
+          `${idx + 1}. **${w.name}** (tool: ${w.id})`,
+          `  ${w.description}`,
+          `  When to use: ${w.whenToUse}`,
+          propLines ? `  Props:
+${propLines}` : ""
+        ].filter(Boolean).join("\n");
+      }).join("\n\n");
+    } else {
+      workflowsText = "No workflow components registered for this project.";
+    }
     const prompts = await promptLoader.loadPrompts("agent-main", {
       SOURCE_SUMMARIES: summariesText,
       DIRECT_TOOLS: directToolsText,
-      MAX_ROWS: String(this.config.maxRowsPerSource),
+      WORKFLOW_COMPONENTS: workflowsText,
+      MAX_ROWS: String(MAIN_AGENT_COMPLETE_ROWS),
+      MAX_ROWS_RENDERED: String(MAX_ROWS_RENDERED),
+      MAX_ROWS_FETCHED: String(MAX_ROWS_FETCHED),
       MAX_SOURCE_CALLS: String(maxSourceCalls),
       GLOBAL_KNOWLEDGE_BASE: this.config.globalKnowledgeBase || "No global knowledge base available.",
       CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
@@ -8673,6 +9004,70 @@ ${formatted}`;
     });
   }
   // ============================================
+  // Workflow Handling
+  // ============================================
+  /**
+   * Capture a workflow selection. We do NOT execute anything — the LLM has
+   * already extracted the props it wants the workflow rendered with. We
+   * record the selection (via the capture callback) and return a short
+   * acknowledgement so the LLM ends its turn cleanly without writing
+   * analysis text or calling more tools.
+   */
+  async handleWorkflow(workflow, toolInput, capture) {
+    const props = { ...workflow.defaultProps || {}, ...toolInput || {} };
+    logger.info(
+      `[MainAgent] Workflow selected: "${workflow.name}" | props: ${JSON.stringify(props).substring(0, 200)}`
+    );
+    if (this.streamBuffer.hasCallback()) {
+      this.streamBuffer.write(`
+
+\u{1F9ED} **Launching ${workflow.name} workflow...**
+
+`);
+      await streamDelay();
+    }
+    capture({ name: workflow.name, props });
+    return `\u2705 Workflow "${workflow.name}" selected. The UI will render now \u2014 do NOT write analysis text or call any other tools. End your turn.`;
+  }
+  /**
+   * Build LLM tool definitions for workflow components. The workflow's
+   * propsSchema becomes the tool's input_schema so the LLM extracts props
+   * directly from the prompt — same mechanic as direct tools.
+   */
+  buildWorkflowToolDefinitions(workflows) {
+    return workflows.map((workflow) => {
+      const properties = {};
+      const required = [];
+      Object.entries(workflow.propsSchema || {}).forEach(([key, typeOrValue]) => {
+        const valueStr = String(typeOrValue).toLowerCase();
+        let schemaType = "string";
+        const typeMatch = valueStr.match(/^(string|number|integer|boolean|array|object)\b/);
+        if (typeMatch) {
+          schemaType = typeMatch[1];
+        }
+        const isOptional = valueStr.includes("(optional)") || valueStr.includes("optional");
+        const description = typeof typeOrValue === "string" ? typeOrValue : `Prop: ${key}`;
+        if (schemaType === "array") {
+          properties[key] = { type: "array", items: {}, description };
+        } else if (schemaType === "object") {
+          properties[key] = { type: "object", description };
+        } else {
+          properties[key] = { type: schemaType, description };
+        }
+        if (!isOptional) required.push(key);
+      });
+      return {
+        name: workflow.id,
+        description: `[WORKFLOW] ${workflow.description} \u2014 When to use: ${workflow.whenToUse}`,
+        input_schema: {
+          type: "object",
+          properties,
+          required: required.length > 0 ? required : void 0
+        }
+      };
+    });
+  }
+  // ============================================
   // Format Result for Main Agent
   // ============================================
   /**
@@ -8682,7 +9077,7 @@ ${formatted}`;
     if (!result.success) {
       return `Data source "${result.sourceName}" could not fulfill the request: ${result.error}. Try rephrasing your intent or querying a different source.`;
     }
-    const { data, metadata } = result;
+    const { metadata } = result;
     let output = `## Data from "${result.sourceName}"
 `;
     output += `Rows returned: ${metadata.rowsReturned}`;
@@ -8693,33 +9088,68 @@ ${formatted}`;
 Execution time: ${metadata.executionTimeMs}ms
 
 `;
-    if (data.length === 0) {
+    const successfulTools = result.allExecutedTools && result.allExecutedTools.length > 0 ? result.allExecutedTools : result.executedTool ? [result.executedTool] : [];
+    const queries = successfulTools.map((t) => ({
+      sql: t?.params?.sql || t?.params?.query,
+      rows: t?.result?._mainAgentRows ?? t?.result?._sampleData ?? [],
+      summary: t?.result?._summary,
+      total: t?.result?._totalRecords,
+      shown: t?.result?._recordsShown
+    })).filter((q) => Boolean(q.sql));
+    if (queries.length === 0) {
       output += "No data returned.";
       return output;
     }
-    const maxRowsForLLM = Math.min(data.length, 10);
-    const truncatedData = data.slice(0, maxRowsForLLM).map((row) => {
-      const truncatedRow = {};
+    const truncRows = (rowsArr) => rowsArr.map((row) => {
+      const out = {};
       for (const [key, value] of Object.entries(row)) {
-        if (typeof value === "string" && value.length > 200) {
-          truncatedRow[key] = value.substring(0, 200) + "...";
-        } else {
-          truncatedRow[key] = value;
-        }
+        out[key] = typeof value === "string" && value.length > 200 ? safeTruncate(value, 200) + "..." : value;
       }
-      return truncatedRow;
+      return out;
     });
-    const truncationNote = data.length > maxRowsForLLM ? `
-(showing ${maxRowsForLLM} of ${data.length} rows)` : "";
-    output += `### Results (${data.length} rows${truncationNote})
+    const jsonBlock = (obj) => {
+      try {
+        return "```json\n" + JSON.stringify(obj, null, 2) + "\n```\n";
+      } catch {
+        return "```\n[Data could not be serialized]\n```\n";
+      }
+    };
+    const scopeWarning = `PRESERVE every filter and scope \u2014 brand, product, date range, AND geography/segment (\`State\`/\`Region\`/\`District\`). Do NOT broaden, drop, or "simplify" any predicate.`;
+    output += queries.length === 1 ? `### The SQL below and its result \u2014 copy the SQL VERBATIM into \`write_script\` (inside \`ctx.query(...)\`)
+${scopeWarning}
+
+` : `### ${queries.length} successful queries \u2014 each block pairs ONE query's SQL with ITS OWN result. Do NOT mix a query's SQL with another query's data.
+${scopeWarning} The script re-runs them, so you can include more than one \`ctx.query(...)\`.
+
 `;
-    output += "```json\n";
-    try {
-      output += JSON.stringify(truncatedData, null, 2);
-    } catch {
-      output += "[Data could not be serialized]";
-    }
-    output += "\n```\n";
+    queries.forEach((q, i) => {
+      const total = q.total ?? q.rows.length;
+      const capped = (q.shown ?? q.rows.length) >= MAX_ROWS_FETCHED;
+      output += queries.length === 1 ? `**SQL:**
+` : `### Query ${i + 1} \u2014 ${capped ? `${MAX_ROWS_FETCHED}+ (TRUNCATED)` : `${total}`} rows
+**SQL for Query ${i + 1}:**
+`;
+      output += "```sql\n" + String(q.sql) + "\n```\n";
+      if (capped) {
+        output += `> \u26A0\uFE0F **TRUNCATED, NON-REPRESENTATIVE SAMPLE.** This query hit the ${MAX_ROWS_FETCHED}-row exploration cap, so it returned only the FIRST ${MAX_ROWS_FETCHED} rows \u2014 and because it is \`ORDER BY\`'d, those rows are likely ALL from one group (e.g. one District/category), not a cross-section. Do NOT conclude any group "has no data" or read any count/total/min/max from this. To reason over all rows, re-query with **aggregation** (\`GROUP BY\` the dimension, e.g. \`SELECT District, COUNT(*) ... GROUP BY District\`) or compute the metric in SQL \u2014 never from this slice.
+`;
+      }
+      if (total <= MAIN_AGENT_COMPLETE_ROWS) {
+        const rows = truncRows(q.rows);
+        output += `**All ${rows.length} rows (COMPLETE result \u2014 this is the entire result set, nothing hidden):**
+`;
+        output += jsonBlock(rows);
+      } else {
+        const samples = truncRows(q.rows.slice(0, 3));
+        output += `**Result summary \u2014 computed over ALL ${total} rows. Use THIS for every count, range, total, "all/none", and per-group number; do NOT infer them from the samples:**
+`;
+        output += jsonBlock(q.summary ?? { totalRows: total });
+        output += `**Sample rows (first ${samples.length} of ${total} \u2014 to show row SHAPE only, NOT representative of the distribution):**
+`;
+        output += jsonBlock(samples);
+      }
+      output += "\n";
+    });
     return output;
   }
   /**
@@ -8745,14 +9175,15 @@ function extractTablesFromSQL(sqls) {
 // src/userResponse/agents/types.ts
 var DEFAULT_AGENT_CONFIG = {
   maxRowsPerSource: 10,
+  maxRowsFetched: MAX_ROWS_FETCHED,
   mainAgentModel: "",
   // will use the provider's default model
   sourceAgentModel: "",
   // will use the provider's default model
   maxRetries: 2,
   // 2 retries = 3 total query attempts (1 initial + 2 retries for SQL errors)
-  maxIterations: 8
-  // schema search (2-3) + query attempts (2) + LLM responses + final
+  maxIterations: 12
+  // schema search (2-3) + query attempts (2) + write_script/execute_script (2-3) + LLM responses + final
 };
 
 // src/userResponse/utils/component-props-processor.ts
@@ -9026,6 +9457,7 @@ function formatExecutedTools(executedTools) {
    Fields:
 ${fieldsText}`;
     }
+    const MAX_SAMPLE_BLOCK_CHARS = 8e3;
     let sampleDataText = "";
     const sampleData = tool.result?._sampleData;
     if (Array.isArray(sampleData) && sampleData.length > 0) {
@@ -9033,8 +9465,11 @@ ${fieldsText}`;
       sampleDataText = `
    \u{1F511} RESULT FIELDS: ${sampleFields.join(", ")}`;
       try {
+        const stringified = JSON.stringify(sampleData, null, 2);
+        const capped = stringified.length > MAX_SAMPLE_BLOCK_CHARS ? `${stringified.substring(0, MAX_SAMPLE_BLOCK_CHARS)}
+... (truncated; ${stringified.length - MAX_SAMPLE_BLOCK_CHARS} more chars)` : stringified;
         sampleDataText += `
-   \u{1F4C4} SAMPLE ROW: ${JSON.stringify(sampleData[0])}`;
+   \u{1F4C4} SAMPLE ROWS (${sampleData.length}): ${capped}`;
       } catch {
       }
     }
@@ -9322,11 +9757,20 @@ Fixed SQL query:`;
   return validatedQuery;
 }
 
+// src/userResponse/scripts/script-metadata-store.ts
+function resolveScriptRecipeStore(collections) {
+  const s = collections?.["script-recipes"];
+  if (s && typeof s.search === "function" && typeof s.getById === "function") {
+    return s;
+  }
+  return null;
+}
+
 // src/userResponse/scripts/script-store.ts
 import * as fs6 from "fs";
 import * as path6 from "path";
+import { createHash, randomBytes } from "crypto";
 var DEFAULT_STORE_DIR = "scripts-store";
-var METADATA_SUBDIR = "metadata";
 function normalizeScriptBody(scriptBody) {
   let i = 0;
   const n = scriptBody.length;
@@ -9351,383 +9795,321 @@ function normalizeScriptBody(scriptBody) {
   } else if (/^\s*function\s+getData\b/.test(cleanBody)) {
     cleanBody = cleanBody.replace(/^\s*function\s+getData\b/, "export async function getData");
   }
+  cleanBody = cleanBody.replace(/(?<!\\)\\([dwsDWS])/g, "\\\\$1");
   return cleanBody;
 }
 var ScriptStore = class {
-  constructor(baseDir) {
-    this.recipes = /* @__PURE__ */ new Map();
-    this.loaded = false;
-    this.storeDir = baseDir || path6.join(process.cwd(), DEFAULT_STORE_DIR);
-  }
-  metadataDir() {
-    return path6.join(this.storeDir, METADATA_SUBDIR);
-  }
-  /**
-   * Filename base for a recipe. Drafts include the per-turn suffix so two
-   * concurrent turns can never clobber each other's draft files; verified
-   * recipes use the bare slug.
-   */
-  fileBaseName(recipe) {
-    const slug = this.toFileName(recipe.name);
-    if (recipe.status === "draft" && recipe.turnId) {
-      return `${slug}-${recipe.turnId}`;
+  constructor(opts) {
+    this.store = opts?.store ?? resolveScriptRecipeStore(opts?.collections) ?? null;
+    this.storeDir = opts?.baseDir || path6.join(process.cwd(), DEFAULT_STORE_DIR);
+    this.projectId = opts?.projectId;
+    if (!this.store) {
+      logger.warn("[ScriptStore] No script-recipes metadata store injected \u2014 script reuse disabled this run.");
     }
-    return slug;
   }
-  /**
-   * Absolute path to the .ts file for a recipe. Callers (e.g. ScriptRunner,
-   * MainAgent's execute_script) use this to hand off the path to the tsx child.
-   */
-  getScriptPath(recipe) {
-    return path6.join(this.storeDir, `${this.fileBaseName(recipe)}.ts`);
-  }
-  /**
-   * Absolute path to the metadata JSON for a recipe.
-   */
-  getMetadataPath(recipe) {
-    return path6.join(this.metadataDir(), `${this.fileBaseName(recipe)}.json`);
+  /** Whether a metadata store is wired (matcher / authoring are gated on this). */
+  hasStore() {
+    return this.store !== null;
   }
-  /**
-   * Get all VERIFIED recipes — drafts are filtered out so the matcher never
-   * considers an unverified script. Loads from disk on first access.
-   */
-  getAll() {
-    this.ensureLoaded();
-    return Array.from(this.recipes.values()).filter((r) => (r.status ?? "verified") === "verified");
+  // ============================================
+  // Read
+  // ============================================
+  /** Number of healthy verified recipes (gates the script-matching path). */
+  async count() {
+    if (!this.store) return 0;
+    try {
+      return await this.store.count({ projectId: this.projectId });
+    } catch (err) {
+      logger.warn(`[ScriptStore] count failed: ${err}`);
+      return 0;
+    }
   }
   /**
-   * Get a recipe by ID — returns drafts as well as verified scripts.
-   * Used by MainAgent and the promotion path.
+   * FTS shortlist for the matcher (metadata only — bodies are loaded lazily by
+   * `get()` once the LLM picks one). Returns verified, healthy recipes ranked
+   * by relevance.
    */
-  get(id) {
-    this.ensureLoaded();
-    return this.recipes.get(id) || null;
+  async search(prompt, limit) {
+    if (!this.store) return [];
+    try {
+      const rows = await this.store.search({ prompt, projectId: this.projectId, limit });
+      return rows.map((r) => this.rowToRecipe(r, ""));
+    } catch (err) {
+      logger.warn(`[ScriptStore] search failed: ${err}`);
+      return [];
+    }
   }
-  /**
-   * Number of verified recipes (matches `getAll().length`).
-   */
-  count() {
-    this.ensureLoaded();
-    return this.getAll().length;
+  /** Fetch one recipe by id with its body loaded from disk. */
+  async get(id) {
+    if (!this.store) return null;
+    try {
+      const row = await this.store.getById(id);
+      if (!row) return null;
+      const body = this.readBody(row.fileBase);
+      return this.rowToRecipe(row, body);
+    } catch (err) {
+      logger.warn(`[ScriptStore] get(${id}) failed: ${err}`);
+      return null;
+    }
   }
-  /**
-   * Save a recipe (create or update).
-   * File is named after the script: "order-status-distribution.json"
-   */
-  save(recipe) {
-    this.ensureLoaded();
-    recipe.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
-    this.recipes.set(recipe.id, recipe);
-    this.saveRecipeToDisk(recipe);
-    logger.info(`[ScriptStore] Saved script "${recipe.name}" (${recipe.id}) \u2014 ${this.recipes.size} total`);
+  // ============================================
+  // Write
+  // ============================================
+  /** Create or update a recipe (metadata upsert + body write when changed). */
+  async save(recipe) {
+    if (!this.store) return;
+    try {
+      if (!recipe.fileBase) {
+        recipe.fileBase = await this.computeFileBase(recipe.name, recipe.id);
+      }
+      if (recipe.scriptBody) {
+        const normalized = normalizeScriptBody(recipe.scriptBody);
+        const hash = this.hash(normalized);
+        if (hash !== recipe.bodyHash) {
+          this.writeBody(recipe.fileBase, normalized);
+          recipe.bodyHash = hash;
+        }
+      }
+      recipe.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
+      await this.store.upsert(this.recipeToRow(recipe));
+      logger.info(`[ScriptStore] Saved "${recipe.name}" (${recipe.id})`);
+    } catch (err) {
+      logger.warn(`[ScriptStore] save("${recipe.name}") failed: ${err}`);
+    }
   }
   /**
-   * Persist (or update) a draft recipe to disk. Always writes immediately so
-   * the `.ts` body is visible in the IDE the moment MainAgent calls
-   * `write_script`. Within one turn, retries that pass the same `recipeId`
-   * overwrite the same files (the LLM rewriting itself); a fresh `recipeId`
-   * is minted only on the first call of the turn.
-   *
-   * Filename includes the `turnId` suffix so concurrent turns never collide.
+   * Persist (or update) a draft. Within a turn, retries that pass the same
+   * `recipeId` overwrite the same row + file; a fresh `recipeId` mints a new
+   * draft. The body is visible at scripts-store/<fileBase>.ts immediately.
    */
-  saveDraft(input) {
-    this.ensureLoaded();
+  async saveDraft(input) {
+    if (!this.store) {
+      throw new Error("[ScriptStore] saveDraft called with no metadata store injected");
+    }
     const now = (/* @__PURE__ */ new Date()).toISOString();
-    const existing = input.recipeId ? this.recipes.get(input.recipeId) : void 0;
-    const reuseExisting = existing && (existing.status ?? "verified") === "draft" && existing.turnId === input.turnId;
-    const recipe = reuseExisting ? {
-      ...existing,
-      name: input.name,
-      intentDescription: input.intentDescription,
-      tags: input.tags,
-      parameters: input.parameters,
-      scriptBody: input.scriptBody,
-      updatedAt: now
-    } : {
-      id: `script_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
-      version: 1,
+    const existing = input.recipeId ? await this.store.getById(input.recipeId) : null;
+    const reuse = existing && existing.status === "draft" && existing.turnId === input.turnId;
+    const id = reuse ? existing.id : `script_${Date.now()}_${randomBytes(3).toString("hex")}`;
+    const fileBase = reuse ? existing.fileBase : await this.computeFileBase(input.name, id);
+    const normalized = normalizeScriptBody(input.scriptBody);
+    const bodyHash = this.hash(normalized);
+    this.writeBody(fileBase, normalized);
+    const recipe = {
+      id,
+      version: existing?.version ?? 1,
       name: input.name,
       intentDescription: input.intentDescription,
       tags: input.tags,
-      sourceIds: [],
-      tables: [],
+      sourceIds: existing?.sourceIds ?? [],
+      tables: existing?.tables ?? [],
       parameters: input.parameters,
-      scriptBody: input.scriptBody,
-      successCount: 0,
-      failureCount: 0,
+      scriptBody: normalized,
+      fileBase,
+      bodyHash,
+      projectId: this.projectId,
+      successCount: existing?.successCount ?? 0,
+      failureCount: existing?.failureCount ?? 0,
       lastUsed: now,
       createdFrom: input.createdFrom,
-      createdAt: now,
+      createdAt: existing?.createdAt ?? now,
       updatedAt: now,
-      forkDepth: 0,
+      forkDepth: existing?.forkDepth ?? 0,
       status: "draft",
       turnId: input.turnId
     };
-    this.recipes.set(recipe.id, recipe);
-    this.saveRecipeToDisk(recipe);
-    logger.info(
-      `[ScriptStore] ${reuseExisting ? "Updated" : "Saved"} draft "${recipe.name}" (${recipe.id}) at ${this.getScriptPath(recipe)}`
-    );
+    await this.store.upsert(this.recipeToRow(recipe));
+    logger.info(`[ScriptStore] ${reuse ? "Updated" : "Saved"} draft "${recipe.name}" (${id}) at ${this.getScriptPath(recipe)}`);
     return recipe;
   }
-  /**
-   * Stamp the draft with the most recent execution failure so it is visible
-   * in the metadata JSON without grepping logs. No-op if the recipe doesn't
-   * exist or has already been promoted.
-   */
-  recordDraftError(recipeId, err) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(recipeId);
-    if (!recipe || (recipe.status ?? "verified") !== "draft") return;
-    recipe.lastError = {
-      phase: err.phase,
-      message: err.message,
-      attempt: err.attempt,
-      at: (/* @__PURE__ */ new Date()).toISOString()
-    };
-    recipe.failureCount++;
-    recipe.updatedAt = (/* @__PURE__ */ new Date()).toISOString();
-    this.saveRecipeToDisk(recipe);
+  /** Stamp a draft's last execution error (metadata only). */
+  async recordDraftError(recipeId, err) {
+    if (!this.store) return;
+    try {
+      await this.store.recordDraftError(recipeId, { ...err, at: (/* @__PURE__ */ new Date()).toISOString() });
+    } catch (e) {
+      logger.warn(`[ScriptStore] recordDraftError failed: ${e}`);
+    }
   }
   /**
    * Promote a successfully-executed draft into a verified script.
-   *
-   * - Renames the on-disk files from `<slug>-<turnId>.{ts,json}` to the bare
-   *   `<slug>.{ts,json}`. If a verified file with the same slug already exists
-   *   (concurrent turn won the race, or a prior session has it), the recipe
-   *   keeps its turn-suffixed filename so we never overwrite verified work —
-   *   the matcher keys on `recipe.id`, so two siblings with similar slugs is fine.
-   * - Flips `status: 'verified'`, clears `lastError`, applies provenance
-   *   (`sourceIds`, `tables`) and optional fork lineage.
-   */
-  promoteToVerified(recipeId, input) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(recipeId);
-    if (!recipe) {
-      logger.warn(`[ScriptStore] promoteToVerified: recipe "${recipeId}" not found`);
-      return null;
-    }
-    if ((recipe.status ?? "verified") === "verified") {
-      logger.info(`[ScriptStore] promoteToVerified: recipe "${recipeId}" already verified \u2014 no-op`);
-      return recipe;
-    }
-    const slug = this.toFileName(recipe.name);
-    const canonicalTs = path6.join(this.storeDir, `${slug}.ts`);
-    const canonicalMeta = path6.join(this.metadataDir(), `${slug}.json`);
-    const canonicalFree = !fs6.existsSync(canonicalTs) && !fs6.existsSync(canonicalMeta);
-    const oldTsPath = this.getScriptPath(recipe);
-    const oldMetaPath = this.getMetadataPath(recipe);
-    recipe.status = "verified";
-    recipe.successCount = Math.max(1, recipe.successCount);
-    recipe.failureCount = 0;
-    recipe.lastError = void 0;
-    recipe.lastUsed = (/* @__PURE__ */ new Date()).toISOString();
-    recipe.updatedAt = recipe.lastUsed;
-    recipe.sourceIds = input.sourceIds;
-    recipe.tables = input.tables;
-    if (input.parentId !== void 0) recipe.parentId = input.parentId;
-    if (input.forkDepth !== void 0) recipe.forkDepth = input.forkDepth;
-    if (input.forkReason !== void 0) recipe.forkReason = input.forkReason;
-    if (canonicalFree) {
-      recipe.turnId = void 0;
-      try {
-        if (fs6.existsSync(oldTsPath)) fs6.unlinkSync(oldTsPath);
-        if (fs6.existsSync(oldMetaPath)) fs6.unlinkSync(oldMetaPath);
-      } catch (err) {
-        logger.warn(`[ScriptStore] promoteToVerified cleanup warning: ${err}`);
+   * The on-disk body already exists at <fileBase>.ts (written at write_script
+   * time) and keeps its name — only the DB row flips status + provenance.
+   */
+  async promoteToVerified(recipeId, input) {
+    if (!this.store) return null;
+    try {
+      const row = await this.store.promote(recipeId, {
+        sourceIds: input.sourceIds,
+        tables: input.tables,
+        parentId: input.parentId,
+        forkDepth: input.forkDepth,
+        forkReason: input.forkReason,
+        components: input.components
+      });
+      if (!row) {
+        logger.warn(`[ScriptStore] promoteToVerified: recipe "${recipeId}" not found`);
+        return null;
       }
-      this.saveRecipeToDisk(recipe);
-      logger.info(
-        `[ScriptStore] Promoted draft "${recipe.name}" (${recipe.id}) \u2192 ${this.getScriptPath(recipe)}`
-      );
-    } else {
-      this.saveRecipeToDisk(recipe);
-      logger.info(
-        `[ScriptStore] Promoted draft "${recipe.name}" (${recipe.id}) \u2014 canonical slug "${slug}" was taken, kept turn-suffixed filename ${this.getScriptPath(recipe)}`
-      );
+      logger.info(`[ScriptStore] Promoted "${row.name}" (${recipeId}) \u2192 verified`);
+      return this.rowToRecipe(row, this.readBody(row.fileBase));
+    } catch (err) {
+      logger.warn(`[ScriptStore] promoteToVerified failed: ${err}`);
+      return null;
     }
-    return recipe;
   }
   /**
-   * Drop a draft from disk + memory (e.g. when an outer error path wants to
-   * clean up). Per the agreed policy, MainAgent's normal failure path does
-   * NOT call this — failed drafts are kept on disk for the user to inspect.
+   * Drop a draft (row + body file). MainAgent calls this at end-of-turn when a
+   * draft was authored but never verified — failed drafts are never matched, so
+   * deleting them immediately avoids unbounded accumulation (#5). No-op if the
+   * recipe isn't a draft (so a promoted/verified script is never removed here).
    */
-  discardDraft(recipeId) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(recipeId);
-    if (!recipe || (recipe.status ?? "verified") !== "draft") return;
-    this.recipes.delete(recipeId);
-    this.deleteRecipeFromDisk(recipe);
-    logger.info(`[ScriptStore] Discarded draft "${recipe.name}" (${recipeId})`);
+  async discardDraft(recipeId) {
+    await this.removeById(recipeId, "draft");
   }
-  /**
-   * Delete a recipe by ID.
-   */
-  delete(id) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(id);
-    if (recipe) {
-      this.recipes.delete(id);
-      this.deleteRecipeFromDisk(recipe);
-      logger.info(`[ScriptStore] Deleted script "${recipe.name}" (${id})`);
-    }
+  /** Delete a recipe (row + body file). */
+  async delete(id) {
+    await this.removeById(id);
   }
-  /**
-   * Record a successful execution for a recipe.
-   */
-  recordSuccess(id) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(id);
-    if (recipe) {
-      recipe.successCount++;
-      recipe.lastUsed = (/* @__PURE__ */ new Date()).toISOString();
-      this.saveRecipeToDisk(recipe);
+  /** Record a successful execution (atomic counter bump). */
+  async recordSuccess(id) {
+    if (!this.store) return;
+    try {
+      await this.store.updateStats(id, { successDelta: 1, lastUsed: (/* @__PURE__ */ new Date()).toISOString() });
+    } catch (err) {
+      logger.warn(`[ScriptStore] recordSuccess failed: ${err}`);
     }
   }
-  /**
-   * Record a failed execution for a recipe.
-   */
-  recordFailure(id) {
-    this.ensureLoaded();
-    const recipe = this.recipes.get(id);
-    if (recipe) {
-      recipe.failureCount++;
-      recipe.lastUsed = (/* @__PURE__ */ new Date()).toISOString();
-      this.saveRecipeToDisk(recipe);
+  /** Record a failed execution (atomic counter bump). */
+  async recordFailure(id) {
+    if (!this.store) return;
+    try {
+      await this.store.updateStats(id, { failureDelta: 1, lastUsed: (/* @__PURE__ */ new Date()).toISOString() });
+    } catch (err) {
+      logger.warn(`[ScriptStore] recordFailure failed: ${err}`);
     }
   }
-  /**
-   * Get the failure rate for a recipe (0 to 1).
-   * Returns 0 if the recipe has fewer than 5 total uses.
-   */
-  getFailureRate(id) {
-    const recipe = this.recipes.get(id);
-    if (!recipe) return 0;
-    const total = recipe.successCount + recipe.failureCount;
-    if (total < 5) return 0;
-    return recipe.failureCount / total;
+  // ============================================
+  // Paths (sync — body lives on disk)
+  // ============================================
+  /** Absolute path to the .ts body for a recipe (used by the runner/MainAgent). */
+  getScriptPath(recipe) {
+    const base = recipe.fileBase || this.toSlug(recipe.name);
+    return path6.join(this.storeDir, `${base}.ts`);
   }
   // ============================================
-  // Persistence — one JSON file per script
+  // Internals
   // ============================================
-  ensureLoaded() {
-    if (!this.loaded) {
-      this.loadAllFromDisk();
-      this.loaded = true;
+  async removeById(id, requireStatus) {
+    if (!this.store) return;
+    try {
+      const row = await this.store.getById(id);
+      if (!row) return;
+      if (requireStatus && row.status !== requireStatus) return;
+      await this.store.remove(id);
+      this.unlinkBody(row.fileBase);
+      logger.info(`[ScriptStore] Removed "${row.name}" (${id})`);
+    } catch (err) {
+      logger.warn(`[ScriptStore] remove(${id}) failed: ${err}`);
     }
   }
-  /**
-   * Convert a script name to a safe filename.
-   * "Order Status Distribution" → "order-status-distribution"
-   */
-  toFileName(name) {
-    return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "unnamed-script";
+  rowToRecipe(row, body) {
+    return {
+      id: row.id,
+      version: row.version ?? 1,
+      name: row.name,
+      intentDescription: row.intentDescription,
+      tags: row.tags ?? [],
+      sourceIds: row.sourceIds ?? [],
+      tables: row.tables ?? [],
+      parameters: row.parameters ?? [],
+      scriptBody: body,
+      fileBase: row.fileBase,
+      bodyHash: row.bodyHash ?? void 0,
+      projectId: row.projectId ?? void 0,
+      successCount: row.successCount ?? 0,
+      failureCount: row.failureCount ?? 0,
+      lastUsed: row.lastUsed ?? (/* @__PURE__ */ new Date()).toISOString(),
+      createdFrom: row.createdFrom ?? "",
+      createdAt: row.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+      updatedAt: row.updatedAt ?? (/* @__PURE__ */ new Date()).toISOString(),
+      parentId: row.parentId ?? void 0,
+      forkDepth: typeof row.forkDepth === "number" ? row.forkDepth : 0,
+      forkReason: row.forkReason ?? void 0,
+      components: row.components ?? void 0,
+      status: row.status ?? "verified",
+      turnId: row.turnId ?? void 0,
+      lastError: row.lastError ?? void 0
+    };
   }
-  /**
-   * Load all script files from disk.
-   *
-   * Supports two layouts:
-   *   - New (preferred): metadata/<name>.json + <name>.ts
-   *   - Legacy: <name>.json with embedded scriptBody (auto-migrated on next save)
-   */
-  loadAllFromDisk() {
-    try {
-      if (!fs6.existsSync(this.storeDir)) return;
-      const metaDir = this.metadataDir();
-      const metaExists = fs6.existsSync(metaDir);
-      if (metaExists) {
-        const metaFiles = fs6.readdirSync(metaDir).filter((f) => f.endsWith(".json"));
-        for (const file of metaFiles) {
-          try {
-            const metaPath = path6.join(metaDir, file);
-            const meta = JSON.parse(fs6.readFileSync(metaPath, "utf-8"));
-            if (!meta.id) continue;
-            const baseName = file.replace(/\.json$/, "");
-            const bodyPath = path6.join(this.storeDir, `${baseName}.ts`);
-            if (!fs6.existsSync(bodyPath)) {
-              logger.warn(`[ScriptStore] Metadata ${file} has no sibling ${baseName}.ts \u2014 skipping`);
-              continue;
-            }
-            meta.scriptBody = fs6.readFileSync(bodyPath, "utf-8");
-            if (typeof meta.forkDepth !== "number") meta.forkDepth = 0;
-            if (!meta.status) meta.status = "verified";
-            this.recipes.set(meta.id, meta);
-          } catch (err) {
-            logger.warn(`[ScriptStore] Failed to load metadata ${file}: ${err}`);
-          }
-        }
-      }
-      const topLevelJson = fs6.readdirSync(this.storeDir).filter((f) => f.endsWith(".json"));
-      for (const file of topLevelJson) {
-        try {
-          const filePath = path6.join(this.storeDir, file);
-          const recipe = JSON.parse(fs6.readFileSync(filePath, "utf-8"));
-          if (!recipe.id || !recipe.scriptBody) continue;
-          if (this.recipes.has(recipe.id)) continue;
-          if (typeof recipe.forkDepth !== "number") recipe.forkDepth = 0;
-          if (!recipe.status) recipe.status = "verified";
-          this.recipes.set(recipe.id, recipe);
-        } catch (err) {
-          logger.warn(`[ScriptStore] Failed to load legacy script ${file}: ${err}`);
-        }
-      }
-      if (this.recipes.size > 0) {
-        logger.info(`[ScriptStore] Loaded ${this.recipes.size} scripts from ${this.storeDir}`);
+  recipeToRow(recipe) {
+    return {
+      id: recipe.id,
+      projectId: recipe.projectId ?? this.projectId ?? null,
+      version: recipe.version ?? 1,
+      name: recipe.name,
+      intentDescription: recipe.intentDescription,
+      tags: recipe.tags ?? null,
+      createdFrom: recipe.createdFrom ?? null,
+      sourceIds: recipe.sourceIds ?? null,
+      tables: recipe.tables ?? null,
+      parameters: recipe.parameters ?? null,
+      components: recipe.components ?? null,
+      fileBase: recipe.fileBase || this.toSlug(recipe.name),
+      bodyHash: recipe.bodyHash ?? null,
+      successCount: recipe.successCount ?? 0,
+      failureCount: recipe.failureCount ?? 0,
+      lastUsed: recipe.lastUsed ?? null,
+      parentId: recipe.parentId ?? null,
+      forkDepth: typeof recipe.forkDepth === "number" ? recipe.forkDepth : 0,
+      forkReason: recipe.forkReason ?? null,
+      status: recipe.status ?? "verified",
+      turnId: recipe.turnId ?? null,
+      lastError: recipe.lastError ?? null
+    };
+  }
+  /** slug of name, with a short id suffix when the bare slug is already taken. */
+  async computeFileBase(name, id) {
+    const slug = this.toSlug(name);
+    if (this.store) {
+      try {
+        const taken = await this.store.fileBaseTaken(slug, id, this.projectId);
+        if (taken) return `${slug}-${id.slice(-6)}`;
+      } catch {
       }
-    } catch (err) {
-      logger.warn(`[ScriptStore] Failed to read store directory ${this.storeDir}: ${err}`);
     }
+    return slug;
   }
-  /**
-   * Save a recipe to disk in split format:
-   *   metadata/<name>.json  (metadata, no scriptBody)
-   *   <name>.ts             (just the function body)
-   *
-   * If a legacy top-level <name>.json exists for the same recipe, remove it
-   * so we don't end up with duplicate sources of truth.
-   */
-  saveRecipeToDisk(recipe) {
+  toSlug(name) {
+    return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "unnamed-script";
+  }
+  hash(body) {
+    return createHash("sha256").update(body).digest("hex");
+  }
+  bodyPath(fileBase) {
+    return path6.join(this.storeDir, `${fileBase}.ts`);
+  }
+  readBody(fileBase) {
     try {
-      const metaDir = this.metadataDir();
-      if (!fs6.existsSync(metaDir)) {
-        fs6.mkdirSync(metaDir, { recursive: true });
-      }
-      const baseName = this.fileBaseName(recipe);
-      const metaPath = path6.join(metaDir, `${baseName}.json`);
-      const bodyPath = path6.join(this.storeDir, `${baseName}.ts`);
-      const { scriptBody, ...meta } = recipe;
-      fs6.writeFileSync(metaPath, JSON.stringify(meta, null, 2));
-      fs6.writeFileSync(bodyPath, normalizeScriptBody(scriptBody));
-      const legacyPath = path6.join(this.storeDir, `${this.toFileName(recipe.name)}.json`);
-      if (fs6.existsSync(legacyPath)) {
-        try {
-          fs6.unlinkSync(legacyPath);
-        } catch {
-        }
-      }
+      return fs6.readFileSync(this.bodyPath(fileBase), "utf-8");
     } catch (err) {
-      logger.warn(`[ScriptStore] Failed to save script "${recipe.name}": ${err}`);
+      logger.warn(`[ScriptStore] body file ${fileBase}.ts missing/unreadable: ${err}`);
+      return "";
     }
   }
-  /**
-   * Delete a recipe's files from disk (both metadata + body, plus any legacy file).
-   */
-  deleteRecipeFromDisk(recipe) {
-    const baseName = this.fileBaseName(recipe);
-    const slug = this.toFileName(recipe.name);
-    const candidates = [
-      path6.join(this.metadataDir(), `${baseName}.json`),
-      path6.join(this.storeDir, `${baseName}.ts`),
-      path6.join(this.storeDir, `${slug}.json`)
-      // legacy single-file format
-    ];
-    for (const p of candidates) {
-      try {
-        if (fs6.existsSync(p)) fs6.unlinkSync(p);
-      } catch (err) {
-        logger.warn(`[ScriptStore] Failed to delete ${p}: ${err}`);
-      }
+  /** Atomic body write (temp + rename) so concurrent reads never see a partial file. */
+  writeBody(fileBase, normalizedBody) {
+    if (!fs6.existsSync(this.storeDir)) fs6.mkdirSync(this.storeDir, { recursive: true });
+    const finalPath = this.bodyPath(fileBase);
+    const tmpPath = `${finalPath}.tmp-${randomBytes(4).toString("hex")}`;
+    fs6.writeFileSync(tmpPath, normalizedBody);
+    fs6.renameSync(tmpPath, finalPath);
+  }
+  unlinkBody(fileBase) {
+    try {
+      const p = this.bodyPath(fileBase);
+      if (fs6.existsSync(p)) fs6.unlinkSync(p);
+    } catch (err) {
+      logger.warn(`[ScriptStore] failed to delete body ${fileBase}.ts: ${err}`);
     }
   }
 };
@@ -9744,14 +10126,14 @@ var ScriptMatcher = class {
    * Returns null if no script matches.
    */
   async match(userPrompt, apiKey, model) {
-    const recipes = this.store.getAll();
+    const recipes = await this.store.search(userPrompt);
     if (recipes.length === 0) return null;
     const healthyRecipes = recipes.filter((r) => {
       const total = r.successCount + r.failureCount;
       return total < 5 || r.failureCount / total <= MAX_FAILURE_RATE;
     });
     if (healthyRecipes.length === 0) {
-      logger.info(`[ScriptMatcher] All ${recipes.length} scripts have high failure rates \u2014 skipping`);
+      logger.info(`[ScriptMatcher] No healthy candidates among ${recipes.length} FTS matches \u2014 skipping`);
       return null;
     }
     const scriptCatalog = this.buildScriptCatalog(healthyRecipes);
@@ -9787,7 +10169,7 @@ var ScriptMatcher = class {
         logger.info(`[ScriptMatcher] No match. Reason: ${reasoning}`);
         return null;
       }
-      const recipe = this.store.get(response.scriptId);
+      const recipe = await this.store.get(response.scriptId);
       if (!recipe) {
         logger.warn(`[ScriptMatcher] LLM returned unknown script ID: "${response.scriptId}"`);
         return null;
@@ -9810,61 +10192,360 @@ var ScriptMatcher = class {
           reasoning
         };
       }
-      logger.info(`[ScriptMatcher] Matched "${recipe.name}" (tier: high). Reason: ${reasoning}`);
-      return {
-        recipe,
-        tier: "high",
-        similarity: 0.95,
-        confidence: "high",
-        extractedParams: response.params || {},
-        reasoning
-      };
-    } catch (error) {
-      const msg = error instanceof Error ? error.message : String(error);
-      logger.error(`[ScriptMatcher] LLM matching failed: ${msg}`);
-      return null;
-    }
+      logger.info(`[ScriptMatcher] Matched "${recipe.name}" (tier: high). Reason: ${reasoning}`);
+      return {
+        recipe,
+        tier: "high",
+        similarity: 0.95,
+        confidence: "high",
+        extractedParams: response.params || {},
+        reasoning
+      };
+    } catch (error) {
+      const msg = error instanceof Error ? error.message : String(error);
+      logger.error(`[ScriptMatcher] LLM matching failed: ${msg}`);
+      return null;
+    }
+  }
+  /**
+   * Build the script catalog string for the LLM prompt.
+   * Each script gets: index, ID, name, description, and parameter definitions.
+   */
+  buildScriptCatalog(recipes) {
+    return recipes.map((r, idx) => {
+      const paramList = r.parameters.length > 0 ? r.parameters.map((p) => {
+        let desc = `${p.name} (${p.type}`;
+        if (!p.required) desc += ", optional";
+        if (p.default !== void 0) desc += `, default: ${JSON.stringify(p.default)}`;
+        if (p.enumValues) desc += `, values: ${Object.keys(p.enumValues).join("/")}`;
+        desc += `) \u2014 ${p.description}`;
+        return `    - ${desc}`;
+      }).join("\n") : "    (no parameters)";
+      return `${idx + 1}. [${r.id}] "${r.name}"
+   ${r.intentDescription}
+   Parameters:
+${paramList}`;
+    }).join("\n\n");
+  }
+};
+
+// src/userResponse/scripts/script-component-generator.ts
+function inferColumns(rows) {
+  const cols = /* @__PURE__ */ new Map();
+  if (!Array.isArray(rows) || rows.length === 0) return cols;
+  const sample = rows.slice(0, 20);
+  const keys = /* @__PURE__ */ new Set();
+  for (const r of sample) {
+    if (r && typeof r === "object") for (const k of Object.keys(r)) keys.add(k);
+  }
+  for (const key of keys) {
+    let type = "TEXT";
+    let typed = false;
+    const distinct = /* @__PURE__ */ new Set();
+    for (const r of sample) {
+      const v = r?.[key];
+      if (v === null || v === void 0) continue;
+      if (!typed) {
+        if (typeof v === "number") type = "NUMBER";
+        else if (typeof v === "boolean") type = "BOOLEAN";
+        else if (v instanceof Date) type = "DATE";
+        else if (typeof v === "string" && v.trim() !== "" && !isNaN(Number(v))) type = "NUMBER";
+        else type = "TEXT";
+        typed = true;
+      }
+      distinct.add(v instanceof Date ? v.toISOString() : String(v));
+    }
+    cols.set(key.toLowerCase(), { name: key, type, invariant: distinct.size <= 1 });
+  }
+  return cols;
+}
+function resolveCol(cols, key) {
+  if (key == null) return void 0;
+  return cols.get(String(key).toLowerCase());
+}
+function firstOfType(cols, type) {
+  for (const c of cols.values()) if (c.type === type) return c;
+  return void 0;
+}
+var ALL_KEY_NAMES = ["xAxisKey", "yAxisKey", "valueKey", "nameKey", "seriesKey", "groupBy", "aggregationField", "categoryKey"];
+var AVG_KEYWORDS = /\b(average|avg|mean)\b/i;
+var COUNT_KEYWORDS = /\b(count|number of|how many|# of|no\.? of)\b/i;
+var SUM_KEYWORDS = /\b(total|sum|overall|combined|aggregate|all)\b/i;
+function validateAndRepairComponent(comp, componentType, cols, rowCount) {
+  if (cols.size === 0) return comp;
+  const props = comp.props || (comp.props = {});
+  const cfg = props.config || (props.config = {});
+  for (const k of ALL_KEY_NAMES) {
+    if (cfg[k] == null) continue;
+    const r = resolveCol(cols, cfg[k]);
+    if (r) cfg[k] = r.name;
+    else delete cfg[k];
+  }
+  if (Array.isArray(cfg.metricKeys)) {
+    cfg.metricKeys = cfg.metricKeys.map((m) => resolveCol(cols, m)?.name).filter(Boolean);
+    if (cfg.metricKeys.length === 0) delete cfg.metricKeys;
+  }
+  const numeric = () => firstOfType(cols, "NUMBER");
+  const category = () => firstOfType(cols, "TEXT") || firstOfType(cols, "DATE");
+  const hasMetrics = Array.isArray(cfg.metricKeys) && cfg.metricKeys.length > 0;
+  switch (componentType) {
+    case "KPICard":
+    case "GaugeChart": {
+      const vk = resolveCol(cols, cfg.valueKey) || numeric() || category();
+      if (!vk) return null;
+      cfg.valueKey = vk.name;
+      const title = String(props.title || "");
+      if (rowCount > 1 && !cfg.aggregation) {
+        if (COUNT_KEYWORDS.test(title)) cfg.aggregation = "count";
+        else if (!vk.invariant) {
+          if (AVG_KEYWORDS.test(title)) cfg.aggregation = "avg";
+          else if (SUM_KEYWORDS.test(title)) cfg.aggregation = "sum";
+        }
+      }
+      if (cfg.aggregation === "sum" || cfg.aggregation === "avg") {
+        const af = resolveCol(cols, cfg.aggregationField);
+        if (af && af.type === "NUMBER") {
+          cfg.aggregationField = af.name;
+        } else {
+          const n = vk.type === "NUMBER" ? vk : numeric();
+          if (n) cfg.aggregationField = n.name;
+          else {
+            delete cfg.aggregation;
+            delete cfg.aggregationField;
+          }
+        }
+      }
+      return comp;
+    }
+    case "PieChart":
+    case "TreemapChart": {
+      let nameC = resolveCol(cols, cfg.nameKey);
+      let valC = resolveCol(cols, cfg.valueKey);
+      if (nameC?.type === "NUMBER" && valC && valC.type !== "NUMBER") {
+        const t = nameC;
+        nameC = valC;
+        valC = t;
+      }
+      if (!nameC) nameC = category();
+      if (!valC || valC.type !== "NUMBER") valC = numeric() || valC;
+      if (!nameC || !valC) return null;
+      cfg.nameKey = nameC.name;
+      cfg.valueKey = valC.name;
+      return comp;
+    }
+    case "WaterfallChart": {
+      const catC = resolveCol(cols, cfg.categoryKey) || category();
+      let valC = resolveCol(cols, cfg.valueKey);
+      if (!valC || valC.type !== "NUMBER") valC = numeric() || valC;
+      if (!catC || !valC) return null;
+      cfg.categoryKey = catC.name;
+      cfg.valueKey = valC.name;
+      return comp;
+    }
+    case "ScatterChart": {
+      let xC = resolveCol(cols, cfg.xAxisKey);
+      let yC = resolveCol(cols, cfg.yAxisKey);
+      if (!xC || xC.type !== "NUMBER") xC = numeric() || xC;
+      if (!yC || yC.type !== "NUMBER") {
+        yC = [...cols.values()].find((c) => c.type === "NUMBER" && c.name !== xC?.name) || numeric() || yC;
+      }
+      if (!xC || !yC) return null;
+      cfg.xAxisKey = xC.name;
+      cfg.yAxisKey = yC.name;
+      return comp;
+    }
+    case "HeatmapChart": {
+      const xC = resolveCol(cols, cfg.xAxisKey) || category();
+      const yC = resolveCol(cols, cfg.yAxisKey) || [...cols.values()].find((c) => (c.type === "TEXT" || c.type === "DATE") && c.name !== xC?.name);
+      let valC = resolveCol(cols, cfg.valueKey);
+      if (!valC || valC.type !== "NUMBER") valC = numeric() || valC;
+      if (!xC || !yC || !valC) return null;
+      cfg.xAxisKey = xC.name;
+      cfg.yAxisKey = yC.name;
+      cfg.valueKey = valC.name;
+      return comp;
+    }
+    case "BarChart":
+    case "LineChart": {
+      let xC = resolveCol(cols, cfg.xAxisKey);
+      let yC = resolveCol(cols, cfg.yAxisKey);
+      if (!hasMetrics && xC?.type === "NUMBER" && yC && yC.type !== "NUMBER") {
+        const t = xC;
+        xC = yC;
+        yC = t;
+      }
+      if (!xC) xC = (componentType === "LineChart" ? firstOfType(cols, "DATE") || category() : category()) || firstOfType(cols, "NUMBER");
+      if (!xC) return null;
+      cfg.xAxisKey = xC.name;
+      if (!hasMetrics) {
+        if (!yC || yC.type !== "NUMBER") yC = numeric();
+        if (!yC) return null;
+        cfg.yAxisKey = yC.name;
+      }
+      if (cfg.seriesKey) {
+        const sC = resolveCol(cols, cfg.seriesKey);
+        if (!sC || sC.type !== "TEXT" || sC.name === cfg.xAxisKey) delete cfg.seriesKey;
+        else cfg.seriesKey = sC.name;
+      }
+      return comp;
+    }
+    case "RadarChart": {
+      if (!hasMetrics) {
+        const vk = resolveCol(cols, cfg.valueKey) || numeric();
+        if (!vk) return null;
+        cfg.valueKey = vk.name;
+      }
+      return comp;
+    }
+    default:
+      return comp;
+  }
+}
+function scriptCompKey(comp) {
+  const id = comp?.componentId || comp?.componentName || "";
+  const src = comp?.sourceIndex ?? "";
+  return `${id}|${src}|${JSON.stringify(comp?.props?.config ?? comp?.config ?? {})}`;
+}
+function extractAnswerComponentObject(text) {
+  const hasMatch = text.match(/"hasAnswerComponent"\s*:\s*(true|false)/);
+  if (!hasMatch || hasMatch[1] !== "true") return null;
+  const startMatch = text.match(/"answerComponent"\s*:\s*\{/);
+  if (!startMatch) return null;
+  const startPos = startMatch.index + startMatch[0].length - 1;
+  let depth = 0, inString = false, escapeNext = false, endPos = -1;
+  for (let i = startPos; i < text.length; i++) {
+    const c = text[i];
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+    if (c === "\\") {
+      escapeNext = true;
+      continue;
+    }
+    if (c === '"') {
+      inString = !inString;
+      continue;
+    }
+    if (!inString) {
+      if (c === "{") depth++;
+      else if (c === "}") {
+        depth--;
+        if (depth === 0) {
+          endPos = i + 1;
+          break;
+        }
+      }
+    }
+  }
+  if (endPos <= startPos) return null;
+  try {
+    return JSON.parse(text.substring(startPos, endPos));
+  } catch {
+    return null;
+  }
+}
+async function buildScriptComponent(comp, ctx) {
+  const { queries, queryIds, columnsByIndex, availableComponents } = ctx;
+  let validationCols = /* @__PURE__ */ new Map();
+  let validationRowCount = 0;
+  let specSourceRef = "";
+  const wantedId = comp.componentId || "";
+  const wantedName = comp.componentName || "";
+  const template = availableComponents.find(
+    (c) => c.id === wantedId || c.name === wantedId || c.id === wantedName || c.name === wantedName
+  );
+  if (!template) {
+    logger.warn(`[ScriptComponentGen] Component "${wantedId || wantedName}" not found \u2014 skipping`);
+    return null;
+  }
+  if (template.type === "MarkdownBlock") {
+    const p = comp.props || {};
+    const content = p.content ?? p.config?.content ?? p.config?.markdown ?? p.config?.text ?? "";
+    return {
+      component: {
+        id: `comp_${Math.random().toString(36).substring(2, 8)}`,
+        name: template.name,
+        displayName: template.displayName,
+        type: template.type,
+        description: template.description,
+        props: { title: p.title, description: p.description, content },
+        category: template.category,
+        keywords: template.keywords
+      },
+      // Persist a content-bearing spec so the markdown survives deterministic
+      // replay (it has no data source, so its text must be stored — otherwise
+      // re-running a matched script rebuilds only the source-bound components
+      // and silently drops the narrative).
+      spec: {
+        componentType: template.name,
+        sourceRef: "markdown",
+        title: p.title,
+        description: p.description,
+        content,
+        config: {}
+      }
+    };
   }
-  /**
-   * Build the script catalog string for the LLM prompt.
-   * Each script gets: index, ID, name, description, and parameter definitions.
-   */
-  buildScriptCatalog(recipes) {
-    return recipes.map((r, idx) => {
-      const paramList = r.parameters.length > 0 ? r.parameters.map((p) => {
-        let desc = `${p.name} (${p.type}`;
-        if (!p.required) desc += ", optional";
-        if (p.default !== void 0) desc += `, default: ${JSON.stringify(p.default)}`;
-        if (p.enumValues) desc += `, values: ${Object.keys(p.enumValues).join("/")}`;
-        desc += `) \u2014 ${p.description}`;
-        return `    - ${desc}`;
-      }).join("\n") : "    (no parameters)";
-      return `${idx + 1}. [${r.id}] "${r.name}"
-   ${r.intentDescription}
-   Parameters:
-${paramList}`;
-    }).join("\n\n");
+  const sourceIdx = typeof comp.sourceIndex === "number" ? comp.sourceIndex : 0;
+  const targetQuery = queries[sourceIdx] || queries[0];
+  const targetQueryId = queryIds[targetQuery.sourceId];
+  if (!targetQueryId) {
+    logger.warn(`[ScriptComponentGen] No queryId for source "${targetQuery.sourceName}" \u2014 skipping`);
+    return null;
   }
-};
-
-// src/userResponse/scripts/script-component-generator.ts
+  validationCols = columnsByIndex[sourceIdx] || inferColumns(targetQuery.data);
+  validationRowCount = targetQuery.count ?? (targetQuery.data?.length || 0);
+  specSourceRef = targetQuery.sourceId;
+  const renderedToolId = targetQuery.virtual ? "script_dataset" : targetQuery.sourceId;
+  const externalToolProp = { toolId: renderedToolId, toolName: targetQuery.sourceName, parameters: { queryId: targetQueryId } };
+  const repaired = validateAndRepairComponent(comp, template.type, validationCols, validationRowCount);
+  if (!repaired) {
+    logger.warn(`[ScriptComponentGen] Component "${template.name}" dropped \u2014 config keys could not be mapped to the data columns`);
+    return null;
+  }
+  if (template.type === "DataTable" && Array.isArray(comp.props?.config?.columns)) {
+    comp.props.config.columns = comp.props.config.columns.map(
+      (c) => typeof c === "string" ? { key: c, label: c.replace(/[_-]+/g, " ").replace(/\b\w/g, (m) => m.toUpperCase()) } : c
+    );
+  }
+  const component = {
+    id: `comp_${Math.random().toString(36).substring(2, 8)}`,
+    name: template.name,
+    displayName: template.displayName,
+    type: template.type,
+    description: template.description,
+    props: { ...comp.props, externalTool: externalToolProp },
+    category: template.category,
+    keywords: template.keywords
+  };
+  const spec = specSourceRef ? {
+    componentType: template.name,
+    sourceRef: specSourceRef,
+    title: comp.props?.title,
+    description: comp.props?.description,
+    config: { ...comp.props?.config || {} }
+  } : void 0;
+  return { component, spec };
+}
 async function generateScriptComponents(params) {
-  const { userPrompt, scriptResult, queryIds, availableComponents, apiKey, model, componentStreamCallback, externalTools, collections } = params;
+  const { userPrompt, scriptResult, queryIds, availableComponents, apiKey, model, componentStreamCallback, analysisText } = params;
   const queries = scriptResult.executedQueries;
   if (!queries || queries.length === 0) {
     logger.warn(`[ScriptComponentGen] No query data available \u2014 returning empty`);
-    return { components: [], layoutTitle: "", layoutDescription: "", actions: [] };
+    return { components: [], layoutTitle: "", layoutDescription: "", actions: [], componentSpecs: [] };
   }
   logger.info(`[ScriptComponentGen] Starting | ${queries.length} queries | ${availableComponents.length} available components`);
-  const availableCompsText = availableComponents.filter((c) => c.name !== "DynamicMarkdownBlock").map((c) => `- ${c.name} (${c.type}): ${c.description || ""}`).join("\n");
+  const columnsByIndex = queries.map((q) => {
+    const data = q.data && q.data.length ? q.data : q.virtual ? scriptResult.data : q.data;
+    return inferColumns(data || []);
+  });
+  const availableCompsText = availableComponents.map((c) => `- ${c.name} (${c.type}): ${c.description || ""}`).join("\n");
   const sourceDescriptions = queries.map((q, idx) => {
     if (!q.data || q.data.length === 0) return null;
-    const sampleRow = q.data[0];
-    const columns = Object.entries(sampleRow).map(([key, value]) => {
-      const type = typeof value === "number" ? "NUMBER" : value instanceof Date ? "DATE" : typeof value === "boolean" ? "BOOLEAN" : "TEXT";
-      return `${key} (${type})`;
-    }).join(", ");
-    const sampleData = JSON.stringify(q.data.slice(0, 3), null, 2);
+    const cols = columnsByIndex[idx];
+    const columns = cols && cols.size > 0 ? Array.from(cols.values()).map((c) => `${c.name} (${c.type})`).join(", ") : Object.keys(q.data[0]).join(", ");
+    const sampleData = JSON.stringify(q.data.slice(0, TOOL_TRACKING_SAMPLE_ROWS), null, 2);
     return `### Source ${idx}: "${q.sourceName}" (sourceIndex: ${idx})
 - Rows: ${q.count}${q.totalCount && q.totalCount > q.count ? ` of ${q.totalCount} total` : ""}
 - Columns: ${columns}
@@ -9873,19 +10554,6 @@ async function generateScriptComponents(params) {
 ${sampleData}
 \`\`\``;
   }).filter(Boolean).join("\n\n");
-  const isMultiSource = queries.length > 1;
-  const federationTool = externalTools?.find((t) => t.id === "federation_query");
-  let federationContext = "";
-  if (isMultiSource && federationTool) {
-    federationContext = `### Cross-Source Federation Available
-You can create components that JOIN data from multiple sources using DuckDB SQL.
-Set sourceIndex to "federation" and provide federationSql.
-${federationTool.fullSchema || federationTool.description || "Use schema-qualified table names."}`;
-  } else if (isMultiSource) {
-    federationContext = `### Multiple Sources Available
-Each component can use data from one source (specify sourceIndex: 0, 1, etc.).
-Cross-source JOINs are not available for this project.`;
-  }
   try {
     const prompts = await promptLoader.loadPrompts("script-components", {
       USER_PROMPT: userPrompt,
@@ -9894,95 +10562,98 @@ Cross-source JOINs are not available for this project.`;
       ROW_COUNT: queries.map((q) => `${q.sourceName}: ${q.count}`).join(", "),
       COLUMN_DESCRIPTIONS: sourceDescriptions,
       SAMPLE_DATA: "",
-      FEDERATION_CONTEXT: federationContext
+      ANALYSIS_TEXT: analysisText && analysisText.trim() ? analysisText.trim() : "No analysis text provided \u2014 infer intent from the user question and the returned data."
     });
+    const ctx = {
+      queries,
+      queryIds,
+      columnsByIndex,
+      availableComponents
+    };
+    let fullText = "";
+    let answerAttempted = false;
+    let streamedAnswerKey = null;
+    const partial = componentStreamCallback ? (chunk) => {
+      fullText += chunk;
+      if (answerAttempted) return;
+      const answerComp = extractAnswerComponentObject(fullText);
+      if (!answerComp) return;
+      answerAttempted = true;
+      streamedAnswerKey = scriptCompKey(answerComp);
+      buildScriptComponent(answerComp, ctx).then((built) => {
+        if (built) {
+          componentStreamCallback(built.component);
+          logger.info("[ScriptComponentGen] Streamed answer component early");
+        } else {
+          streamedAnswerKey = null;
+        }
+      }).catch(() => {
+        streamedAnswerKey = null;
+      });
+    } : void 0;
     const response = await LLM.stream(
       { sys: prompts.system, user: prompts.user },
       {
         model: model || void 0,
-        maxTokens: 1500,
+        maxTokens: 5e3,
         temperature: 0,
-        apiKey
+        apiKey,
+        partial
       },
       true
     );
     if (!response || !response.components || !Array.isArray(response.components)) {
       logger.warn(`[ScriptComponentGen] LLM returned invalid response`);
-      return { components: [], layoutTitle: "", layoutDescription: "", actions: [] };
+      return { components: [], layoutTitle: "", layoutDescription: "", actions: [], componentSpecs: [] };
     }
     const layoutTitle = response.layoutTitle || "Dashboard";
     const layoutDescription = response.layoutDescription || "";
     const matchedComponents = [];
+    const componentSpecs = [];
     for (const comp of response.components) {
-      const wantedId = comp.componentId || "";
-      const wantedName = comp.componentName || "";
-      const template = availableComponents.find(
-        (c) => c.id === wantedId || c.name === wantedId || c.id === wantedName || c.name === wantedName
-      );
-      if (!template) {
-        logger.warn(`[ScriptComponentGen] Component "${wantedId || wantedName}" not found \u2014 skipping`);
-        continue;
-      }
-      let externalToolProp;
-      if (comp.sourceIndex === "federation" && comp.federationSql) {
-        if (!isMultiSource || !federationTool) {
-          logger.warn(
-            `[ScriptComponentGen] Component "${comp.componentName || comp.componentId}" requested federation, but ${!isMultiSource ? "only one source query exists" : "no federation_query tool is registered"} \u2014 falling back to single-source. Computed columns from script's JS post-processing will not appear in this component.`
-          );
-          comp.sourceIndex = 0;
-          delete comp.federationSql;
-        } else {
-          logger.info(`[ScriptComponentGen] Federation component \u2014 executing cross-source SQL`);
-          try {
-            const fedResult = await executeFederationQuery(comp.federationSql, externalTools, collections);
-            if (fedResult) {
-              const fedQueryId = queryCache.storeQuery(comp.federationSql, fedResult);
-              externalToolProp = {
-                toolId: "federation_query",
-                toolName: "Cross-Source Query",
-                parameters: { queryId: fedQueryId }
-              };
-            } else {
-              logger.warn(`[ScriptComponentGen] Federation query failed \u2014 skipping component`);
-              continue;
-            }
-          } catch (fedErr) {
-            logger.warn(`[ScriptComponentGen] Federation query error: ${fedErr} \u2014 skipping component`);
-            continue;
-          }
+      const built = await buildScriptComponent(comp, ctx);
+      if (!built) continue;
+      matchedComponents.push(built.component);
+      if (built.spec) componentSpecs.push(built.spec);
+      if (componentStreamCallback && scriptCompKey(comp) !== streamedAnswerKey) {
+        componentStreamCallback(built.component);
+      }
+    }
+    if (response.hasAnswerComponent && response.answerComponent) {
+      const aKey = scriptCompKey(response.answerComponent);
+      const alreadyIncluded = response.components.some((c) => scriptCompKey(c) === aKey);
+      if (!alreadyIncluded) {
+        const built = await buildScriptComponent(response.answerComponent, ctx);
+        if (built) {
+          matchedComponents.unshift(built.component);
+          if (built.spec) componentSpecs.unshift(built.spec);
+          if (componentStreamCallback && aKey !== streamedAnswerKey) componentStreamCallback(built.component);
         }
       }
-      if (!externalToolProp) {
-        const sourceIdx = typeof comp.sourceIndex === "number" ? comp.sourceIndex : 0;
-        const targetQuery = queries[sourceIdx] || queries[0];
-        const targetQueryId = queryIds[targetQuery.sourceId];
-        if (!targetQueryId) {
-          logger.warn(`[ScriptComponentGen] No queryId for source "${targetQuery.sourceName}" \u2014 skipping`);
-          continue;
-        }
-        const renderedToolId = targetQuery.virtual ? "script_dataset" : targetQuery.sourceId;
-        externalToolProp = {
-          toolId: renderedToolId,
-          toolName: targetQuery.sourceName,
-          parameters: { queryId: targetQueryId }
+    }
+    const hasMarkdown = matchedComponents.some((c) => c.type === "MarkdownBlock");
+    if (!hasMarkdown && analysisText && analysisText.trim()) {
+      const mdTemplate = availableComponents.find((c) => c.type === "MarkdownBlock" || c.name === "DynamicMarkdownBlock");
+      if (mdTemplate) {
+        const mdComponent = {
+          id: `comp_${Math.random().toString(36).substring(2, 8)}`,
+          name: mdTemplate.name,
+          displayName: mdTemplate.displayName,
+          type: mdTemplate.type,
+          description: mdTemplate.description,
+          props: { content: analysisText.trim() },
+          category: mdTemplate.category,
+          keywords: mdTemplate.keywords
         };
-      }
-      const component = {
-        id: `comp_${Math.random().toString(36).substring(2, 8)}`,
-        name: template.name,
-        displayName: template.displayName,
-        type: template.type,
-        description: template.description,
-        props: {
-          ...comp.props,
-          externalTool: externalToolProp
-        },
-        category: template.category,
-        keywords: template.keywords
-      };
-      matchedComponents.push(component);
-      if (componentStreamCallback) {
-        componentStreamCallback(component);
+        matchedComponents.push(mdComponent);
+        componentSpecs.push({
+          componentType: mdTemplate.name,
+          sourceRef: "markdown",
+          content: analysisText.trim(),
+          config: {}
+        });
+        if (componentStreamCallback) componentStreamCallback(mdComponent);
+        logger.info("[ScriptComponentGen] Injected fallback DynamicMarkdownBlock (LLM omitted one)");
       }
     }
     logger.info(`[ScriptComponentGen] Generated ${matchedComponents.length} components`);
@@ -9996,37 +10667,97 @@ Cross-source JOINs are not available for this project.`;
       components: matchedComponents,
       layoutTitle,
       layoutDescription,
-      actions
+      actions,
+      componentSpecs
     };
   } catch (error) {
     const msg = error instanceof Error ? error.message : String(error);
     logger.error(`[ScriptComponentGen] Failed: ${msg}`);
-    return { components: [], layoutTitle: "", layoutDescription: "", actions: [] };
+    return { components: [], layoutTitle: "", layoutDescription: "", actions: [], componentSpecs: [] };
   }
 }
-async function executeFederationQuery(sql, externalTools, collections) {
-  const fedTool = externalTools?.find((t) => t.id === "federation_query");
-  if (fedTool) {
-    try {
-      const result = await fedTool.fn({ sql });
-      return result;
-    } catch (err) {
-      logger.warn(`[ScriptComponentGen] Federation tool execution failed: ${err}`);
+function assembleComponentsFromSpecs(params) {
+  const { specs, scriptResult, queryIds, availableComponents, componentStreamCallback, analysisText } = params;
+  if (!specs || specs.length === 0) return null;
+  const queries = scriptResult.executedQueries || [];
+  const byId = new Map(queries.map((q) => [q.sourceId, q]));
+  const out = [];
+  let answerComponentStreamed = false;
+  const mdTemplate = availableComponents.find((c) => c.type === "MarkdownBlock" || c.name === "DynamicMarkdownBlock");
+  const buildMarkdown = (content, title, description) => {
+    if (!mdTemplate || !content || !content.trim()) return null;
+    return {
+      id: `comp_${Math.random().toString(36).substring(2, 8)}`,
+      name: mdTemplate.name,
+      displayName: mdTemplate.displayName,
+      type: mdTemplate.type,
+      description: mdTemplate.description,
+      props: { title, description, content: content.trim() },
+      category: mdTemplate.category,
+      keywords: mdTemplate.keywords
+    };
+  };
+  for (const spec of specs) {
+    if (spec.sourceRef === "federation") {
+      logger.info("[ScriptComponentGen] Recipe has a federation spec \u2014 not replayable deterministically, falling back to LLM");
+      return null;
+    }
+    if (spec.sourceRef === "markdown") {
+      const md = buildMarkdown(spec.content || analysisText || "", spec.title, spec.description);
+      if (md) {
+        out.push(md);
+        if (componentStreamCallback) componentStreamCallback(md);
+      }
+      continue;
+    }
+    const q = byId.get(spec.sourceRef) || queries.find((x) => x.virtual);
+    if (!q || !q.data || q.data.length === 0) {
+      logger.info(`[ScriptComponentGen] Recipe spec source "${spec.sourceRef}" missing/empty on replay \u2014 falling back to LLM`);
+      return null;
+    }
+    const template = availableComponents.find((c) => c.name === spec.componentType || c.id === spec.componentType);
+    if (!template) {
+      logger.info(`[ScriptComponentGen] Recipe spec component "${spec.componentType}" not in library \u2014 falling back to LLM`);
+      return null;
+    }
+    const cols = inferColumns(q.data);
+    const comp = { props: { title: spec.title, description: spec.description, config: { ...spec.config || {} } } };
+    const repaired = validateAndRepairComponent(comp, template.type, cols, q.count ?? q.data.length);
+    if (!repaired) {
+      logger.info(`[ScriptComponentGen] Recipe spec "${spec.componentType}" no longer maps to columns (shape drift) \u2014 falling back to LLM`);
+      return null;
+    }
+    const queryId = queryIds[q.sourceId] || queryCache.storeQuery(q.sql, { data: q.data, count: q.count });
+    const toolId = q.virtual ? "script_dataset" : q.sourceId;
+    const component = {
+      id: `comp_${Math.random().toString(36).substring(2, 8)}`,
+      name: template.name,
+      displayName: template.displayName,
+      type: template.type,
+      description: template.description,
+      props: {
+        ...comp.props,
+        externalTool: { toolId, toolName: q.sourceName, parameters: { queryId } }
+      },
+      category: template.category,
+      keywords: template.keywords
+    };
+    out.push(component);
+    if (componentStreamCallback && !answerComponentStreamed) {
+      componentStreamCallback(component);
+      answerComponentStreamed = true;
     }
   }
-  if (collections?.["external-tools"]?.execute) {
-    try {
-      const result = await collections["external-tools"].execute({
-        toolId: "federation_query",
-        toolName: "Cross-Source Query",
-        parameters: { sql }
-      });
-      return result;
-    } catch (err) {
-      logger.warn(`[ScriptComponentGen] Federation via collections failed: ${err}`);
+  if (out.length > 0 && !out.some((c) => c.type === "MarkdownBlock")) {
+    const md = buildMarkdown(analysisText || "");
+    if (md) {
+      out.push(md);
+      if (componentStreamCallback) componentStreamCallback(md);
+      logger.info("[ScriptComponentGen] Injected markdown on replay (recipe had no markdown spec)");
     }
   }
-  return null;
+  logger.info(`[ScriptComponentGen] Deterministically assembled ${out.length} component(s) from recipe \u2014 no LLM call`);
+  return out.length > 0 ? out : null;
 }
 
 // src/userResponse/anthropic.ts
@@ -11994,6 +12725,24 @@ var OpenAILLM = class extends BaseLLM {
 var openaiLLM = new OpenAILLM();
 
 // src/userResponse/agent-user-response.ts
+function hasExpiredScriptDataset(component) {
+  if (!component || typeof component !== "object") return false;
+  const et = component?.props?.externalTool;
+  if (et?.toolId === "script_dataset") {
+    const qid = et?.parameters?.queryId;
+    if (typeof qid === "string" && qid.length > 0) {
+      const stored = queryCache.getQuery(qid);
+      if (!stored || stored.data == null) return true;
+    }
+  }
+  const children = component?.props?.config?.components;
+  if (Array.isArray(children)) {
+    for (const child of children) {
+      if (hasExpiredScriptDataset(child)) return true;
+    }
+  }
+  return false;
+}
 function rehydrateCachedComponent(component) {
   const qMap = component?.props?.config?._queryMap;
   if (!qMap || typeof qMap !== "object" || Object.keys(qMap).length === 0) return component;
@@ -12043,7 +12792,24 @@ function getLLMInstance(provider) {
       return anthropicLLM;
   }
 }
-var get_agent_user_response = async (prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, conversationHistory, streamCallback, collections, externalTools, userId, mainAgentModel, sourceAgentModel) => {
+function storeScriptDatasetQuery(q, ctx) {
+  const data = q.data.length > MAX_ROWS_RENDERED ? q.data.slice(0, MAX_ROWS_RENDERED) : q.data;
+  if (q.data.length > MAX_ROWS_RENDERED) {
+    logger.warn(`[AgentFlow] Script dataset "${q.sourceId}" had ${q.data.length} rows \u2014 capped to ${MAX_ROWS_RENDERED} for render/cache. Consider pre-aggregating in the script.`);
+  }
+  const count = data.length;
+  if (q.virtual && ctx.recipeId) {
+    const descriptor = {
+      kind: "script_dataset",
+      recipeId: ctx.recipeId,
+      params: ctx.params || {},
+      datasetRef: q.sourceId
+    };
+    return queryCache.storeQuery(JSON.stringify(descriptor), { data, count });
+  }
+  return queryCache.storeQuery(q.sql, { data, count });
+}
+var get_agent_user_response = async (prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, conversationHistory, streamCallback, collections, externalTools, userId, mainAgentModel, sourceAgentModel, workflows) => {
   const startTime = Date.now();
   const providers = llmProviders || ["anthropic"];
   const provider = providers[0];
@@ -12052,19 +12818,19 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
   logger.logLLMPrompt("agentUserResponse", "user", `User Prompt: ${prompt}`);
   logger.info(`[AgentFlow] Starting | provider: ${provider} | prompt: "${prompt.substring(0, 50)}..."`);
   try {
-    const conversationMatch = await conversation_search_default.searchConversationsWithReranking({
+    const conversationMatch = await conversation_search_default.findExactMatch({
       userPrompt: prompt,
       collections,
-      userId,
-      similarityThreshold: EXACT_MATCH_SIMILARITY_THRESHOLD
+      userId
     });
     if (conversationMatch) {
-      logger.info(`[AgentFlow] Found matching conversation (${(conversationMatch.similarity * 100).toFixed(2)}% similarity)`);
       const rawComponent = conversationMatch.uiBlock?.component || conversationMatch.uiBlock?.generatedComponentMetadata;
       const isValidComponent = rawComponent && typeof rawComponent === "object" && Object.keys(rawComponent).length > 0;
       const component = isValidComponent ? rawComponent : null;
       const cachedTextResponse = conversationMatch.uiBlock?.analysis || conversationMatch.uiBlock?.textResponse || conversationMatch.uiBlock?.text || "";
-      if (conversationMatch.similarity >= EXACT_MATCH_SIMILARITY_THRESHOLD) {
+      if (component && hasExpiredScriptDataset(component)) {
+        logger.info(`[AgentFlow] Exact match found but cached UIBlock has expired script_dataset \u2014 falling through to script matcher`);
+      } else {
         if (streamCallback && cachedTextResponse) {
           streamCallback(cachedTextResponse);
         }
@@ -12078,16 +12844,14 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
             component: rehydratedComponent,
             actions: conversationMatch.uiBlock?.actions || [],
             reasoning: `Exact match from previous conversation`,
-            method: `${provider}-agent-semantic-match`,
-            semanticSimilarity: conversationMatch.similarity
+            method: `${provider}-agent-exact-match`,
+            semanticSimilarity: 1
           },
           errors: []
         };
       }
-      logger.info(`[AgentFlow] Similar match but below exact threshold \u2014 proceeding with agent`);
-    } else {
-      logger.info(`[AgentFlow] No matching conversations \u2014 proceeding with agent`);
     }
+    logger.info(`[AgentFlow] No exact match \u2014 proceeding with agent`);
     const apiKey = (() => {
       switch (provider) {
         case "anthropic":
@@ -12152,11 +12916,11 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
       projectId
     };
     const streamBuffer = new StreamBuffer(streamCallback);
-    const scriptStore = new ScriptStore();
+    const scriptStore = new ScriptStore({ collections, projectId });
     const turnId = `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
     const FORK_DEPTH_CAP = 3;
     let forkContext;
-    if (scriptStore.count() > 0) {
+    if (await scriptStore.count() > 0) {
       const scriptMatcher = new ScriptMatcher(scriptStore);
       const scriptMatch = await scriptMatcher.match(prompt, apiKey, sourceAgentModel);
       if (scriptMatch && scriptMatch.tier === "near") {
@@ -12190,7 +12954,7 @@ var get_agent_user_response = async (prompt, components, anthropicApiKey, groqAp
             { externalTools: agentTools, streamBuffer }
           );
           if (scriptResult.success && scriptResult.data.length > 0) {
-            scriptStore.recordSuccess(scriptMatch.recipe.id);
+            await scriptStore.recordSuccess(scriptMatch.recipe.id);
             const dataContext = scriptResult.executedQueries.map((q) => {
               const preview = q.data.slice(0, 15);
               return `## Data from "${q.sourceName}"
@@ -12228,7 +12992,7 @@ ${JSON.stringify(preview, null, 2)}
               result: {
                 _totalRecords: q.totalCount || q.count,
                 _recordsShown: q.data.length,
-                _sampleData: q.data.slice(0, 3)
+                _sampleData: q.data.slice(0, TOOL_TRACKING_SAMPLE_ROWS)
               }
             }));
             let matchedComponents2 = [];
@@ -12242,19 +13006,31 @@ ${JSON.stringify(preview, null, 2)}
                 streamBuffer.write("__TEXT_COMPLETE__COMPONENT_GENERATION_START__");
                 streamBuffer.flush();
               }
-              const componentStreamCallback = streamBuffer.hasCallback() ? (component) => {
-                const answerMarker = `__ANSWER_COMPONENT_START__${JSON.stringify(component)}__ANSWER_COMPONENT_END__`;
-                streamBuffer.write(answerMarker);
-                streamBuffer.flush();
-              } : void 0;
+              const componentStreamCallback = void 0;
               const scriptQueryIds = {};
               for (const q of scriptResult.executedQueries) {
-                scriptQueryIds[q.sourceId] = queryCache.storeQuery(q.sql, {
-                  data: q.data,
-                  count: q.count
+                scriptQueryIds[q.sourceId] = storeScriptDatasetQuery(q, {
+                  recipeId: scriptMatch.recipe.id,
+                  params: scriptMatch.extractedParams || {}
                 });
               }
-              try {
+              let usedRecipeComponents = false;
+              if (scriptMatch.recipe.components && scriptMatch.recipe.components.length > 0) {
+                const assembled = assembleComponentsFromSpecs({
+                  specs: scriptMatch.recipe.components,
+                  scriptResult,
+                  queryIds: scriptQueryIds,
+                  availableComponents: components,
+                  componentStreamCallback,
+                  analysisText: scriptTextResponse
+                });
+                if (assembled && assembled.length > 0) {
+                  matchedComponents2 = assembled;
+                  layoutTitle2 = scriptMatch.recipe.name || layoutTitle2;
+                  usedRecipeComponents = true;
+                }
+              }
+              if (!usedRecipeComponents) try {
                 const compResult = await generateScriptComponents({
                   userPrompt: prompt,
                   scriptResult,
@@ -12264,13 +13040,22 @@ ${JSON.stringify(preview, null, 2)}
                   model: sourceAgentModel,
                   componentStreamCallback,
                   externalTools: agentTools,
-                  collections
+                  collections,
+                  analysisText: scriptTextResponse
                 });
                 if (compResult.components.length > 0) {
                   matchedComponents2 = compResult.components;
                   layoutTitle2 = compResult.layoutTitle;
                   layoutDescription2 = compResult.layoutDescription;
                   actions2 = compResult.actions;
+                  if (compResult.componentSpecs.length > 0) {
+                    try {
+                      scriptMatch.recipe.components = compResult.componentSpecs;
+                      await scriptStore.save(scriptMatch.recipe);
+                    } catch (e) {
+                      logger.warn(`[AgentFlow] Failed to backfill component recipe: ${e}`);
+                    }
+                  }
                 } else {
                   logger.info(`[AgentFlow] Lightweight component gen returned no components \u2014 falling back to full generator`);
                   const matchResult = await generateAgentComponents({
@@ -12363,12 +13148,12 @@ ${JSON.stringify(preview, null, 2)}
             };
           } else {
             logger.warn(`[AgentFlow] Script "${scriptMatch.recipe.name}" failed: ${scriptResult.error || "no data returned"}`);
-            scriptStore.recordFailure(scriptMatch.recipe.id);
+            await scriptStore.recordFailure(scriptMatch.recipe.id);
           }
         } catch (scriptErr) {
           const errMsg = scriptErr instanceof Error ? scriptErr.message : String(scriptErr);
           logger.warn(`[AgentFlow] Script execution error (falling back to agent flow): ${errMsg}`);
-          scriptStore.recordFailure(scriptMatch.recipe.id);
+          await scriptStore.recordFailure(scriptMatch.recipe.id);
         }
       }
     } else {
@@ -12425,13 +13210,35 @@ User question: ${prompt}`;
         logger.warn(`[AgentFlow] Source routing pre-filter failed (using all tools): ${msg}`);
       }
     }
-    const mainAgent = new MainAgent(filteredTools, agentConfig, scriptStore, turnId, streamBuffer);
+    const mainAgent = new MainAgent(filteredTools, agentConfig, scriptStore, turnId, streamBuffer, workflows || []);
     const agentResponse = await mainAgent.handleQuestion(
       effectivePrompt,
       apiKey,
       conversationHistory,
       streamBuffer.hasCallback() ? (chunk) => streamBuffer.write(chunk) : void 0
     );
+    if (agentResponse.workflow) {
+      streamBuffer.flush();
+      const workflowComponent = {
+        id: `workflow_${Date.now()}`,
+        name: agentResponse.workflow.name,
+        type: `Workflow_${agentResponse.workflow.name}`,
+        description: `Workflow: ${agentResponse.workflow.name}`,
+        props: agentResponse.workflow.props
+      };
+      const elapsedTime2 = Date.now() - startTime;
+      logger.info(`[AgentFlow] Workflow short-circuit | "${agentResponse.workflow.name}" | ${elapsedTime2}ms`);
+      return {
+        success: true,
+        data: {
+          text: "",
+          component: workflowComponent,
+          actions: [],
+          method: `${provider}-agent-workflow`
+        },
+        errors: []
+      };
+    }
     const rawText = streamBuffer.getFullText() || agentResponse.text || "I apologize, but I was unable to generate a response.";
     const textResponse = rawText.replace(/_?_?SB_END_?_?/g, "").replace(/__SB_\w+_(?:START|MSG)_?_?/g, "").replace(/__QUERY_TIMER_START_[^_]*__/g, "").replace(/__QUERY_TIMER_DONE_[\d.]+__/g, "").replace(/__TEXT_COMPLETE__COMPONENT_GENERATION_START__/g, "").replace(/\[COMPLEXITY:\s*(?:simple|medium|complex)\]/gi, "");
     streamBuffer.flush();
@@ -12471,6 +13278,7 @@ User question: ${prompt}`;
     let layoutTitle = "Dashboard";
     let layoutDescription = "Multi-component dashboard";
     let actions = [];
+    let authoredComponentSpecs = [];
     if (!hasExecutedTools) {
       logger.info(`[AgentFlow] No tools executed \u2014 general question, wrapping in DynamicMarkdownBlock`);
       const mainAgentText = agentResponse.text || textResponse;
@@ -12518,9 +13326,9 @@ User question: ${prompt}`;
             logger.info(`[AgentFlow] Using savedScript.executedQueries (${agentResponse.savedScript.executedQueries.length} entries, includes computed columns) for component gen`);
             const scriptQueries = agentResponse.savedScript.executedQueries;
             for (const q of scriptQueries) {
-              agentQueryIds[q.sourceId] = queryCache.storeQuery(q.sql, {
-                data: q.data,
-                count: q.count
+              agentQueryIds[q.sourceId] = storeScriptDatasetQuery(q, {
+                recipeId: agentResponse.savedScript.recipeId,
+                params: {}
               });
             }
             agentScriptResult = {
@@ -12558,7 +13366,7 @@ User question: ${prompt}`;
               executionTimeMs: 0
             };
           }
-          const compResult = await generateScriptComponents({
+          const compGenArgs = {
             userPrompt: prompt,
             scriptResult: agentScriptResult,
             queryIds: agentQueryIds,
@@ -12567,13 +13375,29 @@ User question: ${prompt}`;
             model: sourceAgentModel,
             componentStreamCallback,
             externalTools: agentTools,
-            collections
-          });
-          if (compResult.components.length > 0) {
+            collections,
+            analysisText: agentResponse.text
+          };
+          const MAX_COMP_ATTEMPTS = 3;
+          let compResult;
+          for (let attempt = 1; attempt <= MAX_COMP_ATTEMPTS; attempt++) {
+            try {
+              compResult = await generateScriptComponents(compGenArgs);
+              break;
+            } catch (err) {
+              const msg = err instanceof Error ? err.message : String(err);
+              const transient = /connection error|econnreset|etimedout|fetch failed|socket hang up|overloaded|rate.?limit|\b429\b|\b5\d\d\b|network|timeout/i.test(msg);
+              if (!transient || attempt === MAX_COMP_ATTEMPTS) throw err;
+              logger.warn(`[AgentFlow] Lightweight component gen transient error (attempt ${attempt}/${MAX_COMP_ATTEMPTS}) \u2014 retrying: ${msg}`);
+              await new Promise((res) => setTimeout(res, 400 * attempt));
+            }
+          }
+          if (compResult && compResult.components.length > 0) {
             matchedComponents = compResult.components;
             layoutTitle = compResult.layoutTitle;
             layoutDescription = compResult.layoutDescription;
             actions = compResult.actions;
+            authoredComponentSpecs = compResult.componentSpecs;
             usedLightweight = true;
             logger.info(`[AgentFlow] Lightweight component gen succeeded \u2014 ${matchedComponents.length} components`);
           }
@@ -12676,10 +13500,11 @@ User question: ${prompt}`;
           forkDepth: Math.min(FORK_DEPTH_CAP, forkContext.parentDepth + 1),
           forkReason: forkContext.modificationHint
         } : void 0;
-        const promoted = scriptStore.promoteToVerified(authored.recipeId, {
+        const promoted = await scriptStore.promoteToVerified(authored.recipeId, {
           sourceIds: authored.sourceIds,
           tables: authored.tables,
-          ...lineage
+          ...lineage,
+          components: authoredComponentSpecs
         });
         if (promoted && lineage?.parentId) {
           logger.info(
@@ -12915,7 +13740,7 @@ var CONTEXT_CONFIG = {
 };
 
 // src/handlers/user-prompt-request.ts
-var get_user_request = async (data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel) => {
+var get_user_request = async (data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel, workflows) => {
   const errors = [];
   const parseResult = UserPromptRequestMessageSchema.safeParse(data);
   if (!parseResult.success) {
@@ -13001,7 +13826,8 @@ var get_user_request = async (data, components, sendMessage, anthropicApiKey, gr
     externalTools,
     userId,
     mainAgentModel,
-    sourceAgentModel
+    sourceAgentModel,
+    workflows
   );
   logger.info("User prompt request completed");
   const uiBlockId = existingUiBlockId;
@@ -13154,8 +13980,8 @@ var get_user_request = async (data, components, sendMessage, anthropicApiKey, gr
     wsId
   };
 };
-async function handleUserPromptRequest(data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel) {
-  const response = await get_user_request(data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel);
+async function handleUserPromptRequest(data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel, workflows) {
+  const response = await get_user_request(data, components, sendMessage, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, externalTools, mainAgentModel, sourceAgentModel, workflows);
   if (response.data?.component?.props?.config?.components) {
     response.data.component.props.config.components = response.data.component.props.config.components.map((comp) => ({
       ...comp,
@@ -13587,6 +14413,24 @@ async function handleComponentListResponse(data, storeComponents, collections) {
   }
 }
 
+// src/handlers/workflow-list-response.ts
+async function handleWorkflowListResponse(data, storeWorkflows) {
+  try {
+    const parsed = WorkflowListResponseMessageSchema.parse(data);
+    const { payload } = parsed;
+    const workflowsList = payload.workflows;
+    if (!workflowsList) {
+      logger.error("Workflows list not found in WORKFLOW_LIST_RES payload");
+      return;
+    }
+    const workflows = WorkflowsSchema.parse(workflowsList);
+    storeWorkflows(workflows);
+    logger.info(`Stored ${workflows.length} workflow descriptor(s) from frontend`);
+  } catch (error) {
+    logger.error("Failed to handle workflow list response:", error);
+  }
+}
+
 // src/handlers/users.ts
 async function handleUsersRequest(data, collections, sendMessage) {
   const executeCollection = async (collection, op, params) => {
@@ -16403,17 +17247,45 @@ function formatComponentsForPrompt(components) {
    Props Structure: ${propsPreview}`;
   }).join("\n\n");
 }
+function classifyTool(tool) {
+  if (tool.toolType === "direct") return "direct";
+  if (tool.id.endsWith("_query")) return "sql";
+  if (tool.id.endsWith("_call")) return "rest";
+  if (tool.id.endsWith("_graphql")) return "graphql";
+  return "direct";
+}
 function formatToolsForPrompt(tools) {
   if (!tools || tools.length === 0) {
     return "No external tools available.";
   }
-  return tools.map((tool, idx) => {
+  const directTools = [];
+  const sourceTools = [];
+  for (const t of tools) {
+    (classifyTool(t) === "direct" ? directTools : sourceTools).push(t);
+  }
+  const renderTool = (tool, idx, label) => {
     const paramsStr = Object.entries(tool.params || {}).map(([key, type]) => `${key}: ${type}`).join(", ");
-    return `${idx + 1}. ID: ${tool.id}
+    return `${idx + 1}. [${label}] ID: ${tool.id}
    Name: ${tool.name}
    Description: ${tool.description}
    Parameters: { ${paramsStr} }`;
-  }).join("\n\n");
+  };
+  const sections = [];
+  if (directTools.length > 0) {
+    const body = directTools.map((t, i) => renderTool(t, i, "DIRECT \u2014 PREFER")).join("\n\n");
+    sections.push(`### Direct Tools (try these first \u2014 they answer most questions without SQL)
+${body}`);
+  }
+  if (sourceTools.length > 0) {
+    const body = sourceTools.map((t, i) => {
+      const kind = classifyTool(t);
+      const label = kind === "sql" ? "SQL" : kind === "rest" ? "REST" : "GRAPHQL";
+      return renderTool(t, i, label);
+    }).join("\n\n");
+    sections.push(`### Source Tools (fallback \u2014 use only when no direct tool fits)
+${body}`);
+  }
+  return sections.join("\n\n");
 }
 function formatExistingComponentsForPrompt(existingComponents) {
   if (!existingComponents || existingComponents.length === 0) {
@@ -16438,7 +17310,7 @@ function sendDashCompResponse(id, res, sendMessage, clientId) {
 }
 
 // src/dashComp/pick-component.ts
-async function pickComponentWithLLM(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, dashCompModels, conversationHistory) {
+async function pickComponentWithLLM(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, dashCompModels, conversationHistory, userId) {
   const errors = [];
   const availableComponentsText = formatComponentsForPrompt(components);
   const availableToolsText = formatToolsForPrompt(tools);
@@ -16454,6 +17326,25 @@ async function pickComponentWithLLM(prompt, components, anthropicApiKey, groqApi
       schemaDoc = schema.generateSchemaDocumentation();
     }
     const databaseRules = await promptLoader.loadDatabaseRules();
+    let globalKnowledgeBase = "No global knowledge base available.";
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      globalKnowledgeBase = kbResult.globalContext || globalKnowledgeBase;
+      const dynamicParts = [];
+      if (kbResult.userContext) {
+        dynamicParts.push("## User-Specific Knowledge Base\n" + kbResult.userContext);
+      }
+      if (kbResult.queryContext) {
+        dynamicParts.push("## Relevant Knowledge Base (Query-Matched)\n" + kbResult.queryContext);
+      }
+      knowledgeBaseContext = dynamicParts.join("\n\n") || knowledgeBaseContext;
+    }
     const prompts = await promptLoader.loadPrompts("dash-comp-picker", {
       USER_PROMPT: prompt,
       AVAILABLE_COMPONENTS: availableComponentsText,
@@ -16461,8 +17352,12 @@ async function pickComponentWithLLM(prompt, components, anthropicApiKey, groqApi
       DATABASE_RULES: databaseRules,
       AVAILABLE_TOOLS: availableToolsText,
       CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
-      CONVERSATION_HISTORY: conversationHistory || "No previous conversation"
+      CONVERSATION_HISTORY: conversationHistory || "No previous conversation",
+      GLOBAL_KNOWLEDGE_BASE: globalKnowledgeBase,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext
     });
+    logger.logLLMPrompt("dashCompPicker", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("dashCompPicker", "user", prompts.user);
     logger.debug("[DASH_COMP_REQ] Loaded dash-comp-picker prompts with schema and tools");
     const { apiKey, model } = getApiKeyAndModel(
       anthropicApiKey,
@@ -16828,6 +17723,18 @@ Fixed SQL query:`;
       logger.info(`[DASH_COMP_REQ] Replaced direct query with queryId: ${queryId}`);
     }
     finalComponent = { ...finalComponent, props };
+    const ext = finalComponent.props?.externalTool;
+    if (ext?.toolId && !ext?.parameters?.sql) {
+      const matchedTool = tools?.find((t) => t.id === ext.toolId);
+      if (matchedTool && matchedTool.cache !== false) {
+        const match = executedTools.find((t) => t.id === ext.toolId);
+        if (match) {
+          const cacheKey = buildDirectToolCacheKey(ext.toolId, ext.parameters);
+          queryCache.set(cacheKey, { success: true, data: match.result });
+          logger.info(`[DASH_COMP_REQ] Pre-populated et-direct cache for ${ext.toolId}`);
+        }
+      }
+    }
     if (parsedResult.props.query) {
       logger.info(`[DASH_COMP_REQ] Data source: Database query`);
     }
@@ -16862,7 +17769,7 @@ Fixed SQL query:`;
 }
 
 // src/dashComp/create-filter.ts
-async function createFilterWithLLM(prompt, components, existingComponents, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, tools, dashCompModels, collections) {
+async function createFilterWithLLM(prompt, components, existingComponents, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, tools, dashCompModels, collections, userId) {
   const errors = [];
   try {
     const filterComponents = components.filter((c) => c.type.startsWith("Filter"));
@@ -16882,6 +17789,25 @@ async function createFilterWithLLM(prompt, components, existingComponents, anthr
       schemaDoc = schema.generateSchemaDocumentation();
     }
     const databaseRules = await promptLoader.loadDatabaseRules();
+    let globalKnowledgeBase = "No global knowledge base available.";
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      globalKnowledgeBase = kbResult.globalContext || globalKnowledgeBase;
+      const dynamicParts = [];
+      if (kbResult.userContext) {
+        dynamicParts.push("## User-Specific Knowledge Base\n" + kbResult.userContext);
+      }
+      if (kbResult.queryContext) {
+        dynamicParts.push("## Relevant Knowledge Base (Query-Matched)\n" + kbResult.queryContext);
+      }
+      knowledgeBaseContext = dynamicParts.join("\n\n") || knowledgeBaseContext;
+    }
     const prompts = await promptLoader.loadPrompts("dash-filter-picker", {
       USER_PROMPT: prompt,
       AVAILABLE_COMPONENTS: formatComponentsForPrompt(filterComponents),
@@ -16889,8 +17815,12 @@ async function createFilterWithLLM(prompt, components, existingComponents, anthr
       SCHEMA_DOC: schemaDoc || "No database schema available",
       DATABASE_RULES: databaseRules,
       AVAILABLE_TOOLS: formatToolsForPrompt(tools),
-      CURRENT_DATETIME: getCurrentDateTimeForPrompt()
+      CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
+      GLOBAL_KNOWLEDGE_BASE: globalKnowledgeBase,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext
     });
+    logger.logLLMPrompt("dashFilterPicker", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("dashFilterPicker", "user", prompts.user);
     logger.debug("[DASH_COMP_REQ:FILTER] Loaded dash-filter-picker prompts");
     const { apiKey, model } = getApiKeyAndModel(
       anthropicApiKey,
@@ -16999,7 +17929,35 @@ async function createFilterWithLLM(prompt, components, existingComponents, anthr
     const updatedComponents = result.updatedComponents || [];
     for (const comp of updatedComponents) {
       const extTool = comp.props?.externalTool;
-      if (!extTool?.parameters?.sql || !extTool?.toolId) continue;
+      if (!extTool?.toolId) continue;
+      const isDirectTool = !extTool?.parameters?.sql;
+      if (isDirectTool) {
+        const directTool = tools?.find((t) => t.id === extTool.toolId);
+        if (!directTool) {
+          logger.warn(`[DASH_COMP_REQ:FILTER] direct tool ${extTool.toolId} not found in tool registry`);
+          continue;
+        }
+        const declared = new Set(Object.keys(directTool.params || {}));
+        const sentKeys = Object.keys(extTool.parameters || {});
+        const unknown = sentKeys.filter((k) => !declared.has(k));
+        if (unknown.length > 0) {
+          logger.warn(`[DASH_COMP_REQ:FILTER] dropping unknown params on ${extTool.toolId}: ${unknown.join(", ")}`);
+          unknown.forEach((k) => delete extTool.parameters[k]);
+        }
+        if (directTool.cache !== false) {
+          try {
+            const directResult = await directTool.fn(extTool.parameters || {});
+            const cacheKey2 = buildDirectToolCacheKey(extTool.toolId, extTool.parameters);
+            queryCache.set(cacheKey2, { success: true, data: directResult });
+            logger.info(`[DASH_COMP_REQ:FILTER] direct tool ${extTool.toolId} validated and cached for component: ${comp.id}`);
+          } catch (err) {
+            const errMsg = err instanceof Error ? err.message : String(err);
+            logger.warn(`[DASH_COMP_REQ:FILTER] direct tool validation failed for ${extTool.toolId} on component ${comp.id}: ${errMsg}`);
+          }
+        }
+        continue;
+      }
+      if (!extTool?.parameters?.sql) continue;
       let sql = extTool.parameters.sql;
       const defaultParams = extTool.parameters.params || {};
       const toolId = extTool.toolId;
@@ -17236,7 +18194,8 @@ var processDashCompRequest = async (data, components, _sendMessage, anthropicApi
       llmProviders,
       tools,
       dashCompModels,
-      collections
+      collections,
+      userId
     );
   } else {
     llmResponse = await pickComponentWithLLM(
@@ -17250,7 +18209,8 @@ var processDashCompRequest = async (data, components, _sendMessage, anthropicApi
       collections,
       tools,
       dashCompModels,
-      conversationHistory
+      conversationHistory,
+      userId
     );
   }
   if (llmResponse.success && dashboardId && prompt) {
@@ -17389,7 +18349,7 @@ function sendReportCompResponse(id, res, sendMessage, clientId) {
 }
 
 // src/reportComp/generate-report.ts
-async function generateReportComponents(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, modelConfig, conversationHistory) {
+async function generateReportComponents(prompt, components, anthropicApiKey, groqApiKey, geminiApiKey, openaiApiKey, llmProviders, collections, tools, modelConfig, conversationHistory, userId) {
   const errors = [];
   const availableComponentsText = formatComponentsForPrompt2(components);
   const availableToolsText = formatToolsForPrompt2(tools);
@@ -17405,6 +18365,25 @@ async function generateReportComponents(prompt, components, anthropicApiKey, gro
       schemaDoc = schema.generateSchemaDocumentation();
     }
     const databaseRules = await promptLoader.loadDatabaseRules();
+    let globalKnowledgeBase = "No global knowledge base available.";
+    let knowledgeBaseContext = "No additional knowledge base context available.";
+    if (collections) {
+      const kbResult = await knowledge_base_default.getAllKnowledgeBase({
+        prompt,
+        collections,
+        userId,
+        topK: KNOWLEDGE_BASE_TOP_K
+      });
+      globalKnowledgeBase = kbResult.globalContext || globalKnowledgeBase;
+      const dynamicParts = [];
+      if (kbResult.userContext) {
+        dynamicParts.push("## User-Specific Knowledge Base\n" + kbResult.userContext);
+      }
+      if (kbResult.queryContext) {
+        dynamicParts.push("## Relevant Knowledge Base (Query-Matched)\n" + kbResult.queryContext);
+      }
+      knowledgeBaseContext = dynamicParts.join("\n\n") || knowledgeBaseContext;
+    }
     const prompts = await promptLoader.loadPrompts("report-comp-picker", {
       USER_PROMPT: prompt,
       AVAILABLE_COMPONENTS: availableComponentsText,
@@ -17412,8 +18391,12 @@ async function generateReportComponents(prompt, components, anthropicApiKey, gro
       DATABASE_RULES: databaseRules,
       AVAILABLE_TOOLS: availableToolsText,
       CURRENT_DATETIME: getCurrentDateTimeForPrompt(),
-      CONVERSATION_HISTORY: conversationHistory || "No previous conversation"
+      CONVERSATION_HISTORY: conversationHistory || "No previous conversation",
+      GLOBAL_KNOWLEDGE_BASE: globalKnowledgeBase,
+      KNOWLEDGE_BASE_CONTEXT: knowledgeBaseContext
     });
+    logger.logLLMPrompt("reportCompPicker", "system", extractPromptText(prompts.system));
+    logger.logLLMPrompt("reportCompPicker", "user", prompts.user);
     logger.debug("[REPORT_COMP_REQ] Loaded report-comp-picker prompts with schema and tools");
     const { apiKey, model } = getApiKeyAndModel2(
       anthropicApiKey,
@@ -17638,13 +18621,21 @@ async function validateAllExternalToolQueries(components, collections, tools, mo
           data: {}
         });
         if (result?.success !== false && !result?.error) {
-          const resultData = result?.data?.data ?? result?.data ?? [];
-          const dataArray = Array.isArray(resultData) ? resultData : [resultData];
+          const toolResult = result?.data ?? result;
+          const valueKey = comp.props?.config?.valueKey;
+          const isKpi = comp.type === "KPICard" || comp.name === "DynamicKPICard";
+          let dataArray;
+          if (isKpi && valueKey && toolResult && typeof toolResult === "object" && !Array.isArray(toolResult) && toolResult[valueKey] !== void 0) {
+            dataArray = [toolResult];
+          } else {
+            const resultData = toolResult?.data ?? toolResult ?? [];
+            dataArray = Array.isArray(resultData) ? resultData : [resultData];
+          }
           if (!comp.props.config) {
             comp.props.config = {};
           }
           comp.props.config.data = dataArray;
-          logger.info(`[REPORT_COMP_REQ] \u2713 ${comp.name} prefetched ${dataArray.length} rows (non-SQL tool)`);
+          logger.info(`[REPORT_COMP_REQ] \u2713 ${comp.name} prefetched ${dataArray.length} ${dataArray.length === 1 && isKpi ? "aggregate" : "rows"} (non-SQL tool)`);
         }
       } catch (err) {
         logger.warn(`[REPORT_COMP_REQ] \u26A0 ${comp.name} non-SQL prefetch failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -17784,7 +18775,8 @@ var processReportCompRequest = async (data, components, _sendMessage, anthropicA
     collections,
     tools,
     modelConfig,
-    conversationHistory
+    conversationHistory,
+    userId
   );
   if (llmResponse.success && reportId && prompt) {
     const comps = llmResponse.data?.components;
@@ -17914,7 +18906,7 @@ function getLLMProviders() {
 // src/auth/user-manager.ts
 import fs8 from "fs";
 import path8 from "path";
-import os from "os";
+import os2 from "os";
 var UserManager = class {
   /**
    * Initialize UserManager with file path and sync interval
@@ -17926,7 +18918,7 @@ var UserManager = class {
     this.hasChanged = false;
     this.syncInterval = null;
     this.isInitialized = false;
-    this.filePath = path8.join(os.homedir(), ".superatom", "projects", projectId, "users.json");
+    this.filePath = path8.join(os2.homedir(), ".superatom", "projects", projectId, "users.json");
     this.syncIntervalMs = syncIntervalMs;
   }
   /**
@@ -18212,7 +19204,7 @@ var UserManager = class {
 // src/dashboards/dashboard-manager.ts
 import fs9 from "fs";
 import path9 from "path";
-import os2 from "os";
+import os3 from "os";
 var DashboardManager = class {
   /**
    * Initialize DashboardManager with project ID
@@ -18221,7 +19213,7 @@ var DashboardManager = class {
   constructor(projectId = "snowflake-dataset") {
     this.projectId = projectId;
     this.dashboardsBasePath = path9.join(
-      os2.homedir(),
+      os3.homedir(),
       ".superatom",
       "projects",
       projectId,
@@ -18379,7 +19371,7 @@ var DashboardManager = class {
 // src/reports/report-manager.ts
 import fs10 from "fs";
 import path10 from "path";
-import os3 from "os";
+import os4 from "os";
 var ReportManager = class {
   /**
    * Initialize ReportManager with project ID
@@ -18388,7 +19380,7 @@ var ReportManager = class {
   constructor(projectId = "snowflake-dataset") {
     this.projectId = projectId;
     this.reportsBasePath = path10.join(
-      os3.homedir(),
+      os4.homedir(),
       ".superatom",
       "projects",
       projectId,
@@ -18909,23 +19901,27 @@ var CleanupService = class _CleanupService {
 // src/index.ts
 var DEFAULT_WS_URL = "wss://ws.superatom.ai/websocket";
 var SuperatomSDK = class {
-  // 3.5 minutes (PING_INTERVAL + 30s grace)
   constructor(config) {
     this.ws = null;
     this.messageHandlers = /* @__PURE__ */ new Map();
     this.messageTypeHandlers = /* @__PURE__ */ new Map();
     this.connected = false;
     this.reconnectAttempts = 0;
-    this.maxReconnectAttempts = 5;
+    // Retry forever — the backend must self-heal across NAT timeouts, DO
+    // redeploys, and long network outages. The previous cap (5) gave up after
+    // ~30s and left the singleton SDK dead until process restart.
+    this.maxReconnectAttempts = Infinity;
     this.collections = {};
     this.components = [];
     this.tools = [];
-    // Heartbeat properties for keeping WebSocket connection alive
+    this.workflows = [];
+    // Heartbeat properties for keeping WebSocket connection alive.
+    // 25s ping + 10s grace stays under common NAT/LB idle thresholds (~60-100s)
+    // so we detect dead sockets within seconds instead of minutes.
     this.pingInterval = null;
     this.lastPong = Date.now();
-    this.PING_INTERVAL_MS = 18e4;
-    // 3 minutes
-    this.PONG_TIMEOUT_MS = 21e4;
+    this.PING_INTERVAL_MS = 25e3;
+    this.PONG_TIMEOUT_MS = 35e3;
     if (config.logLevel) {
       logger.setLogLevel(config.logLevel);
     }
@@ -19081,7 +20077,7 @@ var SuperatomSDK = class {
           this.handlePong();
           break;
         case "DATA_REQ":
-          handleDataRequest(parsed, this.collections, (msg) => this.send(msg)).catch((error) => {
+          handleDataRequest(parsed, this.collections, (msg) => this.send(msg), this.tools).catch((error) => {
             logger.error("Failed to handle data request:", error);
           });
           break;
@@ -19101,7 +20097,7 @@ var SuperatomSDK = class {
           });
           break;
         case "USER_PROMPT_REQ":
-          handleUserPromptRequest(parsed, this.components, (msg) => this.send(msg), this.anthropicApiKey, this.groqApiKey, this.geminiApiKey, this.openaiApiKey, this.llmProviders, this.collections, this.tools, this.mainAgentModel, this.sourceAgentModel).catch((error) => {
+          handleUserPromptRequest(parsed, this.components, (msg) => this.send(msg), this.anthropicApiKey, this.groqApiKey, this.geminiApiKey, this.openaiApiKey, this.llmProviders, this.collections, this.tools, this.mainAgentModel, this.sourceAgentModel, this.workflows).catch((error) => {
             logger.error("Failed to handle user prompt request:", error);
           });
           break;
@@ -19120,6 +20116,11 @@ var SuperatomSDK = class {
             logger.error("Failed to handle component list request:", error);
           });
           break;
+        case "WORKFLOW_LIST_RES":
+          handleWorkflowListResponse(parsed, (wf) => this.setWorkflows(wf)).catch((error) => {
+            logger.error("Failed to handle workflow list request:", error);
+          });
+          break;
         case "USERS":
           handleUsersRequest(parsed, this.collections, (msg) => this.send(msg)).catch((error) => {
             logger.error("Failed to handle users request:", error);
@@ -19279,9 +20280,9 @@ var SuperatomSDK = class {
   handleReconnect() {
     if (this.reconnectAttempts < this.maxReconnectAttempts) {
       this.reconnectAttempts++;
-      const delay = Math.min(1e3 * Math.pow(2, this.reconnectAttempts), 1e4);
+      const delay = Math.min(1e3 * Math.pow(2, this.reconnectAttempts), 3e4);
       setTimeout(() => {
-        logger.info(`Attempting to reconnect (${this.reconnectAttempts}/${this.maxReconnectAttempts})...`);
+        logger.info(`Attempting to reconnect (attempt ${this.reconnectAttempts})...`);
         this.connect().catch((error) => {
           logger.error("Reconnection failed:", error);
         });
@@ -19359,6 +20360,24 @@ var SuperatomSDK = class {
   getTools() {
     return this.tools;
   }
+  /**
+   * Register workflow components for the SDK instance.
+   *
+   * Workflows are pre-built multi-step UI flows the main agent can pick when
+   * the user's prompt matches a workflow's `whenToUse` trigger. Picking a
+   * workflow short-circuits analysis text + dashboard component generation —
+   * the workflow component is returned directly, with the LLM-extracted props.
+   */
+  setWorkflows(workflows) {
+    this.workflows = workflows;
+    logger.info(`Workflows stored in SDK: ${workflows.length} workflow(s)`);
+  }
+  /**
+   * Get the registered workflow components.
+   */
+  getWorkflows() {
+    return this.workflows;
+  }
   /**
    * Apply model strategy to all LLM provider singletons
    * @param strategy - 'best', 'fast', or 'balanced'
@@ -19418,6 +20437,8 @@ export {
   LLM,
   MainAgent,
   STORAGE_CONFIG,
+  ScriptMatcher,
+  ScriptStore,
   SuperatomSDK,
   Thread,
   ThreadManager,
@@ -19431,10 +20452,13 @@ export {
   hybridRerank,
   llmUsageLogger,
   logger,
+  normalizeScriptBody,
   openaiLLM,
   queryCache,
   rerankChromaResults,
   rerankConversationResults,
+  resolveScriptRecipeStore,
+  runScript,
   userPromptErrorLogger
 };
 //# sourceMappingURL=index.mjs.map