@superatomai/sdk-node 0.0.15-mds → 0.0.16-dsp

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.mts CHANGED
@@ -777,6 +777,14 @@ declare const ToolSchema: z.ZodObject<{
777
777
  description: string;
778
778
  }[];
779
779
  }>>;
780
+ /** Cache policy. `false` = never cache (live data, write ops). Mirrors HTTP `Cache-Control: no-store`. */
781
+ cache: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<false>, z.ZodObject<{
782
+ ttlMs: z.ZodOptional<z.ZodNumber>;
783
+ }, "strip", z.ZodTypeAny, {
784
+ ttlMs?: number | undefined;
785
+ }, {
786
+ ttlMs?: number | undefined;
787
+ }>]>>;
780
788
  }, "strip", z.ZodTypeAny, {
781
789
  id: string;
782
790
  params: Record<string, string>;
@@ -793,6 +801,9 @@ declare const ToolSchema: z.ZodObject<{
793
801
  description: string;
794
802
  }[];
795
803
  } | undefined;
804
+ cache?: false | {
805
+ ttlMs?: number | undefined;
806
+ } | undefined;
796
807
  }, {
797
808
  id: string;
798
809
  params: Record<string, string>;
@@ -809,6 +820,9 @@ declare const ToolSchema: z.ZodObject<{
809
820
  description: string;
810
821
  }[];
811
822
  } | undefined;
823
+ cache?: false | {
824
+ ttlMs?: number | undefined;
825
+ } | undefined;
812
826
  }>;
813
827
  type Tool$1 = z.infer<typeof ToolSchema>;
814
828
  type CollectionOperation = 'getMany' | 'getOne' | 'query' | 'mutation' | 'updateOne' | 'deleteOne' | 'createOne';
@@ -860,6 +874,18 @@ interface SuperatomSDKConfig {
860
874
  * - 'balanced': Use best model for complex tasks, fast model for simple tasks (default)
861
875
  */
862
876
  modelStrategy?: ModelStrategy;
877
+ /**
878
+ * Model for the main agent (routing + analysis).
879
+ * Format: "provider/model-name" (e.g., "anthropic/claude-haiku-4-5-20251001")
880
+ * If not set, uses the provider's default model.
881
+ */
882
+ mainAgentModel?: string;
883
+ /**
884
+ * Model for source agents (per-source query generation).
885
+ * Format: "provider/model-name" (e.g., "anthropic/claude-haiku-4-5-20251001")
886
+ * If not set, uses the provider's default model.
887
+ */
888
+ sourceAgentModel?: string;
863
889
  /**
864
890
  * Separate model configuration for DASH_COMP flow (dashboard component picking)
865
891
  * If not provided, falls back to provider-based model selection
@@ -1294,198 +1320,1198 @@ declare class ReportManager {
1294
1320
  getReportCount(): number;
1295
1321
  }
1296
1322
 
1297
- type SystemPrompt = string | Anthropic.Messages.TextBlockParam[];
1298
- interface LLMMessages {
1299
- sys: SystemPrompt;
1300
- user: string;
1301
- prefill?: string;
1302
- }
1303
- interface LLMOptions {
1304
- model?: string;
1305
- maxTokens?: number;
1306
- temperature?: number;
1307
- topP?: number;
1308
- apiKey?: string;
1309
- partial?: (chunk: string) => void;
1310
- }
1311
- interface Tool {
1312
- name: string;
1313
- description: string;
1314
- input_schema: {
1315
- type: string;
1316
- properties: Record<string, any>;
1317
- required?: string[];
1318
- };
1319
- }
1320
- declare class LLM {
1321
- static text(messages: LLMMessages, options?: LLMOptions): Promise<string>;
1322
- static stream<T = string>(messages: LLMMessages, options?: LLMOptions, json?: boolean): Promise<T extends string ? string : any>;
1323
- static streamWithTools(messages: LLMMessages, tools: Tool[], toolHandler: (toolName: string, toolInput: any) => Promise<any>, options?: LLMOptions, maxIterations?: number): Promise<string>;
1324
- /**
1325
- * Normalize system prompt to Anthropic format
1326
- * Converts string to array format if needed
1327
- * @param sys - System prompt (string or array of blocks)
1328
- * @returns Normalized system prompt for Anthropic API
1329
- */
1330
- private static _normalizeSystemPrompt;
1331
- /**
1332
- * Log cache usage metrics from Anthropic API response
1333
- * Shows cache hits, costs, and savings
1334
- */
1335
- private static _logCacheUsage;
1336
- /**
1337
- * Parse model string to extract provider and model name
1338
- * @param modelString - Format: "provider/model-name" or just "model-name"
1339
- * @returns [provider, modelName]
1340
- *
1341
- * @example
1342
- * "anthropic/claude-sonnet-4-5" → ["anthropic", "claude-sonnet-4-5"]
1343
- * "groq/openai/gpt-oss-120b" → ["groq", "openai/gpt-oss-120b"]
1344
- * "claude-sonnet-4-5" → ["anthropic", "claude-sonnet-4-5"] (default)
1345
- */
1346
- private static _parseModel;
1347
- private static _anthropicText;
1348
- private static _anthropicStream;
1349
- private static _anthropicStreamWithTools;
1350
- private static _groqText;
1351
- private static _groqStream;
1352
- private static _geminiText;
1353
- private static _geminiStream;
1354
- /**
1355
- * Recursively strip unsupported JSON Schema properties for Gemini
1356
- * Gemini doesn't support: additionalProperties, $schema, etc.
1357
- */
1358
- private static _cleanSchemaForGemini;
1359
- private static _geminiStreamWithTools;
1360
- private static _openaiText;
1361
- private static _openaiStream;
1362
- private static _openaiStreamWithTools;
1363
- /**
1364
- * Parse JSON string, handling markdown code blocks and surrounding text
1365
- * Enhanced version with jsonrepair to handle malformed JSON from LLMs
1366
- * @param text - Text that may contain JSON wrapped in ```json...``` or with surrounding text
1367
- * @returns Parsed JSON object or array
1368
- */
1369
- private static _parseJSON;
1370
- }
1371
-
1372
- interface CapturedLog {
1373
- timestamp: number;
1374
- level: 'info' | 'error' | 'warn' | 'debug';
1375
- message: string;
1376
- type?: 'explanation' | 'query' | 'general';
1377
- data?: Record<string, any>;
1378
- }
1379
1323
  /**
1380
- * UILogCollector captures logs during user prompt processing
1381
- * and sends them to runtime via ui_logs message with uiBlockId as the message id
1382
- * Logs are sent in real-time for streaming effect in the UI
1383
- * Respects the global log level configuration
1324
+ * StreamBuffer - Buffered streaming utility for smoother text delivery
1325
+ * Batches small chunks together and flushes at regular intervals
1384
1326
  */
1385
- declare class UILogCollector {
1386
- private logs;
1387
- private uiBlockId;
1388
- private clientId;
1389
- private sendMessage;
1390
- private currentLogLevel;
1391
- constructor(clientId: string, sendMessage: (message: Message) => void, uiBlockId?: string);
1392
- /**
1393
- * Check if logging is enabled (uiBlockId is provided)
1394
- */
1395
- isEnabled(): boolean;
1396
- /**
1397
- * Check if a message should be logged based on current log level
1398
- */
1399
- private shouldLog;
1400
- /**
1401
- * Add a log entry with timestamp and immediately send to runtime
1402
- * Only logs that pass the log level filter are captured and sent
1403
- */
1404
- private addLog;
1405
- /**
1406
- * Send a single log to runtime immediately
1407
- */
1408
- private sendLogImmediately;
1409
- /**
1410
- * Log info message
1411
- */
1412
- info(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
1413
- /**
1414
- * Log error message
1415
- */
1416
- error(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
1417
- /**
1418
- * Log warning message
1419
- */
1420
- warn(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
1421
- /**
1422
- * Log debug message
1423
- */
1424
- debug(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
1327
+ type StreamCallback = (chunk: string) => void;
1328
+ /**
1329
+ * StreamBuffer class for managing buffered streaming output
1330
+ * Provides smooth text delivery by batching small chunks
1331
+ */
1332
+ declare class StreamBuffer {
1333
+ private buffer;
1334
+ private flushTimer;
1335
+ private callback;
1336
+ private fullText;
1337
+ constructor(callback?: StreamCallback);
1425
1338
  /**
1426
- * Log LLM explanation with typed metadata
1339
+ * Check if the buffer has a callback configured
1427
1340
  */
1428
- logExplanation(message: string, explanation: string, data?: Record<string, any>): void;
1341
+ hasCallback(): boolean;
1429
1342
  /**
1430
- * Log generated query with typed metadata
1343
+ * Get all text that has been written (including already flushed)
1431
1344
  */
1432
- logQuery(message: string, query: string, data?: Record<string, any>): void;
1345
+ getFullText(): string;
1433
1346
  /**
1434
- * Send all collected logs at once (optional, for final summary)
1347
+ * Write a chunk to the buffer
1348
+ * Large chunks or chunks with newlines are flushed immediately
1349
+ * Small chunks are batched and flushed after a short interval
1350
+ *
1351
+ * @param chunk - Text chunk to write
1435
1352
  */
1436
- sendAllLogs(): void;
1353
+ write(chunk: string): void;
1437
1354
  /**
1438
- * Get all collected logs
1355
+ * Flush the buffer immediately
1356
+ * Call this before tool execution or other operations that need clean output
1439
1357
  */
1440
- getLogs(): CapturedLog[];
1358
+ flush(): void;
1441
1359
  /**
1442
- * Clear all logs
1360
+ * Internal flush implementation
1443
1361
  */
1444
- clearLogs(): void;
1362
+ private flushNow;
1445
1363
  /**
1446
- * Set uiBlockId (in case it's provided later)
1364
+ * Clean up resources
1365
+ * Call this when done with the buffer
1447
1366
  */
1448
- setUIBlockId(uiBlockId: string): void;
1367
+ dispose(): void;
1449
1368
  }
1450
1369
 
1451
1370
  /**
1452
- * Represents an action that can be performed on a UIBlock
1371
+ * ToolExecutorService - Handles execution of SQL queries and external tools
1372
+ * Extracted from BaseLLM.generateTextResponse for better separation of concerns
1453
1373
  */
1454
- interface Action {
1374
+
1375
+ /**
1376
+ * External tool definition
1377
+ */
1378
+ interface ExternalTool {
1455
1379
  id: string;
1456
1380
  name: string;
1457
- type: string;
1458
- [key: string]: any;
1381
+ description?: string;
1382
+ /** Tool type: "source" = routed through SourceAgent, "direct" = called directly by MainAgent */
1383
+ toolType?: 'source' | 'direct';
1384
+ /** Full untruncated schema for source agent (all columns visible) */
1385
+ fullSchema?: string;
1386
+ /** Schema size tier: small (≤50 tables), medium (51-200), large (201-500), very_large (500+) */
1387
+ schemaTier?: string;
1388
+ /** Schema search function for very_large tier — keyword search over entities */
1389
+ schemaSearchFn?: (keywords: string[]) => string;
1390
+ fn: (input: any) => Promise<any>;
1391
+ limit?: number;
1392
+ outputSchema?: any;
1393
+ executionType?: 'immediate' | 'deferred';
1394
+ userProvidedData?: any;
1395
+ params?: Record<string, any>;
1396
+ }
1397
+ /**
1398
+ * Executed tool tracking info
1399
+ */
1400
+ interface ExecutedToolInfo {
1401
+ id: string;
1402
+ name: string;
1403
+ params: any;
1404
+ result: {
1405
+ _totalRecords: number;
1406
+ _recordsShown: number;
1407
+ _metadata?: any;
1408
+ _sampleData: any[];
1409
+ /** Bounded summary over the FULL fetched result (complete structure). */
1410
+ _summary?: any;
1411
+ /** Up to MAIN_AGENT_COMPLETE_ROWS rows — the complete result when small. */
1412
+ _mainAgentRows?: any[];
1413
+ };
1414
+ outputSchema?: any;
1415
+ sourceSchema?: string;
1416
+ sourceType?: string;
1459
1417
  }
1460
1418
 
1461
1419
  /**
1462
- * UIBlock represents a single user and assistant message block in a thread
1463
- * Contains user question, component metadata, component data, text response, and available actions
1420
+ * Multi-Agent Architecture Types
1421
+ *
1422
+ * Defines interfaces for the hierarchical agent system:
1423
+ * - Main Agent: ONE LLM.streamWithTools() call with source agent tools
1424
+ * - Source Agents: independent agents that query individual data sources
1425
+ *
1426
+ * The main agent sees only source summaries. When it calls a source tool,
1427
+ * the SourceAgent runs independently (own LLM, own retries) and returns clean data.
1464
1428
  */
1465
- declare class UIBlock {
1466
- private id;
1467
- private userQuestion;
1468
- private generatedComponentMetadata;
1469
- private componentData;
1470
- private textResponse;
1471
- private actions;
1472
- private createdAt;
1473
- /**
1474
- * Creates a new UIBlock instance
1475
- * @param userQuestion - The user's question or input
1476
- * @param componentData - The component data object
1477
- * @param generatedComponentMetadata - Optional metadata about the generated component
1478
- * @param actions - Optional array of available actions
1479
- * @param id - Optional custom ID, generates UUID if not provided
1480
- * @param textResponse - Optional text response from LLM
1481
- */
1482
- constructor(userQuestion: string, componentData?: Record<string, any>, generatedComponentMetadata?: Record<string, any>, actions?: Action[], id?: string, textResponse?: string | null);
1483
- /**
1484
- * Get the UIBlock ID
1485
- */
1486
- getId(): string;
1487
- /**
1488
- * Get the user question
1429
+
1430
+ /**
1431
+ * Per-entity detail: name, row count, and column names.
1432
+ * Gives the main agent enough context to route to the right source.
1433
+ */
1434
+ interface EntityDetail {
1435
+ /** Entity name (table, sheet, endpoint) */
1436
+ name: string;
1437
+ /** Approximate row count */
1438
+ rowCount?: number;
1439
+ /** Column/field names */
1440
+ columns: string[];
1441
+ }
1442
+ /**
1443
+ * Representation of a data source for the main agent.
1444
+ * Contains entity names WITH column names so the LLM can route accurately.
1445
+ */
1446
+ interface SourceSummary {
1447
+ /** Source ID (matches tool ID prefix) */
1448
+ id: string;
1449
+ /** Human-readable source name */
1450
+ name: string;
1451
+ /** Source type: postgres, excel, rest_api, etc. */
1452
+ type: string;
1453
+ /** Brief description of what data this source contains */
1454
+ description: string;
1455
+ /** Detailed entity info with column names for routing */
1456
+ entityDetails: EntityDetail[];
1457
+ /** The tool ID associated with this source */
1458
+ toolId: string;
1459
+ }
1460
+ /**
1461
+ * What a source agent returns after querying its data source.
1462
+ * The main agent uses this to analyze and compose the final response.
1463
+ */
1464
+ interface SourceAgentResult {
1465
+ /** Source ID */
1466
+ sourceId: string;
1467
+ /** Source name */
1468
+ sourceName: string;
1469
+ /** Whether the query succeeded */
1470
+ success: boolean;
1471
+ /** Result data rows */
1472
+ data: any[];
1473
+ /** Metadata about the query execution */
1474
+ metadata: SourceAgentMetadata;
1475
+ /** Tool execution info for the last successful query (backward compat) */
1476
+ executedTool: ExecutedToolInfo;
1477
+ /** All successful tool executions (primary + follow-up queries) */
1478
+ allExecutedTools?: ExecutedToolInfo[];
1479
+ /** Error message if failed */
1480
+ error?: string;
1481
+ }
1482
+ interface SourceAgentMetadata {
1483
+ /** Total rows that matched the query (before limit) */
1484
+ totalRowsMatched: number;
1485
+ /** Rows actually returned (after limit) */
1486
+ rowsReturned: number;
1487
+ /** Whether the result was truncated by the row limit */
1488
+ isLimited: boolean;
1489
+ /** The query/params that were executed */
1490
+ queryExecuted?: string;
1491
+ /** Execution time in milliseconds */
1492
+ executionTimeMs: number;
1493
+ }
1494
+ /**
1495
+ * A pre-built, multi-step UI flow registered with the SDK.
1496
+ *
1497
+ * When the main agent decides a user's question matches a workflow's whenToUse
1498
+ * trigger, it picks the workflow instead of running source agents / generating
1499
+ * dashboard components. The LLM extracts the workflow's required props from the
1500
+ * prompt (using `propsSchema` as the tool input_schema) and the SDK returns the
1501
+ * workflow component directly — no analysis text, no chart generation. The
1502
+ * frontend renders the registered workflow component with the LLM-extracted
1503
+ * props.
1504
+ */
1505
+ interface WorkflowDescriptor {
1506
+ /** Unique workflow id (used as the LLM tool name) */
1507
+ id: string;
1508
+ /** Component name on the frontend (matches the registered React component) */
1509
+ name: string;
1510
+ /** Short human-readable description of what this workflow does */
1511
+ description: string;
1512
+ /**
1513
+ * 1–2 sentence trigger condition. The LLM uses this to decide if the
1514
+ * user's prompt matches this workflow. Be specific — e.g.
1515
+ * "User wants to *initiate* an inventory transfer (review + submit POs),
1516
+ * not just see analysis or charts."
1517
+ */
1518
+ whenToUse: string;
1519
+ /**
1520
+ * JSON-schema-style description of the props the workflow needs. Becomes
1521
+ * the LLM tool's input_schema, so the model fills these from the prompt.
1522
+ * Use the same shape as `params` on direct tools — string descriptors with
1523
+ * an optional "(optional)" suffix.
1524
+ *
1525
+ * Example:
1526
+ * ```
1527
+ * {
1528
+ * selectedStore: 'object — { id, name } of the source branch',
1529
+ * minROI: 'number (optional) — only show transfers with ROI ≥ this',
1530
+ * }
1531
+ * ```
1532
+ */
1533
+ propsSchema: Record<string, string>;
1534
+ /**
1535
+ * Optional: static prop defaults merged with LLM-extracted props before
1536
+ * the component is returned. Useful for things like the embedded
1537
+ * `externalTool` config that the workflow uses to fetch its own data.
1538
+ */
1539
+ defaultProps?: Record<string, any>;
1540
+ }
1541
+ /**
1542
+ * The workflow selection captured during a routing call.
1543
+ * Set on AgentResponse when the LLM picks a workflow tool.
1544
+ */
1545
+ interface SelectedWorkflow {
1546
+ /** Component name (matches WorkflowDescriptor.name) */
1547
+ name: string;
1548
+ /** Props extracted from the prompt + merged with workflow.defaultProps */
1549
+ props: Record<string, any>;
1550
+ }
1551
+ /**
1552
+ * The complete response from the multi-agent system.
1553
+ * Contains everything needed for text display + component generation.
1554
+ */
1555
+ interface AgentResponse {
1556
+ /** Generated text response (analysis of the data) */
1557
+ text: string;
1558
+ /** All executed tools across all source agents (for component generation) */
1559
+ executedTools: ExecutedToolInfo[];
1560
+ /** Individual results from each source agent */
1561
+ sourceResults: SourceAgentResult[];
1562
+ /**
1563
+ * Populated when MainAgent wrote AND successfully executed a script during its turn.
1564
+ * Caller (agent-user-response.ts) persists it via ScriptStore.save().
1565
+ * Absent when MainAgent didn't write one (trivial question / all attempts failed).
1566
+ */
1567
+ savedScript?: AgentWrittenScript;
1568
+ /**
1569
+ * Set when the LLM routed the question to a registered workflow component.
1570
+ * When present, the upstream caller should skip component generation and
1571
+ * return this workflow as the response.
1572
+ */
1573
+ workflow?: SelectedWorkflow;
1574
+ }
1575
+ /**
1576
+ * A script MainAgent authored + verified during its turn. Shape aligns with
1577
+ * what ScriptStore.save() needs — minus store-assigned fields (id, timestamps, counts).
1578
+ */
1579
+ interface AgentWrittenScript {
1580
+ /**
1581
+ * `ScriptRecipe.id` of the draft that was authored + verified during this turn.
1582
+ * The caller passes this to `ScriptStore.promoteToVerified(recipeId, …)` to
1583
+ * flip the draft to verified status and (when possible) drop the turn-suffix
1584
+ * from its filename.
1585
+ */
1586
+ recipeId: string;
1587
+ name: string;
1588
+ intentDescription: string;
1589
+ tags: string[];
1590
+ parameters: Array<{
1591
+ name: string;
1592
+ type: 'string' | 'number' | 'date' | 'date_range' | 'enum' | 'boolean';
1593
+ required: boolean;
1594
+ default?: any;
1595
+ enumValues?: Record<string, string>;
1596
+ description: string;
1597
+ }>;
1598
+ scriptBody: string;
1599
+ /** Source IDs referenced by the script (extracted from ctx.query calls) */
1600
+ sourceIds: string[];
1601
+ /** Tables referenced in the script's SQL (regex-extracted) */
1602
+ tables: string[];
1603
+ /** Executed queries from the verified run — fed to component generation */
1604
+ executedQueries: Array<{
1605
+ sourceId: string;
1606
+ sourceName: string;
1607
+ sql: string;
1608
+ data: any[];
1609
+ count: number;
1610
+ totalCount?: number;
1611
+ executionTimeMs: number;
1612
+ /**
1613
+ * True for synthetic entries (ctx.emit datasets, the computed:_final
1614
+ * post-JS data). The component generator routes virtual sources through
1615
+ * the script_dataset sentinel toolId so the frontend resolves them via
1616
+ * queryCache instead of attempting to re-execute SQL.
1617
+ */
1618
+ virtual?: boolean;
1619
+ }>;
1620
+ }
1621
+ /**
1622
+ * Configuration for the multi-agent system.
1623
+ * Controls limits, models, and behavior.
1624
+ */
1625
+ interface AgentConfig {
1626
+ /** Max rows shown to the UI preview / inlined per source (default: 10) */
1627
+ maxRowsPerSource: number;
1628
+ /**
1629
+ * Max rows a source query may FETCH from the DB server-side (default: 2000).
1630
+ * Decoupled from what the main agent is shown: the full result is fetched and
1631
+ * summarized (bounded), but only a small/complete slice enters LLM context.
1632
+ * This lets small lookups (benchmark maps) arrive COMPLETE without letting
1633
+ * large results blow up context.
1634
+ */
1635
+ maxRowsFetched: number;
1636
+ /** Model for the main agent (routing + analysis in one LLM call) */
1637
+ mainAgentModel: string;
1638
+ /** Model for source agent query generation */
1639
+ sourceAgentModel: string;
1640
+ /** API key for LLM calls */
1641
+ apiKey?: string;
1642
+ /** Max retry attempts per source agent */
1643
+ maxRetries: number;
1644
+ /** Max tool calling iterations for the main agent loop */
1645
+ maxIterations: number;
1646
+ /** Global knowledge base context (static, same for all users/questions — cached in system prompt) */
1647
+ globalKnowledgeBase?: string;
1648
+ /** Per-request knowledge base context (user-specific + query-matched — dynamic, not cached) */
1649
+ knowledgeBaseContext?: string;
1650
+ /** Collections registry (ChromaDB search hooks) for embedding-based schema + source search */
1651
+ collections?: any;
1652
+ /** Optional project ID for scoping embedding searches */
1653
+ projectId?: string;
1654
+ }
1655
+ /**
1656
+ * Default agent configuration
1657
+ */
1658
+ declare const DEFAULT_AGENT_CONFIG: AgentConfig;
1659
+
1660
+ /**
1661
+ * Script Flow Types
1662
+ *
1663
+ * Defines interfaces for the script-based query architecture:
1664
+ * - ScriptRecipe: metadata for matching, validation, and quality tracking
1665
+ * - ScriptResult: output from executing a script
1666
+ * - ScriptMatch: result from the LLM-based script matcher
1667
+ */
1668
+ /**
1669
+ * Recipe metadata stored alongside each script.
1670
+ * Used for matching, validation, and quality tracking.
1671
+ */
1672
+ interface ScriptRecipe {
1673
+ /** Unique script identifier */
1674
+ id: string;
1675
+ /** Version number (incremented on regeneration) */
1676
+ version: number;
1677
+ /** Human-readable name (e.g., "Revenue by Dimension") */
1678
+ name: string;
1679
+ /** Natural language description of what this script does */
1680
+ intentDescription: string;
1681
+ /** Keyword tags for quick filtering */
1682
+ tags: string[];
1683
+ /** Source tool IDs this script queries (e.g., ["mssql-abc123_query"]) */
1684
+ sourceIds: string[];
1685
+ /** Table names used (for future schema drift detection) */
1686
+ tables: string[];
1687
+ /** Parameter definitions — what can vary */
1688
+ parameters: ScriptParameter[];
1689
+ /** The script function body as a string. Loaded from disk (scripts-store/<fileBase>.ts). */
1690
+ scriptBody: string;
1691
+ /**
1692
+ * On-disk filename stem for the body: scripts-store/<fileBase>.ts.
1693
+ * Editable in the IDE. Decided at authoring time (slug of `name`, with a
1694
+ * short id suffix on collision) and stable across promotion.
1695
+ */
1696
+ fileBase?: string;
1697
+ /** sha256 of the on-disk body — lets the runtime detect manual edits. */
1698
+ bodyHash?: string;
1699
+ /** Project scope (single-VM deployments may leave this undefined). */
1700
+ projectId?: string;
1701
+ /** Times this script was used successfully */
1702
+ successCount: number;
1703
+ /** Times this script failed */
1704
+ failureCount: number;
1705
+ /** ISO timestamp of last usage */
1706
+ lastUsed: string;
1707
+ /** Original user question that created this script */
1708
+ createdFrom: string;
1709
+ /** ISO timestamp */
1710
+ createdAt: string;
1711
+ /** ISO timestamp */
1712
+ updatedAt: string;
1713
+ /**
1714
+ * `recipe.id` of the parent this script was forked from.
1715
+ * Undefined for root scripts (those written from scratch by MainAgent).
1716
+ * See backend/docs/SCRIPT-FLOW-FORK-ADAPT.md.
1717
+ */
1718
+ parentId?: string;
1719
+ /** 0 for root scripts; `parent.forkDepth + 1` for forks. Capped at 3. */
1720
+ forkDepth?: number;
1721
+ /**
1722
+ * Brief description of what this fork changed vs its parent
1723
+ * (sourced from the matcher's `modificationHint`).
1724
+ */
1725
+ forkReason?: string;
1726
+ /**
1727
+ * Validated component specs captured at authoring time. On a tier-high
1728
+ * replay these are rebound to fresh queryIds deterministically — no
1729
+ * component-generation LLM call, and the rendered columns can't drift from
1730
+ * what was validated when the script was authored. Absent on recipes
1731
+ * authored before this landed; those fall back to LLM component generation.
1732
+ * See backend/docs/SCRIPT-COMPONENT-CONSISTENCY.md.
1733
+ */
1734
+ components?: ScriptComponentSpec[];
1735
+ /**
1736
+ * Lifecycle stage of this recipe on disk.
1737
+ * - 'draft': written by MainAgent's write_script during a turn; filtered out
1738
+ * of FTS results (status='verified' only) so the matcher never picks it.
1739
+ * Filename is suffixed with `turnId` to keep concurrent turns
1740
+ * from clobbering each other's drafts.
1741
+ * - 'verified': promoted after `execute_script` succeeded; the matcher sees it.
1742
+ * Filename drops the turn suffix unless a verified file with
1743
+ * the same slug already exists (collision case keeps the suffix).
1744
+ *
1745
+ * Recipes loaded from disk without this field default to 'verified' so
1746
+ * existing scripts keep working unchanged.
1747
+ */
1748
+ status?: 'draft' | 'verified';
1749
+ /**
1750
+ * Per-turn unique suffix used for draft filenames (e.g. `1714745623-x9k2`).
1751
+ * Set when the draft is saved; carried until the recipe is promoted.
1752
+ */
1753
+ turnId?: string;
1754
+ /**
1755
+ * Last execution error captured by `recordDraftError` while the recipe was
1756
+ * still a draft. Lets users open the draft .json file and see why it failed
1757
+ * without grepping logs. Cleared on promotion to 'verified'.
1758
+ */
1759
+ lastError?: {
1760
+ phase: 'compile' | 'runtime' | 'timeout' | 'ipc';
1761
+ message: string;
1762
+ at: string;
1763
+ attempt: number;
1764
+ };
1765
+ }
1766
+ interface ScriptParameter {
1767
+ /** Parameter name (used in script body as params.name) */
1768
+ name: string;
1769
+ /** Parameter type */
1770
+ type: 'string' | 'number' | 'date' | 'date_range' | 'enum' | 'boolean';
1771
+ /** Whether this parameter is required */
1772
+ required: boolean;
1773
+ /** Default value if not provided */
1774
+ default?: any;
1775
+ /** For enum type — maps user-facing values to internal values */
1776
+ enumValues?: Record<string, string>;
1777
+ /** Human-readable description (used in the matcher LLM prompt) */
1778
+ description: string;
1779
+ }
1780
+ /**
1781
+ * A reusable component binding captured when a script is authored. Stored on
1782
+ * the recipe so tier-high replays rebuild components deterministically (rebind
1783
+ * to fresh queryIds) instead of re-running the component-picker LLM.
1784
+ */
1785
+ interface ScriptComponentSpec {
1786
+ /** Registered component name (e.g. "DynamicBarChart") — matched against the available component library. */
1787
+ componentType: string;
1788
+ /** `executedQuery.sourceId` to bind to (e.g. a tool id or 'computed:_final'), 'federation' for a cross-source component, or 'markdown' for a content-only narrative block (no data source). */
1789
+ sourceRef: string;
1790
+ /** Present only when sourceRef === 'federation' — the DuckDB SQL to re-execute on replay. */
1791
+ federationSql?: string;
1792
+ /** Present only when sourceRef === 'markdown' — the narrative text to render on replay (markdown has no data source, so its content must be persisted). */
1793
+ content?: string;
1794
+ title?: string;
1795
+ description?: string;
1796
+ /** Validated axis/value keys + aggregation — all referencing real columns of the bound source. */
1797
+ config: Record<string, any>;
1798
+ }
1799
+ /**
1800
+ * Result from executing a script via ScriptRunner.
1801
+ */
1802
+ interface ScriptResult {
1803
+ /** Whether the script executed successfully */
1804
+ success: boolean;
1805
+ /** Combined data from all queries */
1806
+ data: any[];
1807
+ /** Individual query results tracked during execution */
1808
+ executedQueries: ScriptQueryResult[];
1809
+ /** Error message if failed */
1810
+ error?: string;
1811
+ /**
1812
+ * Where in the lifecycle the error occurred. Lets MainAgent's fix-loop
1813
+ * decide between "rewrite the whole draft" (compile) and "patch the
1814
+ * specific line" (runtime).
1815
+ */
1816
+ errorPhase?: 'compile' | 'runtime' | 'timeout' | 'ipc';
1817
+ /** Total execution time in milliseconds */
1818
+ executionTimeMs: number;
1819
+ }
1820
+ /**
1821
+ * A single query executed during script runtime.
1822
+ * Tracked by ScriptContext for component generation and debugging.
1823
+ */
1824
+ interface ScriptQueryResult {
1825
+ /** Source tool ID */
1826
+ sourceId: string;
1827
+ /** Human-readable source name */
1828
+ sourceName: string;
1829
+ /** The SQL that was executed */
1830
+ sql: string;
1831
+ /** Result data rows */
1832
+ data: any[];
1833
+ /** Number of rows returned */
1834
+ count: number;
1835
+ /** Total rows that matched before limit (if available) */
1836
+ totalCount?: number;
1837
+ /** Query execution time in milliseconds */
1838
+ executionTimeMs: number;
1839
+ /**
1840
+ * True for rows that did NOT come from a real SQL execution — either a
1841
+ * ctx.emit() dataset or the synthesized "computed:_final" entry that
1842
+ * carries the script's post-JS returned data. The component generator
1843
+ * uses this to route the resulting component through the script_dataset
1844
+ * sentinel toolId so the frontend resolves it via the queryCache short-circuit.
1845
+ */
1846
+ virtual?: boolean;
1847
+ }
1848
+ /**
1849
+ * Match tier returned by the LLM script matcher.
1850
+ *
1851
+ * - 'high': the script answers the question directly; only parameter values
1852
+ * may differ. The runtime replays it with extracted params (cheapest path).
1853
+ * - 'near': the script answers a STRUCTURALLY similar question but needs
1854
+ * body modification (different metric, dimension, table, filter shape).
1855
+ * The runtime forks the parent and adapts the body via MainAgent's normal
1856
+ * write_script + execute_script loop — no SourceAgent dispatch needed.
1857
+ * See backend/docs/SCRIPT-FLOW-FORK-ADAPT.md for the full design.
1858
+ * - 'none': no script is relevant; full agent flow runs.
1859
+ */
1860
+ type MatchTier = 'high' | 'near' | 'none';
1861
+ /**
1862
+ * Result from the LLM-based script matcher.
1863
+ *
1864
+ * For `tier: 'high'`, `extractedParams` carries the values to pass to the
1865
+ * existing script. For `tier: 'near'`, `gaps` and `modificationHint` describe
1866
+ * what the fork-author needs to change in the parent body.
1867
+ */
1868
+ interface ScriptMatch {
1869
+ /** The matched script recipe */
1870
+ recipe: ScriptRecipe;
1871
+ /** Match tier — see MatchTier docs */
1872
+ tier: MatchTier;
1873
+ /** Similarity score (0-1, derived from LLM tier) */
1874
+ similarity: number;
1875
+ /**
1876
+ * Legacy confidence level. Mirrors `tier === 'high'`/`'near'` for now;
1877
+ * kept so existing callers compile while we migrate to tier-based logic.
1878
+ */
1879
+ confidence: 'high' | 'medium';
1880
+ /** Parameters extracted from the user question by the LLM (tier='high') */
1881
+ extractedParams?: Record<string, any>;
1882
+ /** What the user question needs that the parent doesn't cover (tier='near') */
1883
+ gaps?: string[];
1884
+ /** One-sentence description of the change the fork-author should make (tier='near') */
1885
+ modificationHint?: string;
1886
+ /** Why the matcher made this choice (for logs and telemetry) */
1887
+ reasoning?: string;
1888
+ }
1889
+
1890
+ /**
1891
+ * ScriptRecipeStore — injected metadata backend for the script flow.
1892
+ *
1893
+ * The SDK is standalone (no DB dependency). The backend implements this
1894
+ * interface over Postgres (full-text search + atomic counters) and injects it
1895
+ * via `collections['script-recipes']`, exactly like `collections['source-embeddings']`.
1896
+ * `ScriptStore` consumes it for all METADATA operations while keeping the
1897
+ * executable body on disk as scripts-store/<fileBase>.ts.
1898
+ *
1899
+ * All metadata rows are plain JSON (no scriptBody — that lives on disk).
1900
+ * See backend/docs/SCRIPT-FLOW-SCALING-ISSUES.md (#1, #3, #7).
1901
+ */
1902
+
1903
+ /** One recipe's metadata as stored in Postgres (mirrors the script_recipes table). */
1904
+ interface ScriptRecipeMetaRow {
1905
+ id: string;
1906
+ projectId?: string | null;
1907
+ version: number;
1908
+ name: string;
1909
+ intentDescription: string;
1910
+ tags: string[] | null;
1911
+ createdFrom: string | null;
1912
+ sourceIds: string[] | null;
1913
+ tables: string[] | null;
1914
+ parameters: ScriptParameter[] | null;
1915
+ components?: ScriptComponentSpec[] | null;
1916
+ fileBase: string;
1917
+ bodyHash?: string | null;
1918
+ successCount: number;
1919
+ failureCount: number;
1920
+ lastUsed: string | null;
1921
+ parentId?: string | null;
1922
+ forkDepth?: number | null;
1923
+ forkReason?: string | null;
1924
+ status: 'draft' | 'verified' | string;
1925
+ turnId?: string | null;
1926
+ lastError?: {
1927
+ phase: 'compile' | 'runtime' | 'timeout' | 'ipc';
1928
+ message: string;
1929
+ at: string;
1930
+ attempt: number;
1931
+ } | null;
1932
+ createdAt?: string | null;
1933
+ updatedAt?: string | null;
1934
+ }
1935
+ interface ScriptRecipeStore {
1936
+ /** FTS shortlist of healthy verified recipes for the matcher (metadata only). */
1937
+ search(params: {
1938
+ prompt: string;
1939
+ projectId?: string;
1940
+ limit?: number;
1941
+ }): Promise<ScriptRecipeMetaRow[]>;
1942
+ /** Fetch one recipe by id (any status). */
1943
+ getById(id: string): Promise<ScriptRecipeMetaRow | null>;
1944
+ /** Count healthy verified recipes (drives the "any scripts?" gate). */
1945
+ count(params?: {
1946
+ projectId?: string;
1947
+ }): Promise<number>;
1948
+ /** Insert or update a recipe row (keyed by id). */
1949
+ upsert(row: ScriptRecipeMetaRow): Promise<void>;
1950
+ /** Atomically bump counters / last-used. */
1951
+ updateStats(id: string, patch: {
1952
+ successDelta?: number;
1953
+ failureDelta?: number;
1954
+ lastUsed?: string;
1955
+ }): Promise<void>;
1956
+ /** Flip a draft to verified, applying provenance + optional fork lineage. */
1957
+ promote(id: string, patch: {
1958
+ sourceIds: string[];
1959
+ tables: string[];
1960
+ fileBase?: string;
1961
+ parentId?: string;
1962
+ forkDepth?: number;
1963
+ forkReason?: string;
1964
+ components?: ScriptComponentSpec[];
1965
+ }): Promise<ScriptRecipeMetaRow | null>;
1966
+ /** Stamp a draft's last execution error. */
1967
+ recordDraftError(id: string, err: {
1968
+ phase: string;
1969
+ message: string;
1970
+ attempt: number;
1971
+ at: string;
1972
+ }): Promise<void>;
1973
+ /** Delete a recipe row (body file removed separately). */
1974
+ remove(id: string): Promise<void>;
1975
+ /** True if `fileBase` is taken by a different recipe in this project. */
1976
+ fileBaseTaken(fileBase: string, excludeId: string, projectId?: string): Promise<boolean>;
1977
+ }
1978
+ /** Pull the injected store off the collections bag (or null if not wired). */
1979
+ declare function resolveScriptRecipeStore(collections: any): ScriptRecipeStore | null;
1980
+
1981
+ /**
1982
+ * ScriptStore — Postgres metadata + on-disk body for script recipes.
1983
+ *
1984
+ * Split of responsibilities:
1985
+ * - METADATA → injected `ScriptRecipeStore` (Postgres FTS + atomic counters),
1986
+ * resolved from `collections['script-recipes']`.
1987
+ * - BODY → scripts-store/<fileBase>.ts, editable in your IDE. Written
1988
+ * atomically (temp + rename); `bodyHash` (sha256) detects edits.
1989
+ *
1990
+ * The old "read every file every turn + send the whole catalog to the LLM"
1991
+ * matcher is gone — matching is `store.search(prompt)` (FTS shortlist). The
1992
+ * draft/verified filename dance is gone too: `status` is a DB column and the
1993
+ * file keeps a stable `<fileBase>.ts` name across promotion.
1994
+ *
1995
+ * When no metadata store is injected, the store degrades to a safe no-op
1996
+ * (count 0 → script flow disabled) instead of crashing.
1997
+ *
1998
+ * See backend/docs/SCRIPT-FLOW-SCALING-ISSUES.md.
1999
+ */
2000
+
2001
+ interface SaveDraftInput {
2002
+ /** Reuse an existing draft (retry); omit to mint a new one. */
2003
+ recipeId?: string;
2004
+ /** Per-turn unique suffix, stable across retries within the turn. */
2005
+ turnId: string;
2006
+ name: string;
2007
+ intentDescription: string;
2008
+ tags: string[];
2009
+ parameters: ScriptParameter[];
2010
+ scriptBody: string;
2011
+ createdFrom: string;
2012
+ }
2013
+ interface PromoteToVerifiedInput {
2014
+ sourceIds: string[];
2015
+ tables: string[];
2016
+ parentId?: string;
2017
+ forkDepth?: number;
2018
+ forkReason?: string;
2019
+ components?: ScriptComponentSpec[];
2020
+ }
2021
+ interface ScriptStoreOptions {
2022
+ /** Explicit metadata store, or resolved from `collections['script-recipes']`. */
2023
+ store?: ScriptRecipeStore | null;
2024
+ collections?: any;
2025
+ /** Body directory (defaults to <cwd>/scripts-store). */
2026
+ baseDir?: string;
2027
+ /** Project scope stamped on every row. */
2028
+ projectId?: string;
2029
+ }
2030
+ /**
2031
+ * Normalize a scriptBody into the on-disk form (strip a leading comment block,
2032
+ * ensure `export async function getData`). Exported for MainAgent.
2033
+ */
2034
+ declare function normalizeScriptBody(scriptBody: string): string;
2035
+ declare class ScriptStore {
2036
+ private store;
2037
+ private storeDir;
2038
+ private projectId?;
2039
+ constructor(opts?: ScriptStoreOptions);
2040
+ /** Whether a metadata store is wired (matcher / authoring are gated on this). */
2041
+ hasStore(): boolean;
2042
+ /** Number of healthy verified recipes (gates the script-matching path). */
2043
+ count(): Promise<number>;
2044
+ /**
2045
+ * FTS shortlist for the matcher (metadata only — bodies are loaded lazily by
2046
+ * `get()` once the LLM picks one). Returns verified, healthy recipes ranked
2047
+ * by relevance.
2048
+ */
2049
+ search(prompt: string, limit?: number): Promise<ScriptRecipe[]>;
2050
+ /** Fetch one recipe by id with its body loaded from disk. */
2051
+ get(id: string): Promise<ScriptRecipe | null>;
2052
+ /** Create or update a recipe (metadata upsert + body write when changed). */
2053
+ save(recipe: ScriptRecipe): Promise<void>;
2054
+ /**
2055
+ * Persist (or update) a draft. Within a turn, retries that pass the same
2056
+ * `recipeId` overwrite the same row + file; a fresh `recipeId` mints a new
2057
+ * draft. The body is visible at scripts-store/<fileBase>.ts immediately.
2058
+ */
2059
+ saveDraft(input: SaveDraftInput): Promise<ScriptRecipe>;
2060
+ /** Stamp a draft's last execution error (metadata only). */
2061
+ recordDraftError(recipeId: string, err: {
2062
+ phase: 'compile' | 'runtime' | 'timeout' | 'ipc';
2063
+ message: string;
2064
+ attempt: number;
2065
+ }): Promise<void>;
2066
+ /**
2067
+ * Promote a successfully-executed draft into a verified script.
2068
+ * The on-disk body already exists at <fileBase>.ts (written at write_script
2069
+ * time) and keeps its name — only the DB row flips status + provenance.
2070
+ */
2071
+ promoteToVerified(recipeId: string, input: PromoteToVerifiedInput): Promise<ScriptRecipe | null>;
2072
+ /**
2073
+ * Drop a draft (row + body file). MainAgent calls this at end-of-turn when a
2074
+ * draft was authored but never verified — failed drafts are never matched, so
2075
+ * deleting them immediately avoids unbounded accumulation (#5). No-op if the
2076
+ * recipe isn't a draft (so a promoted/verified script is never removed here).
2077
+ */
2078
+ discardDraft(recipeId: string): Promise<void>;
2079
+ /** Delete a recipe (row + body file). */
2080
+ delete(id: string): Promise<void>;
2081
+ /** Record a successful execution (atomic counter bump). */
2082
+ recordSuccess(id: string): Promise<void>;
2083
+ /** Record a failed execution (atomic counter bump). */
2084
+ recordFailure(id: string): Promise<void>;
2085
+ /** Absolute path to the .ts body for a recipe (used by the runner/MainAgent). */
2086
+ getScriptPath(recipe: ScriptRecipe): string;
2087
+ private removeById;
2088
+ private rowToRecipe;
2089
+ private recipeToRow;
2090
+ /** slug of name, with a short id suffix when the bare slug is already taken. */
2091
+ private computeFileBase;
2092
+ private toSlug;
2093
+ private hash;
2094
+ private bodyPath;
2095
+ private readBody;
2096
+ /** Atomic body write (temp + rename) so concurrent reads never see a partial file. */
2097
+ private writeBody;
2098
+ private unlinkBody;
2099
+ }
2100
+
2101
+ /**
2102
+ * Main Agent (Orchestrator)
2103
+ *
2104
+ * A single LLM.streamWithTools() call that handles everything:
2105
+ * - Routing: decides which source(s) to query based on summaries
2106
+ * - Querying: calls source tools (each wraps an independent SourceAgent)
2107
+ * - Direct tools: calls pre-built function tools directly with LLM-provided params
2108
+ * - Re-querying: if data is wrong/incomplete, calls tools again with modified intent
2109
+ * - Analysis: generates final text response from the data
2110
+ *
2111
+ * Two tool types:
2112
+ * - "source" tools: main agent sees summaries, SourceAgent handles SQL generation independently
2113
+ * - "direct" tools: main agent calls fn() directly with structured params (no SourceAgent)
2114
+ */
2115
+
2116
+ declare class MainAgent {
2117
+ private externalTools;
2118
+ private workflows;
2119
+ private config;
2120
+ private streamBuffer;
2121
+ /**
2122
+ * Optional: when provided, MainAgent exposes the `write_script` /
2123
+ * `execute_script` tools to the LLM and persists drafts to disk via the
2124
+ * store. Headless callers (alert analyzer, metric resolver) omit these to
2125
+ * suppress script authoring entirely — drafts would otherwise leak onto
2126
+ * disk with no caller to promote or clean them up.
2127
+ */
2128
+ private scriptStore;
2129
+ private turnId;
2130
+ private createdFromPrompt;
2131
+ private scriptState;
2132
+ /**
2133
+ * Fork mode — set when this turn is adapting a near-matching parent script.
2134
+ * In fork mode there is no legitimate "answer with bare text" outcome: the
2135
+ * only correct first move is a tool call (write_script, or a source tool for
2136
+ * schema discovery). We therefore force tool use on the first LLM iteration
2137
+ * so the model can't end its turn with a bare "I'll adapt…" preamble and zero
2138
+ * tool calls. Never set on the fresh-authoring / general-question path.
2139
+ */
2140
+ private forkMode;
2141
+ /**
2142
+ * Per-turn cancellation signal (user hit "Stop"). Set at the top of
2143
+ * handleQuestion and read by the tool handler, the SourceAgent dispatch, and
2144
+ * the script subprocess so an abort tears down every layer of the turn.
2145
+ */
2146
+ private abortSignal?;
2147
+ constructor(externalTools: ExternalTool[], config: AgentConfig, scriptStore?: ScriptStore, turnId?: string, streamBuffer?: StreamBuffer, workflows?: WorkflowDescriptor[], forkMode?: boolean);
2148
+ private get scriptingEnabled();
2149
+ /**
2150
+ * Handle a user question using the multi-agent system.
2151
+ *
2152
+ * This is ONE LLM.streamWithTools() call. The LLM:
2153
+ * 1. Sees source summaries + direct tool descriptions in system prompt
2154
+ * 2. Decides which tool(s) to call (routing)
2155
+ * 3. Source tools → SourceAgent runs independently → returns data
2156
+ * 4. Direct tools → fn() called directly with LLM params → returns data
2157
+ * 5. Generates final analysis text
2158
+ */
2159
+ handleQuestion(userPrompt: string, apiKey?: string, conversationHistory?: string, streamCallback?: (chunk: string) => void, signal?: AbortSignal): Promise<AgentResponse>;
2160
+ private handleWriteScript;
2161
+ private handleExecuteScript;
2162
+ /**
2163
+ * Build the AgentWrittenScript payload the caller will hand to
2164
+ * `ScriptStore.promoteToVerified()`. Only returned when a verified
2165
+ * successful execution is on record.
2166
+ */
2167
+ private buildSavedScript;
2168
+ private normalizeParameterList;
2169
+ /**
2170
+ * Use the schema embedding collection to pre-select relevant tables for
2171
+ * this source + intent. Returns a formatted schema block if confidence is
2172
+ * high (top match ≥ 0.55 and ≥3 candidates), otherwise null.
2173
+ *
2174
+ * When this returns a block, we can skip the SourceAgent's `search_schema`
2175
+ * loop and reduce iteration budget. When it returns null, the SourceAgent
2176
+ * falls back to the existing LLM-driven keyword search (same as today).
2177
+ */
2178
+ private preResolveSchema;
2179
+ /**
2180
+ * Execute a direct tool — call fn() with LLM-provided params, no SourceAgent.
2181
+ */
2182
+ private handleDirectTool;
2183
+ /**
2184
+ * Build the main agent's system prompt with source summaries, direct tool descriptions,
2185
+ * and workflow component descriptions.
2186
+ */
2187
+ private buildSystemPrompt;
2188
+ /**
2189
+ * Build tool definitions for source tools — summary-only descriptions.
2190
+ * The full schema is inside the SourceAgent which runs independently.
2191
+ */
2192
+ private buildSourceToolDefinitions;
2193
+ /**
2194
+ * Build tool definitions for direct tools — expose their actual params.
2195
+ * These are called directly by the main agent LLM, no SourceAgent.
2196
+ */
2197
+ private buildDirectToolDefinitions;
2198
+ /**
2199
+ * Capture a workflow selection. We do NOT execute anything — the LLM has
2200
+ * already extracted the props it wants the workflow rendered with. We
2201
+ * record the selection (via the capture callback) and return a short
2202
+ * acknowledgement so the LLM ends its turn cleanly without writing
2203
+ * analysis text or calling more tools.
2204
+ */
2205
+ private handleWorkflow;
2206
+ /**
2207
+ * Build LLM tool definitions for workflow components. The workflow's
2208
+ * propsSchema becomes the tool's input_schema so the LLM extracts props
2209
+ * directly from the prompt — same mechanic as direct tools.
2210
+ */
2211
+ private buildWorkflowToolDefinitions;
2212
+ /**
2213
+ * Format a source agent's result as a clean string for the main agent LLM.
2214
+ */
2215
+ private formatResultForMainAgent;
2216
+ /**
2217
+ * Get source summaries (for external inspection/debugging).
2218
+ */
2219
+ getSourceSummaries(): SourceSummary[];
2220
+ }
2221
+
2222
+ /**
2223
+ * Represents an action that can be performed on a UIBlock
2224
+ */
2225
+ interface Action {
2226
+ id: string;
2227
+ name: string;
2228
+ type: string;
2229
+ [key: string]: any;
2230
+ }
2231
+
2232
+ type SystemPrompt = string | Anthropic.Messages.TextBlockParam[];
2233
+ interface LLMMessages {
2234
+ sys: SystemPrompt;
2235
+ user: string;
2236
+ prefill?: string;
2237
+ }
2238
+ interface LLMOptions {
2239
+ model?: string;
2240
+ maxTokens?: number;
2241
+ temperature?: number;
2242
+ topP?: number;
2243
+ apiKey?: string;
2244
+ baseURL?: string;
2245
+ partial?: (chunk: string) => void;
2246
+ /**
2247
+ * Per-request cancellation. When the caller aborts this signal (user hit
2248
+ * "Stop"), the underlying provider request is cancelled and the call throws
2249
+ * a RequestAbortedError. Threaded into the provider `messages.create` request
2250
+ * options and checked between tool-loop iterations. Currently honored on the
2251
+ * Anthropic path (the agent flow's default provider).
2252
+ */
2253
+ signal?: AbortSignal;
2254
+ /**
2255
+ * Forces a tool call on the FIRST iteration of streamWithTools only
2256
+ * (subsequent iterations revert to auto). Used by fork mode to stop the
2257
+ * model from ending its turn with a bare "I'll adapt the script…" preamble
2258
+ * and zero tool calls. `{ type: 'any' }` lets the model pick which tool
2259
+ * (write_script in the common case, a source tool for schema discovery);
2260
+ * `{ type: 'tool', name }` pins a specific tool. Honored on both the
2261
+ * Anthropic path and the OpenAI/OpenRouter path (mapped to OpenAI's
2262
+ * tool_choice: 'required' / a named function).
2263
+ */
2264
+ firstIterationToolChoice?: {
2265
+ type: 'any';
2266
+ } | {
2267
+ type: 'tool';
2268
+ name: string;
2269
+ };
2270
+ /**
2271
+ * Internal — set only by the OpenRouter wrappers when the target is a Claude
2272
+ * model. Tells the OpenAI-wire path to emit Anthropic `cache_control`
2273
+ * breakpoints (OpenRouter forwards them to Anthropic for prompt caching).
2274
+ * Never set for direct OpenAI/Groq calls, so their requests are unchanged.
2275
+ */
2276
+ _openrouterClaudeCaching?: boolean;
2277
+ /**
2278
+ * Internal — OpenRouter provider-routing preferences (forwarded as the
2279
+ * `provider` body field). Set by the OpenRouter wrappers to steer routing to
2280
+ * a fast backend (e.g. {sort:'throughput'}). Never set for direct OpenAI/Groq.
2281
+ */
2282
+ _openrouterProvider?: Record<string, unknown>;
2283
+ }
2284
+ interface Tool {
2285
+ name: string;
2286
+ description: string;
2287
+ input_schema: {
2288
+ type: string;
2289
+ properties: Record<string, any>;
2290
+ required?: string[];
2291
+ };
2292
+ }
2293
+ declare class LLM {
2294
+ static text(messages: LLMMessages, options?: LLMOptions): Promise<string>;
2295
+ static stream<T = string>(messages: LLMMessages, options?: LLMOptions, json?: boolean): Promise<T extends string ? string : any>;
2296
+ static streamWithTools(messages: LLMMessages, tools: Tool[], toolHandler: (toolName: string, toolInput: any) => Promise<any>, options?: LLMOptions, maxIterations?: number): Promise<string>;
2297
+ /**
2298
+ * Normalize system prompt to Anthropic format
2299
+ * Converts string to array format if needed
2300
+ * @param sys - System prompt (string or array of blocks)
2301
+ * @returns Normalized system prompt for Anthropic API
2302
+ */
2303
+ private static _normalizeSystemPrompt;
2304
+ /**
2305
+ * Strip unpaired UTF-16 surrogates from every text field of a message set.
2306
+ *
2307
+ * A lone surrogate (from mid-pair string slicing or corrupt source data)
2308
+ * serializes to a bare `\udXXX` escape that strict JSON parsers — including
2309
+ * the one on Anthropic's API — reject with "no low surrogate in string",
2310
+ * failing the whole request. Sanitizing here, at the single boundary every
2311
+ * provider call flows through, guarantees no request can carry one.
2312
+ */
2313
+ private static _sanitizeMessages;
2314
+ /**
2315
+ * Log cache usage metrics from Anthropic API response
2316
+ * Shows cache hits, costs, and savings
2317
+ */
2318
+ private static _logCacheUsage;
2319
+ /**
2320
+ * Parse model string to extract provider and model name
2321
+ * @param modelString - Format: "provider/model-name" or just "model-name"
2322
+ * @returns [provider, modelName]
2323
+ *
2324
+ * @example
2325
+ * "anthropic/claude-sonnet-4-5" → ["anthropic", "claude-sonnet-4-5"]
2326
+ * "groq/openai/gpt-oss-120b" → ["groq", "openai/gpt-oss-120b"]
2327
+ * "claude-sonnet-4-5" → ["anthropic", "claude-sonnet-4-5"] (default)
2328
+ */
2329
+ private static _parseModel;
2330
+ /**
2331
+ * Map an Anthropic model id (e.g. "claude-sonnet-4-5-20250929") to the OpenRouter slug
2332
+ * (e.g. "claude-sonnet-4.5"). OpenRouter slugs drop the date suffix and use dotted versions.
2333
+ */
2334
+ private static _toOpenRouterSlug;
2335
+ /**
2336
+ * Per-provider proxy base URL. Returns `${SUPERATOM_LLM_PROXY_URL}/<provider>`
2337
+ * when our Cloudflare LLM proxy is configured, else undefined (→ talk to the
2338
+ * provider directly, legacy behaviour). An explicit options.baseURL (e.g.
2339
+ * OpenRouter) always wins and is never overridden. See backend/docs/llm-proxy.md.
2340
+ */
2341
+ private static _proxyBaseURL;
2342
+ private static _openrouterOptions;
2343
+ private static _openrouterText;
2344
+ private static _openrouterStream;
2345
+ private static _openrouterStreamWithTools;
2346
+ /**
2347
+ * Build an Anthropic client. Routes through our Cloudflare LLM proxy when
2348
+ * SUPERATOM_LLM_PROXY_URL is set (each client ships a per-client proxy key as
2349
+ * ANTHROPIC_API_KEY and never holds the real key); otherwise talks to
2350
+ * api.anthropic.com directly. See backend/docs/llm-proxy.md.
2351
+ */
2352
+ private static _anthropicClient;
2353
+ /** True when OpenRouter is configured as a fail-open fallback for Claude. */
2354
+ private static _openrouterAvailable;
2355
+ /** Remap an Anthropic model id to the OpenRouter model path for fail-open. */
2356
+ private static _anthropicFallbackModel;
2357
+ private static _anthropicText;
2358
+ private static _anthropicStream;
2359
+ private static _anthropicStreamWithTools;
2360
+ private static _groqText;
2361
+ private static _groqStream;
2362
+ /**
2363
+ * Gemini request options carrying the proxy base URL, or undefined → talk to
2364
+ * generativelanguage.googleapis.com directly. The Google SDK takes baseUrl as a
2365
+ * per-model request option, not a constructor arg. See backend/docs/llm-proxy.md.
2366
+ */
2367
+ private static _geminiRequestOptions;
2368
+ private static _geminiText;
2369
+ private static _geminiStream;
2370
+ /**
2371
+ * Recursively strip unsupported JSON Schema properties for Gemini
2372
+ * Gemini doesn't support: additionalProperties, $schema, etc.
2373
+ */
2374
+ private static _cleanSchemaForGemini;
2375
+ private static _geminiStreamWithTools;
2376
+ /** True for Anthropic/Claude model ids — gates OpenRouter prompt caching. */
2377
+ private static _isClaudeModel;
2378
+ /**
2379
+ * Build the OpenAI-wire system message. For OpenRouter + Claude
2380
+ * (cacheClaude=true) it emits content parts carrying Anthropic
2381
+ * `cache_control` breakpoints (preserving any the caller set, else marking
2382
+ * the last block), so OpenRouter forwards them to Anthropic for prompt
2383
+ * caching. Otherwise it returns a plain flattened string — unchanged for
2384
+ * direct OpenAI/Groq.
2385
+ */
2386
+ private static _openaiSystemMessage;
2387
+ /**
2388
+ * Split an OpenAI-wire usage object. `prompt_tokens` INCLUDES cached tokens,
2389
+ * so we subtract them out (Anthropic-style: input excludes cache reads) and
2390
+ * report cached separately — this makes calculateCost price cache reads at
2391
+ * the discounted rate and reflects OpenRouter prompt-cache savings in logs.
2392
+ */
2393
+ private static _openaiUsage;
2394
+ private static _openaiText;
2395
+ private static _openaiStream;
2396
+ /** Map the Anthropic-style firstIterationToolChoice to OpenAI's tool_choice. */
2397
+ private static _openaiToolChoice;
2398
+ private static _openaiStreamWithTools;
2399
+ /**
2400
+ * Parse JSON string, handling markdown code blocks and surrounding text
2401
+ * Enhanced version with jsonrepair to handle malformed JSON from LLMs
2402
+ * @param text - Text that may contain JSON wrapped in ```json...``` or with surrounding text
2403
+ * @returns Parsed JSON object or array
2404
+ */
2405
+ private static _parseJSON;
2406
+ }
2407
+
2408
+ interface CapturedLog {
2409
+ timestamp: number;
2410
+ level: 'info' | 'error' | 'warn' | 'debug';
2411
+ message: string;
2412
+ type?: 'explanation' | 'query' | 'general';
2413
+ data?: Record<string, any>;
2414
+ }
2415
+ /**
2416
+ * UILogCollector captures logs during user prompt processing
2417
+ * and sends them to runtime via ui_logs message with uiBlockId as the message id
2418
+ * Logs are sent in real-time for streaming effect in the UI
2419
+ * Respects the global log level configuration
2420
+ */
2421
+ declare class UILogCollector {
2422
+ private logs;
2423
+ private uiBlockId;
2424
+ private clientId;
2425
+ private sendMessage;
2426
+ private currentLogLevel;
2427
+ constructor(clientId: string, sendMessage: (message: Message) => void, uiBlockId?: string);
2428
+ /**
2429
+ * Check if logging is enabled (uiBlockId is provided)
2430
+ */
2431
+ isEnabled(): boolean;
2432
+ /**
2433
+ * Check if a message should be logged based on current log level
2434
+ */
2435
+ private shouldLog;
2436
+ /**
2437
+ * Add a log entry with timestamp and immediately send to runtime
2438
+ * Only logs that pass the log level filter are captured and sent
2439
+ */
2440
+ private addLog;
2441
+ /**
2442
+ * Send a single log to runtime immediately
2443
+ */
2444
+ private sendLogImmediately;
2445
+ /**
2446
+ * Log info message
2447
+ */
2448
+ info(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
2449
+ /**
2450
+ * Log error message
2451
+ */
2452
+ error(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
2453
+ /**
2454
+ * Log warning message
2455
+ */
2456
+ warn(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
2457
+ /**
2458
+ * Log debug message
2459
+ */
2460
+ debug(message: string, type?: 'explanation' | 'query' | 'general', data?: Record<string, any>): void;
2461
+ /**
2462
+ * Log LLM explanation with typed metadata
2463
+ */
2464
+ logExplanation(message: string, explanation: string, data?: Record<string, any>): void;
2465
+ /**
2466
+ * Log generated query with typed metadata
2467
+ */
2468
+ logQuery(message: string, query: string, data?: Record<string, any>): void;
2469
+ /**
2470
+ * Send all collected logs at once (optional, for final summary)
2471
+ */
2472
+ sendAllLogs(): void;
2473
+ /**
2474
+ * Get all collected logs
2475
+ */
2476
+ getLogs(): CapturedLog[];
2477
+ /**
2478
+ * Clear all logs
2479
+ */
2480
+ clearLogs(): void;
2481
+ /**
2482
+ * Set uiBlockId (in case it's provided later)
2483
+ */
2484
+ setUIBlockId(uiBlockId: string): void;
2485
+ }
2486
+
2487
+ /**
2488
+ * UIBlock represents a single user and assistant message block in a thread
2489
+ * Contains user question, component metadata, component data, text response, and available actions
2490
+ */
2491
+ declare class UIBlock {
2492
+ private id;
2493
+ private userQuestion;
2494
+ private generatedComponentMetadata;
2495
+ private componentData;
2496
+ private textResponse;
2497
+ private actions;
2498
+ private createdAt;
2499
+ /**
2500
+ * Creates a new UIBlock instance
2501
+ * @param userQuestion - The user's question or input
2502
+ * @param componentData - The component data object
2503
+ * @param generatedComponentMetadata - Optional metadata about the generated component
2504
+ * @param actions - Optional array of available actions
2505
+ * @param id - Optional custom ID, generates UUID if not provided
2506
+ * @param textResponse - Optional text response from LLM
2507
+ */
2508
+ constructor(userQuestion: string, componentData?: Record<string, any>, generatedComponentMetadata?: Record<string, any>, actions?: Action[], id?: string, textResponse?: string | null);
2509
+ /**
2510
+ * Get the UIBlock ID
2511
+ */
2512
+ getId(): string;
2513
+ /**
2514
+ * Get the user question
1489
2515
  */
1490
2516
  getUserQuestion(): string;
1491
2517
  /**
@@ -1640,12 +2666,20 @@ declare class Thread {
1640
2666
 
1641
2667
  /**
1642
2668
  * ThreadManager manages all threads globally
1643
- * Provides methods to create, retrieve, and delete threads
2669
+ * Provides methods to create, retrieve, and delete threads.
2670
+ * Includes automatic cleanup to prevent unbounded memory growth.
1644
2671
  */
1645
2672
  declare class ThreadManager {
1646
2673
  private static instance;
1647
2674
  private threads;
2675
+ private cleanupInterval;
2676
+ private readonly threadTtlMs;
1648
2677
  private constructor();
2678
+ /**
2679
+ * Periodically remove threads older than 7 days.
2680
+ * Runs every hour to avoid frequent iteration over the map.
2681
+ */
2682
+ private startCleanup;
1649
2683
  /**
1650
2684
  * Get singleton instance of ThreadManager
1651
2685
  */
@@ -2092,134 +3126,42 @@ declare class QueryExecutionService {
2092
3126
  * Get the cache key for a query
2093
3127
  * This ensures the cache key matches what the frontend will send
2094
3128
  */
2095
- getQueryCacheKey(query: any): string;
2096
- /**
2097
- * Execute a query against the database
2098
- * @param query - The SQL query to execute (string or object with sql/values)
2099
- * @param collections - Collections object containing database execute function
2100
- * @returns Object with result data and cache key
2101
- */
2102
- executeQuery(query: any, collections: any): Promise<{
2103
- result: any;
2104
- cacheKey: string;
2105
- }>;
2106
- /**
2107
- * Request the LLM to fix a failed SQL query
2108
- * @param failedQuery - The query that failed execution
2109
- * @param errorMessage - The error message from the failed execution
2110
- * @param componentContext - Context about the component
2111
- * @param apiKey - Optional API key
2112
- * @returns Fixed query string
2113
- */
2114
- requestQueryFix(failedQuery: string, errorMessage: string, componentContext: ComponentContext, apiKey?: string): Promise<string>;
2115
- /**
2116
- * Validate a single component's query with retry logic
2117
- * @param component - The component to validate
2118
- * @param collections - Collections object containing database execute function
2119
- * @param apiKey - Optional API key for LLM calls
2120
- * @returns Validation result with component, query key, and result
2121
- */
2122
- validateSingleQuery(component: Component, collections: any, apiKey?: string): Promise<QueryValidationResult>;
2123
- /**
2124
- * Validate multiple component queries in parallel
2125
- * @param components - Array of components with potential queries
2126
- * @param collections - Collections object containing database execute function
2127
- * @param apiKey - Optional API key for LLM calls
2128
- * @returns Object with validated components and query results map
2129
- */
2130
- validateComponentQueries(components: Component[], collections: any, apiKey?: string): Promise<BatchValidationResult>;
2131
- }
2132
-
2133
- /**
2134
- * StreamBuffer - Buffered streaming utility for smoother text delivery
2135
- * Batches small chunks together and flushes at regular intervals
2136
- */
2137
- type StreamCallback = (chunk: string) => void;
2138
- /**
2139
- * StreamBuffer class for managing buffered streaming output
2140
- * Provides smooth text delivery by batching small chunks
2141
- */
2142
- declare class StreamBuffer {
2143
- private buffer;
2144
- private flushTimer;
2145
- private callback;
2146
- private fullText;
2147
- constructor(callback?: StreamCallback);
2148
- /**
2149
- * Check if the buffer has a callback configured
2150
- */
2151
- hasCallback(): boolean;
2152
- /**
2153
- * Get all text that has been written (including already flushed)
2154
- */
2155
- getFullText(): string;
3129
+ getQueryCacheKey(query: any): string;
2156
3130
  /**
2157
- * Write a chunk to the buffer
2158
- * Large chunks or chunks with newlines are flushed immediately
2159
- * Small chunks are batched and flushed after a short interval
2160
- *
2161
- * @param chunk - Text chunk to write
3131
+ * Execute a query against the database
3132
+ * @param query - The SQL query to execute (string or object with sql/values)
3133
+ * @param collections - Collections object containing database execute function
3134
+ * @returns Object with result data and cache key
2162
3135
  */
2163
- write(chunk: string): void;
3136
+ executeQuery(query: any, collections: any): Promise<{
3137
+ result: any;
3138
+ cacheKey: string;
3139
+ }>;
2164
3140
  /**
2165
- * Flush the buffer immediately
2166
- * Call this before tool execution or other operations that need clean output
3141
+ * Request the LLM to fix a failed SQL query
3142
+ * @param failedQuery - The query that failed execution
3143
+ * @param errorMessage - The error message from the failed execution
3144
+ * @param componentContext - Context about the component
3145
+ * @param apiKey - Optional API key
3146
+ * @returns Fixed query string
2167
3147
  */
2168
- flush(): void;
3148
+ requestQueryFix(failedQuery: string, errorMessage: string, componentContext: ComponentContext, apiKey?: string): Promise<string>;
2169
3149
  /**
2170
- * Internal flush implementation
3150
+ * Validate a single component's query with retry logic
3151
+ * @param component - The component to validate
3152
+ * @param collections - Collections object containing database execute function
3153
+ * @param apiKey - Optional API key for LLM calls
3154
+ * @returns Validation result with component, query key, and result
2171
3155
  */
2172
- private flushNow;
3156
+ validateSingleQuery(component: Component, collections: any, apiKey?: string): Promise<QueryValidationResult>;
2173
3157
  /**
2174
- * Clean up resources
2175
- * Call this when done with the buffer
3158
+ * Validate multiple component queries in parallel
3159
+ * @param components - Array of components with potential queries
3160
+ * @param collections - Collections object containing database execute function
3161
+ * @param apiKey - Optional API key for LLM calls
3162
+ * @returns Object with validated components and query results map
2176
3163
  */
2177
- dispose(): void;
2178
- }
2179
-
2180
- /**
2181
- * ToolExecutorService - Handles execution of SQL queries and external tools
2182
- * Extracted from BaseLLM.generateTextResponse for better separation of concerns
2183
- */
2184
-
2185
- /**
2186
- * External tool definition
2187
- */
2188
- interface ExternalTool {
2189
- id: string;
2190
- name: string;
2191
- description?: string;
2192
- /** Tool type: "source" = routed through SourceAgent, "direct" = called directly by MainAgent */
2193
- toolType?: 'source' | 'direct';
2194
- /** Full untruncated schema for source agent (all columns visible) */
2195
- fullSchema?: string;
2196
- /** Schema size tier: small (≤50 tables), medium (51-200), large (201-500), very_large (500+) */
2197
- schemaTier?: string;
2198
- /** Schema search function for very_large tier — keyword search over entities */
2199
- schemaSearchFn?: (keywords: string[]) => string;
2200
- fn: (input: any) => Promise<any>;
2201
- limit?: number;
2202
- outputSchema?: any;
2203
- executionType?: 'immediate' | 'deferred';
2204
- userProvidedData?: any;
2205
- params?: Record<string, any>;
2206
- }
2207
- /**
2208
- * Executed tool tracking info
2209
- */
2210
- interface ExecutedToolInfo {
2211
- id: string;
2212
- name: string;
2213
- params: any;
2214
- result: {
2215
- _totalRecords: number;
2216
- _recordsShown: number;
2217
- _metadata?: any;
2218
- _sampleData: any[];
2219
- };
2220
- outputSchema?: any;
2221
- sourceSchema?: string;
2222
- sourceType?: string;
3164
+ validateComponentQueries(components: Component[], collections: any, apiKey?: string): Promise<BatchValidationResult>;
2223
3165
  }
2224
3166
 
2225
3167
  /**
@@ -2377,7 +3319,7 @@ declare abstract class BaseLLM {
2377
3319
  * This helps provide intelligent suggestions for follow-up queries
2378
3320
  * For general/conversational questions without components, pass textResponse instead
2379
3321
  */
2380
- generateNextQuestions(originalUserPrompt: string, component?: Component | null, componentData?: Record<string, unknown>, apiKey?: string, conversationHistory?: string, textResponse?: string): Promise<string[]>;
3322
+ generateNextQuestions(originalUserPrompt: string, component?: Component | null, componentData?: Record<string, unknown>, apiKey?: string, conversationHistory?: string, textResponse?: string, signal?: AbortSignal): Promise<string[]>;
2381
3323
  }
2382
3324
 
2383
3325
  interface AnthropicLLMConfig extends BaseLLMConfig {
@@ -2437,18 +3379,30 @@ declare class OpenAILLM extends BaseLLM {
2437
3379
  declare const openaiLLM: OpenAILLM;
2438
3380
 
2439
3381
  /**
2440
- * Query Cache - Stores query results with configurable TTL
2441
- * Used to avoid re-executing queries that were already validated
3382
+ * Query Cache Two mechanisms:
3383
+ *
3384
+ * 1. `cache` (query string → result data) — TTL-based with max size, for avoiding re-execution
3385
+ * of recently validated queries. True LRU eviction: reads bubble entries to the back via
3386
+ * delete+re-set so the oldest *unused* entry is evicted, not the oldest *inserted*.
3387
+ *
3388
+ * 2. Encrypted queryId tokens — SQL is encrypted into the queryId itself (self-contained).
3389
+ * No server-side storage needed for SQL mappings. The token is decrypted on each request.
3390
+ * This eliminates the unbounded queryIdCache that previously grew forever and caused
3391
+ * memory bloat (hundreds of MBs after thousands of queries).
3392
+ *
3393
+ * Result data can still be cached temporarily via the data cache (mechanism 1).
2442
3394
  */
2443
3395
  declare class QueryCache {
2444
3396
  private cache;
2445
- private queryIdCache;
2446
3397
  private ttlMs;
3398
+ private maxCacheSize;
2447
3399
  private cleanupInterval;
3400
+ private readonly algorithm;
3401
+ private encryptionKey;
2448
3402
  constructor();
2449
3403
  /**
2450
3404
  * Set the cache TTL (Time To Live)
2451
- * @param minutes - TTL in minutes (default: 5)
3405
+ * @param minutes - TTL in minutes (default: 10)
2452
3406
  */
2453
3407
  setTTL(minutes: number): void;
2454
3408
  /**
@@ -2456,12 +3410,16 @@ declare class QueryCache {
2456
3410
  */
2457
3411
  getTTL(): number;
2458
3412
  /**
2459
- * Store query result in cache
2460
- * Key is the exact query string (or JSON for parameterized queries)
3413
+ * Store query result in data cache.
3414
+ * If the key already exists, it's removed first so the re-insert places it
3415
+ * at the back of the iteration order (LRU). Eviction only fires when adding
3416
+ * a genuinely new key past the size limit.
2461
3417
  */
2462
3418
  set(query: string, data: any): void;
2463
3419
  /**
2464
- * Get cached result if exists and not expired
3420
+ * Get cached result if exists and not expired.
3421
+ * On hit, re-inserts the entry so it moves to the back of the Map's
3422
+ * iteration order — turning FIFO eviction into true LRU.
2465
3423
  */
2466
3424
  get(query: string): any | null;
2467
3425
  /**
@@ -2481,30 +3439,37 @@ declare class QueryCache {
2481
3439
  */
2482
3440
  getStats(): {
2483
3441
  size: number;
3442
+ queryIdCount: number;
2484
3443
  oldestEntryAge: number | null;
2485
3444
  };
2486
3445
  /**
2487
- * Start periodic cleanup of expired entries
3446
+ * Start periodic cleanup of expired data cache entries.
2488
3447
  */
2489
3448
  private startCleanup;
2490
3449
  /**
2491
- * Generate a unique query ID
3450
+ * Encrypt a payload into a self-contained token.
3451
+ */
3452
+ private encrypt;
3453
+ /**
3454
+ * Decrypt a token back to the original payload.
2492
3455
  */
2493
- private generateQueryId;
3456
+ private decrypt;
2494
3457
  /**
2495
- * Store a query by ID. Returns the generated queryId.
2496
- * The query is stored server-side; only the queryId is sent to the frontend.
3458
+ * Store a query by generating an encrypted token as queryId.
3459
+ * The SQL is encrypted INTO the token nothing stored in memory.
3460
+ * If data is provided, it's cached temporarily in the data cache.
2497
3461
  */
2498
3462
  storeQuery(query: any, data?: any): string;
2499
3463
  /**
2500
- * Get a stored query by its ID (not expired)
3464
+ * Get a stored query by decrypting its token.
3465
+ * Returns the SQL + any cached result data.
2501
3466
  */
2502
3467
  getQuery(queryId: string): {
2503
3468
  query: any;
2504
3469
  data: any;
2505
3470
  } | null;
2506
3471
  /**
2507
- * Update cached data for a queryId
3472
+ * Update cached data for a queryId token
2508
3473
  */
2509
3474
  setQueryData(queryId: string, data: any): void;
2510
3475
  /**
@@ -2561,175 +3526,69 @@ declare class DashboardConversationHistory {
2561
3526
  declare const dashboardConversationHistory: DashboardConversationHistory;
2562
3527
 
2563
3528
  /**
2564
- * Multi-Agent Architecture Types
3529
+ * ScriptMatcher — LLM-Based Script Matching + Parameter Extraction
2565
3530
  *
2566
- * Defines interfaces for the hierarchical agent system:
2567
- * - Main Agent: ONE LLM.streamWithTools() call with source agent tools
2568
- * - Source Agents: independent agents that query individual data sources
3531
+ * Uses ONE LLM call to:
3532
+ * 1. Pick the best matching script from the library (or "none")
3533
+ * 2. Extract parameter values from the user question
2569
3534
  *
2570
- * The main agent sees only source summaries. When it calls a source tool,
2571
- * the SourceAgent runs independently (own LLM, own retries) and returns clean data.
3535
+ * Why LLM over embeddings:
3536
+ * - Embeddings capture topic similarity ("overstock" "inventory" ≈ "revenue")
3537
+ * but can't distinguish structurally different questions about the same domain
3538
+ * - LLM understands that "overstock by warehouse" needs a different script than
3539
+ * "revenue by warehouse" even though they're semantically close
3540
+ * - One call does both matching AND parameter extraction
3541
+ *
3542
+ * When script library grows past ~50, add an embedding pre-filter
3543
+ * (ChromaDB narrows to top 10 → LLM picks from those 10).
2572
3544
  */
2573
3545
 
2574
- /**
2575
- * Per-entity detail: name, row count, and column names.
2576
- * Gives the main agent enough context to route to the right source.
2577
- */
2578
- interface EntityDetail {
2579
- /** Entity name (table, sheet, endpoint) */
2580
- name: string;
2581
- /** Approximate row count */
2582
- rowCount?: number;
2583
- /** Column/field names */
2584
- columns: string[];
2585
- }
2586
- /**
2587
- * Representation of a data source for the main agent.
2588
- * Contains entity names WITH column names so the LLM can route accurately.
2589
- */
2590
- interface SourceSummary {
2591
- /** Source ID (matches tool ID prefix) */
2592
- id: string;
2593
- /** Human-readable source name */
2594
- name: string;
2595
- /** Source type: postgres, excel, rest_api, etc. */
2596
- type: string;
2597
- /** Brief description of what data this source contains */
2598
- description: string;
2599
- /** Detailed entity info with column names for routing */
2600
- entityDetails: EntityDetail[];
2601
- /** The tool ID associated with this source */
2602
- toolId: string;
2603
- }
2604
- /**
2605
- * What a source agent returns after querying its data source.
2606
- * The main agent uses this to analyze and compose the final response.
2607
- */
2608
- interface SourceAgentResult {
2609
- /** Source ID */
2610
- sourceId: string;
2611
- /** Source name */
2612
- sourceName: string;
2613
- /** Whether the query succeeded */
2614
- success: boolean;
2615
- /** Result data rows */
2616
- data: any[];
2617
- /** Metadata about the query execution */
2618
- metadata: SourceAgentMetadata;
2619
- /** Tool execution info (reused for component generation) */
2620
- executedTool: ExecutedToolInfo;
2621
- /** Error message if failed */
2622
- error?: string;
2623
- }
2624
- interface SourceAgentMetadata {
2625
- /** Total rows that matched the query (before limit) */
2626
- totalRowsMatched: number;
2627
- /** Rows actually returned (after limit) */
2628
- rowsReturned: number;
2629
- /** Whether the result was truncated by the row limit */
2630
- isLimited: boolean;
2631
- /** The query/params that were executed */
2632
- queryExecuted?: string;
2633
- /** Execution time in milliseconds */
2634
- executionTimeMs: number;
2635
- }
2636
- /**
2637
- * The complete response from the multi-agent system.
2638
- * Contains everything needed for text display + component generation.
2639
- */
2640
- interface AgentResponse {
2641
- /** Generated text response (analysis of the data) */
2642
- text: string;
2643
- /** All executed tools across all source agents (for component generation) */
2644
- executedTools: ExecutedToolInfo[];
2645
- /** Individual results from each source agent */
2646
- sourceResults: SourceAgentResult[];
2647
- }
2648
- /**
2649
- * Configuration for the multi-agent system.
2650
- * Controls limits, models, and behavior.
2651
- */
2652
- interface AgentConfig {
2653
- /** Max rows a source agent can return (default: 50) */
2654
- maxRowsPerSource: number;
2655
- /** Model for the main agent (routing + analysis in one LLM call) */
2656
- mainAgentModel: string;
2657
- /** Model for source agent query generation */
2658
- sourceAgentModel: string;
2659
- /** API key for LLM calls */
2660
- apiKey?: string;
2661
- /** Max retry attempts per source agent */
2662
- maxRetries: number;
2663
- /** Max tool calling iterations for the main agent loop */
2664
- maxIterations: number;
2665
- /** Global knowledge base context (static, same for all users/questions — cached in system prompt) */
2666
- globalKnowledgeBase?: string;
2667
- /** Per-request knowledge base context (user-specific + query-matched — dynamic, not cached) */
2668
- knowledgeBaseContext?: string;
3546
+ declare class ScriptMatcher {
3547
+ private store;
3548
+ constructor(store: ScriptStore);
3549
+ /**
3550
+ * Find the best matching script for a user question.
3551
+ * Uses ONE LLM call that picks the script AND extracts parameters.
3552
+ * Returns null if no script matches.
3553
+ */
3554
+ match(userPrompt: string, apiKey?: string, model?: string, signal?: AbortSignal): Promise<ScriptMatch | null>;
3555
+ /**
3556
+ * Build the script catalog string for the LLM prompt.
3557
+ * Each script gets: index, ID, name, description, and parameter definitions.
3558
+ */
3559
+ private buildScriptCatalog;
2669
3560
  }
2670
- /**
2671
- * Default agent configuration
2672
- */
2673
- declare const DEFAULT_AGENT_CONFIG: AgentConfig;
2674
3561
 
2675
3562
  /**
2676
- * Main Agent (Orchestrator)
3563
+ * ScriptRunner Execute scripts in an isolated tsx subprocess.
2677
3564
  *
2678
- * A single LLM.streamWithTools() call that handles everything:
2679
- * - Routing: decides which source(s) to query based on summaries
2680
- * - Querying: calls source tools (each wraps an independent SourceAgent)
2681
- * - Direct tools: calls pre-built function tools directly with LLM-provided params
2682
- * - Re-querying: if data is wrong/incomplete, calls tools again with modified intent
2683
- * - Analysis: generates final text response from the data
3565
+ * The subprocess approach replaces the earlier `new Function()` eval and gives us:
3566
+ * - Real sandbox (separate process, SIGKILL on timeout).
3567
+ * - Real TypeScript (tsx transpiles on the fly).
3568
+ * - npm imports available to scripts (clustering, stats, geo, etc.).
2684
3569
  *
2685
- * Two tool types:
2686
- * - "source" tools: main agent sees summaries, SourceAgent handles SQL generation independently
2687
- * - "direct" tools: main agent calls fn() directly with structured params (no SourceAgent)
3570
+ * Protocol: NDJSON over the child's stdin/stdout. See script-ipc.ts + backend/docs/SCRIPT-FLOW-IMPLEMENTATION.md.
2688
3571
  */
2689
3572
 
2690
- declare class MainAgent {
2691
- private externalTools;
2692
- private config;
2693
- private streamBuffer;
2694
- constructor(externalTools: ExternalTool[], config: AgentConfig, streamBuffer?: StreamBuffer);
2695
- /**
2696
- * Handle a user question using the multi-agent system.
2697
- *
2698
- * This is ONE LLM.streamWithTools() call. The LLM:
2699
- * 1. Sees source summaries + direct tool descriptions in system prompt
2700
- * 2. Decides which tool(s) to call (routing)
2701
- * 3. Source tools → SourceAgent runs independently → returns data
2702
- * 4. Direct tools → fn() called directly with LLM params → returns data
2703
- * 5. Generates final analysis text
2704
- */
2705
- handleQuestion(userPrompt: string, apiKey?: string, conversationHistory?: string, streamCallback?: (chunk: string) => void): Promise<AgentResponse>;
2706
- /**
2707
- * Execute a direct tool — call fn() with LLM-provided params, no SourceAgent.
2708
- */
2709
- private handleDirectTool;
2710
- /**
2711
- * Build the main agent's system prompt with source summaries and direct tool descriptions.
2712
- */
2713
- private buildSystemPrompt;
2714
- /**
2715
- * Build tool definitions for source tools — summary-only descriptions.
2716
- * The full schema is inside the SourceAgent which runs independently.
2717
- */
2718
- private buildSourceToolDefinitions;
2719
- /**
2720
- * Build tool definitions for direct tools — expose their actual params.
2721
- * These are called directly by the main agent LLM, no SourceAgent.
2722
- */
2723
- private buildDirectToolDefinitions;
2724
- /**
2725
- * Format a source agent's result as a clean string for the main agent LLM.
2726
- */
2727
- private formatResultForMainAgent;
2728
- /**
2729
- * Get source summaries (for external inspection/debugging).
2730
- */
2731
- getSourceSummaries(): SourceSummary[];
3573
+ interface RunScriptOptions {
3574
+ /** Data sources the script is allowed to query via ctx.query */
3575
+ externalTools: ExternalTool[];
3576
+ /** Optional — for propagating per-query UI progress to the user */
3577
+ streamBuffer?: StreamBuffer;
3578
+ /** Override the wall-clock timeout (default `SCRIPT_TIMEOUT_MS`, 60s). */
3579
+ timeoutMs?: number;
3580
+ /**
3581
+ * Per-turn cancellation signal. When the user hits "Stop" mid-run, the child
3582
+ * process group is SIGKILLed and the run resolves as an aborted failure (the
3583
+ * caller is already unwinding, so the result is discarded).
3584
+ */
3585
+ signal?: AbortSignal;
2732
3586
  }
3587
+ /**
3588
+ * Execute a recipe by spawning a tsx child on the script's .ts file.
3589
+ * `scriptPath` is the absolute path to the saved `.ts` body.
3590
+ */
3591
+ declare function runScript(recipe: ScriptRecipe, scriptPath: string, params: Record<string, any>, options: RunScriptOptions): Promise<ScriptResult>;
2733
3592
 
2734
3593
  type MessageTypeHandler = (message: IncomingMessage) => void | Promise<void>;
2735
3594
  declare class SuperatomSDK {
@@ -2747,6 +3606,7 @@ declare class SuperatomSDK {
2747
3606
  private collections;
2748
3607
  private components;
2749
3608
  private tools;
3609
+ private workflows;
2750
3610
  private anthropicApiKey;
2751
3611
  private groqApiKey;
2752
3612
  private geminiApiKey;
@@ -2754,6 +3614,9 @@ declare class SuperatomSDK {
2754
3614
  private llmProviders;
2755
3615
  private databaseType;
2756
3616
  private modelStrategy;
3617
+ private mainAgentModel;
3618
+ private sourceAgentModel;
3619
+ private dashCompModels?;
2757
3620
  private conversationSimilarityThreshold;
2758
3621
  private userManager;
2759
3622
  private dashboardManager;
@@ -2853,6 +3716,19 @@ declare class SuperatomSDK {
2853
3716
  * Get the stored tools
2854
3717
  */
2855
3718
  getTools(): Tool$1[];
3719
+ /**
3720
+ * Register workflow components for the SDK instance.
3721
+ *
3722
+ * Workflows are pre-built multi-step UI flows the main agent can pick when
3723
+ * the user's prompt matches a workflow's `whenToUse` trigger. Picking a
3724
+ * workflow short-circuits analysis text + dashboard component generation —
3725
+ * the workflow component is returned directly, with the LLM-extracted props.
3726
+ */
3727
+ setWorkflows(workflows: WorkflowDescriptor[]): void;
3728
+ /**
3729
+ * Get the registered workflow components.
3730
+ */
3731
+ getWorkflows(): WorkflowDescriptor[];
2856
3732
  /**
2857
3733
  * Apply model strategy to all LLM provider singletons
2858
3734
  * @param strategy - 'best', 'fast', or 'balanced'
@@ -2883,4 +3759,4 @@ declare class SuperatomSDK {
2883
3759
  getConversationSimilarityThreshold(): number;
2884
3760
  }
2885
3761
 
2886
- export { type Action, type AgentConfig, type AgentResponse, BM25L, type BM25LOptions, type BaseLLMConfig, CONTEXT_CONFIG, type CapturedLog, CleanupService, type CollectionHandler, type CollectionOperation, type DBUIBlock, DEFAULT_AGENT_CONFIG, type DatabaseType, type HybridSearchOptions, type IncomingMessage, type KbNodesQueryFilters, type KbNodesRequestPayload, LLM, type LLMUsageEntry, type LogLevel, MainAgent, type Message, type ModelStrategy, type OutputField, type RerankedResult, STORAGE_CONFIG, SuperatomSDK, type SuperatomSDKConfig, type TaskType, Thread, ThreadManager, type Tool$1 as Tool, type ToolOutputSchema, UIBlock, UILogCollector, type User, UserManager, type UsersData, anthropicLLM, dashboardConversationHistory, geminiLLM, groqLLM, hybridRerank, llmUsageLogger, logger, openaiLLM, queryCache, rerankChromaResults, rerankConversationResults, userPromptErrorLogger };
3762
+ export { type Action, type AgentConfig, type AgentResponse, BM25L, type BM25LOptions, type BaseLLMConfig, CONTEXT_CONFIG, type CapturedLog, CleanupService, type CollectionHandler, type CollectionOperation, type DBUIBlock, DEFAULT_AGENT_CONFIG, type DatabaseType, type HybridSearchOptions, type IncomingMessage, type KbNodesQueryFilters, type KbNodesRequestPayload, LLM, type LLMUsageEntry, type LogLevel, MainAgent, type Message, type ModelStrategy, type OutputField, type RerankedResult, STORAGE_CONFIG, type ScriptComponentSpec, ScriptMatcher, type ScriptParameter, type ScriptRecipe, type ScriptRecipeMetaRow, type ScriptRecipeStore, type ScriptResult, ScriptStore, type ScriptStoreOptions, type SelectedWorkflow, SuperatomSDK, type SuperatomSDKConfig, type TaskType, Thread, ThreadManager, type Tool$1 as Tool, type ToolOutputSchema, UIBlock, UILogCollector, type User, UserManager, type UsersData, type WorkflowDescriptor, anthropicLLM, dashboardConversationHistory, geminiLLM, groqLLM, hybridRerank, llmUsageLogger, logger, normalizeScriptBody, openaiLLM, queryCache, rerankChromaResults, rerankConversationResults, resolveScriptRecipeStore, runScript, userPromptErrorLogger };