@larkup/workspace-server 0.0.0-stage → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/LICENSE +176 -0
  2. package/README.md +2 -2
  3. package/dist-types/app/api/chat/feedback/route.d.ts +4 -0
  4. package/dist-types/app/api/chat/feedback/route.d.ts.map +1 -0
  5. package/dist-types/app/api/chat/route.d.ts +3 -0
  6. package/dist-types/app/api/chat/route.d.ts.map +1 -0
  7. package/dist-types/app/api/chat/tools.d.ts +67 -0
  8. package/dist-types/app/api/chat/tools.d.ts.map +1 -0
  9. package/dist-types/app/api/config/test-embedding/route.d.ts +3 -0
  10. package/dist-types/app/api/config/test-embedding/route.d.ts.map +1 -0
  11. package/dist-types/app/api/config/verify/route.d.ts +3 -0
  12. package/dist-types/app/api/config/verify/route.d.ts.map +1 -0
  13. package/dist-types/app/api/describe-image/route.d.ts +4 -0
  14. package/dist-types/app/api/describe-image/route.d.ts.map +1 -0
  15. package/dist-types/app/api/doc-editor/edit/route.d.ts +10 -0
  16. package/dist-types/app/api/doc-editor/edit/route.d.ts.map +1 -0
  17. package/dist-types/app/api/doc-editor/export/route.d.ts +9 -0
  18. package/dist-types/app/api/doc-editor/export/route.d.ts.map +1 -0
  19. package/dist-types/app/api/doc-editor/parse/route.d.ts +10 -0
  20. package/dist-types/app/api/doc-editor/parse/route.d.ts.map +1 -0
  21. package/dist-types/app/api/doc-editor/restore/route.d.ts +2 -0
  22. package/dist-types/app/api/doc-editor/restore/route.d.ts.map +1 -0
  23. package/dist-types/app/api/document/sign/route.d.ts +2 -0
  24. package/dist-types/app/api/document/sign/route.d.ts.map +1 -0
  25. package/dist-types/app/api/files/remote/route.d.ts +5 -0
  26. package/dist-types/app/api/files/remote/route.d.ts.map +1 -0
  27. package/dist-types/app/api/mcp/[connectionId]/route.d.ts +9 -0
  28. package/dist-types/app/api/mcp/[connectionId]/route.d.ts.map +1 -0
  29. package/dist-types/app/api/mcp/[connectionId]/test/route.d.ts +8 -0
  30. package/dist-types/app/api/mcp/[connectionId]/test/route.d.ts.map +1 -0
  31. package/dist-types/app/api/mcp/route.d.ts +4 -0
  32. package/dist-types/app/api/mcp/route.d.ts.map +1 -0
  33. package/dist-types/app/api/parse-file/route.d.ts +6 -0
  34. package/dist-types/app/api/parse-file/route.d.ts.map +1 -0
  35. package/dist-types/app/api/pdf-page/route.d.ts +4 -0
  36. package/dist-types/app/api/pdf-page/route.d.ts.map +1 -0
  37. package/dist-types/app/api/proxy/verify/route.d.ts +4 -0
  38. package/dist-types/app/api/proxy/verify/route.d.ts.map +1 -0
  39. package/dist-types/app/api/search/bing/route.d.ts +5 -0
  40. package/dist-types/app/api/search/bing/route.d.ts.map +1 -0
  41. package/dist-types/app/api/search/brave/route.d.ts +5 -0
  42. package/dist-types/app/api/search/brave/route.d.ts.map +1 -0
  43. package/dist-types/app/api/search/exa/route.d.ts +5 -0
  44. package/dist-types/app/api/search/exa/route.d.ts.map +1 -0
  45. package/dist-types/app/api/search/google/route.d.ts +30 -0
  46. package/dist-types/app/api/search/google/route.d.ts.map +1 -0
  47. package/dist-types/app/api/search/route.d.ts +7 -0
  48. package/dist-types/app/api/search/route.d.ts.map +1 -0
  49. package/dist-types/app/api/search/tavily/route.d.ts +5 -0
  50. package/dist-types/app/api/search/tavily/route.d.ts.map +1 -0
  51. package/dist-types/app/api/search/verify/route.d.ts +4 -0
  52. package/dist-types/app/api/search/verify/route.d.ts.map +1 -0
  53. package/dist-types/app/api/tabular/route.d.ts +9 -0
  54. package/dist-types/app/api/tabular/route.d.ts.map +1 -0
  55. package/dist-types/lib/chat/assistant-source-scope.d.ts +10 -0
  56. package/dist-types/lib/chat/assistant-source-scope.d.ts.map +1 -0
  57. package/dist-types/lib/chat/conversation-memory.d.ts +80 -0
  58. package/dist-types/lib/chat/conversation-memory.d.ts.map +1 -0
  59. package/dist-types/lib/chat/gateway-fallbacks.d.ts +7 -0
  60. package/dist-types/lib/chat/gateway-fallbacks.d.ts.map +1 -0
  61. package/dist-types/lib/chat/image-description.d.ts +10 -0
  62. package/dist-types/lib/chat/image-description.d.ts.map +1 -0
  63. package/dist-types/lib/chat/media-retrieval-routing.d.ts +14 -0
  64. package/dist-types/lib/chat/media-retrieval-routing.d.ts.map +1 -0
  65. package/dist-types/lib/chat/media-source-routing.d.ts +53 -0
  66. package/dist-types/lib/chat/media-source-routing.d.ts.map +1 -0
  67. package/dist-types/lib/chat/message-input.d.ts +8 -0
  68. package/dist-types/lib/chat/message-input.d.ts.map +1 -0
  69. package/dist-types/lib/chat/model-provider.d.ts +23 -0
  70. package/dist-types/lib/chat/model-provider.d.ts.map +1 -0
  71. package/dist-types/lib/chat/response-style.d.ts +2 -0
  72. package/dist-types/lib/chat/response-style.d.ts.map +1 -0
  73. package/dist-types/lib/chat/retrieval-ranking.d.ts +32 -0
  74. package/dist-types/lib/chat/retrieval-ranking.d.ts.map +1 -0
  75. package/dist-types/lib/chat/retrieval-routing.d.ts +39 -0
  76. package/dist-types/lib/chat/retrieval-routing.d.ts.map +1 -0
  77. package/dist-types/lib/chat/tabular-dataset-routing.d.ts +15 -0
  78. package/dist-types/lib/chat/tabular-dataset-routing.d.ts.map +1 -0
  79. package/dist-types/lib/chat/tabular-query-plan.d.ts +12 -0
  80. package/dist-types/lib/chat/tabular-query-plan.d.ts.map +1 -0
  81. package/dist-types/lib/chat/tabular-routing.d.ts +41 -0
  82. package/dist-types/lib/chat/tabular-routing.d.ts.map +1 -0
  83. package/dist-types/lib/chat/tabular-sandbox-files.d.ts +33 -0
  84. package/dist-types/lib/chat/tabular-sandbox-files.d.ts.map +1 -0
  85. package/dist-types/lib/chat/tabular-visualization.d.ts +31 -0
  86. package/dist-types/lib/chat/tabular-visualization.d.ts.map +1 -0
  87. package/dist-types/lib/chat/tool-call-repair.d.ts +11 -0
  88. package/dist-types/lib/chat/tool-call-repair.d.ts.map +1 -0
  89. package/dist-types/lib/chat/tool-context.d.ts +72 -0
  90. package/dist-types/lib/chat/tool-context.d.ts.map +1 -0
  91. package/dist-types/lib/chat/tool-registry.d.ts +3 -0
  92. package/dist-types/lib/chat/tool-registry.d.ts.map +1 -0
  93. package/dist-types/lib/chat/video-rag-routing.d.ts +33 -0
  94. package/dist-types/lib/chat/video-rag-routing.d.ts.map +1 -0
  95. package/dist-types/lib/chat/visual-routing.d.ts +53 -0
  96. package/dist-types/lib/chat/visual-routing.d.ts.map +1 -0
  97. package/dist-types/lib/enterprise-client.d.ts +31 -0
  98. package/dist-types/lib/enterprise-client.d.ts.map +1 -0
  99. package/dist-types/lib/marketplace/tool-runtime-config.d.ts +7 -0
  100. package/dist-types/lib/marketplace/tool-runtime-config.d.ts.map +1 -0
  101. package/dist-types/lib/media/knowledge.d.ts +47 -0
  102. package/dist-types/lib/media/knowledge.d.ts.map +1 -0
  103. package/dist-types/lib/media/pdf-inspection.d.ts +30 -0
  104. package/dist-types/lib/media/pdf-inspection.d.ts.map +1 -0
  105. package/dist-types/lib/media/source-identity.d.ts +12 -0
  106. package/dist-types/lib/media/source-identity.d.ts.map +1 -0
  107. package/dist-types/lib/media/source-utils.d.ts +157 -0
  108. package/dist-types/lib/media/source-utils.d.ts.map +1 -0
  109. package/dist-types/lib/media/video/quick-look.d.ts +60 -0
  110. package/dist-types/lib/media/video/quick-look.d.ts.map +1 -0
  111. package/dist-types/lib/media/video-intelligence-adapter.d.ts +521 -0
  112. package/dist-types/lib/media/video-intelligence-adapter.d.ts.map +1 -0
  113. package/dist-types/lib/media/video-intelligence-connection.d.ts +32 -0
  114. package/dist-types/lib/media/video-intelligence-connection.d.ts.map +1 -0
  115. package/dist-types/lib/remote-files.d.ts +12 -0
  116. package/dist-types/lib/remote-files.d.ts.map +1 -0
  117. package/dist-types/lib/search/provider-verification.d.ts +5 -0
  118. package/dist-types/lib/search/provider-verification.d.ts.map +1 -0
  119. package/dist-types/runtime-host.d.ts +20 -0
  120. package/dist-types/runtime-host.d.ts.map +1 -0
  121. package/package.json +338 -3
  122. package/src/app/api/chat/feedback/route.ts +77 -0
  123. package/src/app/api/chat/route.ts +2277 -0
  124. package/src/app/api/chat/tools.ts +3563 -0
  125. package/src/app/api/config/test-embedding/route.ts +39 -0
  126. package/src/app/api/config/verify/route.ts +309 -0
  127. package/src/app/api/describe-image/route.ts +108 -0
  128. package/src/app/api/doc-editor/edit/route.ts +87 -0
  129. package/src/app/api/doc-editor/export/route.ts +32 -0
  130. package/src/app/api/doc-editor/parse/route.ts +74 -0
  131. package/src/app/api/doc-editor/restore/route.ts +70 -0
  132. package/src/app/api/document/sign/route.ts +55 -0
  133. package/src/app/api/files/remote/route.ts +32 -0
  134. package/src/app/api/mcp/[connectionId]/route.ts +29 -0
  135. package/src/app/api/mcp/[connectionId]/test/route.ts +13 -0
  136. package/src/app/api/mcp/route.ts +22 -0
  137. package/src/app/api/parse-file/route.ts +86 -0
  138. package/src/app/api/pdf-page/route.ts +40 -0
  139. package/src/app/api/proxy/verify/route.ts +78 -0
  140. package/src/app/api/search/bing/route.ts +86 -0
  141. package/src/app/api/search/brave/route.ts +88 -0
  142. package/src/app/api/search/exa/route.ts +100 -0
  143. package/src/app/api/search/google/route.ts +147 -0
  144. package/src/app/api/search/route.ts +28 -0
  145. package/src/app/api/search/tavily/route.ts +93 -0
  146. package/src/app/api/search/verify/route.ts +27 -0
  147. package/src/app/api/tabular/route.ts +68 -0
  148. package/src/lib/chat/assistant-source-scope.ts +32 -0
  149. package/src/lib/chat/conversation-memory.ts +621 -0
  150. package/src/lib/chat/gateway-fallbacks.ts +12 -0
  151. package/src/lib/chat/image-description.ts +21 -0
  152. package/src/lib/chat/media-retrieval-routing.ts +37 -0
  153. package/src/lib/chat/media-source-routing.ts +100 -0
  154. package/src/lib/chat/message-input.ts +48 -0
  155. package/src/lib/chat/model-provider.ts +157 -0
  156. package/src/lib/chat/response-style.ts +3 -0
  157. package/src/lib/chat/retrieval-ranking.ts +126 -0
  158. package/src/lib/chat/retrieval-routing.ts +221 -0
  159. package/src/lib/chat/tabular-dataset-routing.ts +93 -0
  160. package/src/lib/chat/tabular-query-plan.ts +195 -0
  161. package/src/lib/chat/tabular-routing.ts +122 -0
  162. package/src/lib/chat/tabular-sandbox-files.ts +78 -0
  163. package/src/lib/chat/tabular-visualization.ts +165 -0
  164. package/src/lib/chat/tool-call-repair.ts +84 -0
  165. package/src/lib/chat/tool-context.ts +1222 -0
  166. package/src/lib/chat/tool-registry.ts +6 -0
  167. package/src/lib/chat/video-rag-routing.ts +149 -0
  168. package/src/lib/chat/visual-routing.ts +225 -0
  169. package/src/lib/enterprise-client.ts +96 -0
  170. package/src/lib/marketplace/tool-runtime-config.ts +155 -0
  171. package/src/lib/media/knowledge.ts +489 -0
  172. package/src/lib/media/pdf-inspection.ts +280 -0
  173. package/src/lib/media/source-identity.ts +50 -0
  174. package/src/lib/media/source-utils.ts +1156 -0
  175. package/src/lib/media/video/quick-look.ts +368 -0
  176. package/src/lib/media/video-intelligence-adapter.ts +1946 -0
  177. package/src/lib/media/video-intelligence-connection.ts +151 -0
  178. package/src/lib/remote-files.ts +182 -0
  179. package/src/lib/search/provider-verification.ts +95 -0
  180. package/src/runtime-host.ts +31 -0
@@ -0,0 +1,93 @@
1
+ import type { TabularDatasetMeta } from '@larkup/core/tabular-store';
2
+
3
+ function normalized(value: string) {
4
+ return value
5
+ .toLocaleLowerCase()
6
+ .replace(/[^\p{L}\p{N}]+/gu, ' ')
7
+ .trim();
8
+ }
9
+
10
+ function hasColumnMention(text: string, column: string) {
11
+ const query = normalized(text);
12
+ const name = normalized(column);
13
+ if (!query || !name) return false;
14
+ if (query.includes(name)) return true;
15
+
16
+ const words = name.split(' ').filter(Boolean);
17
+ return (
18
+ words.length === 1 && words[0].length >= 4 && new RegExp(`\\b${words[0]}\\b`, 'iu').test(query)
19
+ );
20
+ }
21
+
22
+ function isTemporalRequest(text: string) {
23
+ return /\b(?:month(?:ly)?|quarter(?:ly)?|year(?:ly)?|trend|time\s+series|over\s+time)\b/iu.test(
24
+ text,
25
+ );
26
+ }
27
+
28
+ function namesRequestedInText(text: string, dataset: TabularDatasetMeta) {
29
+ return dataset.columns.filter((column) => hasColumnMention(text, column.name));
30
+ }
31
+
32
+ function distinctiveFileNameScore(text: string, fileName: string) {
33
+ const query = normalized(text);
34
+ const fileTerms = normalized(fileName)
35
+ .split(' ')
36
+ .filter(
37
+ (term) => term.length >= 7 && !['sample', 'returns', 'orders', 'people'].includes(term),
38
+ );
39
+ return fileTerms.some((term) => new RegExp(`\\b${term}\\b`, 'iu').test(query)) ? 8 : 0;
40
+ }
41
+
42
+ function scoreDataset(text: string, dataset: TabularDatasetMeta) {
43
+ let score = distinctiveFileNameScore(text, dataset.fileName);
44
+ const temporal = isTemporalRequest(text);
45
+ for (const column of namesRequestedInText(text, dataset)) {
46
+ if (column.type === 'number') score += 6;
47
+ else if (column.type === 'date' && temporal) score += 2;
48
+ else if (/\bby\b/iu.test(text)) score += 3;
49
+ }
50
+ return score;
51
+ }
52
+
53
+ /**
54
+ * Find one unambiguous dataset for code analysis when a model omitted the
55
+ * optional datasetId. This is intentionally schema-led: a matching filename
56
+ * alone is not enough to stage a potentially large source in the sandbox.
57
+ */
58
+ export function selectTabularDatasetForQuestion(
59
+ text: string | undefined,
60
+ datasets: TabularDatasetMeta[],
61
+ ) {
62
+ if (datasets.length === 1) return datasets[0]?.id;
63
+ if (!text) return undefined;
64
+
65
+ const candidates = datasets
66
+ .map((dataset) => ({ dataset, score: scoreDataset(text, dataset) }))
67
+ .sort((left, right) => right.score - left.score);
68
+ const best = candidates[0];
69
+ const next = candidates[1];
70
+ if (!best || best.score < 6 || best.score === next?.score) return undefined;
71
+ return best.dataset.id;
72
+ }
73
+
74
+ /**
75
+ * Correct a model-selected sheet only when the question's schema makes a
76
+ * different dataset objectively better. This keeps workbook selection
77
+ * generic while avoiding a filename such as "sales.xlsx" winning over a
78
+ * sheet that actually contains both Sales and Profit.
79
+ */
80
+ export function resolveTabularDatasetForQuestion(
81
+ text: string | undefined,
82
+ datasets: TabularDatasetMeta[],
83
+ selectedDatasetId: string,
84
+ ) {
85
+ const selected = datasets.find((dataset) => dataset.id === selectedDatasetId);
86
+ if (!text || !selected || datasets.length < 2) return selectedDatasetId;
87
+
88
+ const selectedScore = scoreDataset(text, selected);
89
+ const resolved = selectTabularDatasetForQuestion(text, datasets);
90
+ if (!resolved || resolved === selectedDatasetId) return selectedDatasetId;
91
+ const best = datasets.find((dataset) => dataset.id === resolved);
92
+ return best && scoreDataset(text, best) > selectedScore ? best.id : selectedDatasetId;
93
+ }
@@ -0,0 +1,195 @@
1
+ import type { TabularDataset, TabularQueryRequest } from '@larkup/core/tabular-store';
2
+
3
+ export type InferredTabularPlan = {
4
+ request: TabularQueryRequest;
5
+ chartTitle?: string;
6
+ };
7
+
8
+ function numericColumns(dataset: TabularDataset): string[] {
9
+ return dataset.columns.filter((column) => column.type === 'number').map((column) => column.name);
10
+ }
11
+
12
+ function normalized(value: string) {
13
+ return value
14
+ .toLocaleLowerCase()
15
+ .replace(/[^\p{L}\p{N}]+/gu, ' ')
16
+ .trim();
17
+ }
18
+
19
+ function columnMentioned(text: string, column: string) {
20
+ const query = normalized(text);
21
+ const name = normalized(column);
22
+ if (!query || !name) return false;
23
+ if (query.includes(name)) return true;
24
+ const words = name.split(' ').filter(Boolean);
25
+ return (
26
+ words.length === 1 && words[0].length >= 4 && new RegExp(`\\b${words[0]}\\b`, 'iu').test(query)
27
+ );
28
+ }
29
+
30
+ function explicitNumericMeasures(
31
+ text: string,
32
+ dataset: TabularDataset,
33
+ request: TabularQueryRequest,
34
+ ) {
35
+ const numeric = numericColumns(dataset);
36
+ const requested = (request.aggregations ?? [])
37
+ .map((aggregation) => aggregation.column)
38
+ .filter((column) => numeric.includes(column));
39
+ const mentioned = numeric.filter((column) => columnMentioned(text, column));
40
+ return [...new Set([...requested, ...mentioned])];
41
+ }
42
+
43
+ function columnPattern(column: string) {
44
+ return normalized(column)
45
+ .split(' ')
46
+ .map((term) => term.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
47
+ .join('[-_\\s]+');
48
+ }
49
+
50
+ function groupingColumn(text: string, dataset: TabularDataset) {
51
+ return dataset.columns.find(
52
+ (column) =>
53
+ column.type !== 'number' &&
54
+ new RegExp(`\\bby\\s+(?:the\\s+)?${columnPattern(column.name)}\\b`, 'iu').test(text),
55
+ )?.name;
56
+ }
57
+
58
+ function requestedOperation(
59
+ measure: string,
60
+ request: TabularQueryRequest,
61
+ text: string,
62
+ ): NonNullable<TabularQueryRequest['aggregations']>[number]['op'] {
63
+ const supplied = request.aggregations?.find((aggregation) => aggregation.column === measure)?.op;
64
+ if (supplied) return supplied;
65
+ if (/\b(?:average|avg|mean)\b/iu.test(text)) return 'avg';
66
+ if (/\bmedian\b/iu.test(text)) return 'median';
67
+ // “Lowest to highest” and “highest month” usually describe ordering or a
68
+ // later comparison of grouped totals, not a request for the minimum raw
69
+ // order-level value. Reserve min/max for an explicit aggregate operation.
70
+ if (/\b(?:minimum|min)\b/iu.test(text)) return 'min';
71
+ if (/\b(?:maximum|max)\b/iu.test(text)) return 'max';
72
+ return 'sum';
73
+ }
74
+
75
+ function requestedSortOrder(text: string) {
76
+ if (/\b(?:lowest\s+to\s+highest|ascending|asc)\b/iu.test(text)) return 'asc' as const;
77
+ if (/\b(?:highest\s+to\s+lowest|descending|desc)\b/iu.test(text)) return 'desc' as const;
78
+ return undefined;
79
+ }
80
+
81
+ function normalizeAggregations(
82
+ dataset: TabularDataset,
83
+ request: TabularQueryRequest,
84
+ ): TabularQueryRequest {
85
+ if (!request.aggregations?.length) return request;
86
+ const numeric = numericColumns(dataset);
87
+ const isCount = (aggregation: NonNullable<TabularQueryRequest['aggregations']>[number]) =>
88
+ aggregation.op === 'count' || aggregation.op === 'countDistinct';
89
+ const valid = request.aggregations.filter(
90
+ (aggregation) => isCount(aggregation) || numeric.includes(aggregation.column),
91
+ );
92
+ if (valid.length === request.aggregations.length || numeric.length !== 1) return request;
93
+
94
+ // The model may select a text label such as a representative/name as a
95
+ // measure. A dataset with exactly one numeric field has an unambiguous,
96
+ // schema-derived correction; otherwise leave the request unchanged.
97
+ return {
98
+ ...request,
99
+ aggregations: request.aggregations.map((aggregation) =>
100
+ isCount(aggregation) ? aggregation : { ...aggregation, column: numeric[0] },
101
+ ),
102
+ };
103
+ }
104
+
105
+ function normalizeDistinctCountIntent(
106
+ text: string,
107
+ request: TabularQueryRequest,
108
+ ): TabularQueryRequest {
109
+ if (!/\b(?:unique|distinct)\b/i.test(text) || !request.aggregations?.length) return request;
110
+
111
+ return {
112
+ ...request,
113
+ aggregations: request.aggregations.map((aggregation) =>
114
+ aggregation.op === 'count' ? { ...aggregation, op: 'countDistinct' } : aggregation,
115
+ ),
116
+ };
117
+ }
118
+
119
+ /**
120
+ * Normalize a clear calendar request from the dataset schema. It never
121
+ * assumes business columns or domain names: a time series is derived only
122
+ * when there is one date column and one numeric measure.
123
+ */
124
+ export function inferTabularPlan(
125
+ requestText: string | undefined,
126
+ dataset: TabularDataset,
127
+ request: TabularQueryRequest,
128
+ ): InferredTabularPlan {
129
+ const text = requestText?.toLocaleLowerCase() ?? '';
130
+ const dateColumn = dataset.columns.find((column) => column.type === 'date')?.name;
131
+ const numeric = numericColumns(dataset);
132
+ const requestedMeasures = explicitNumericMeasures(text, dataset, request);
133
+ const measureColumns =
134
+ requestedMeasures.length > 0 ? requestedMeasures : numeric.length === 1 ? [numeric[0]] : [];
135
+ const year = text.match(/\b(20\d{2})\b/)?.[1];
136
+
137
+ if (
138
+ dateColumn &&
139
+ measureColumns.length > 0 &&
140
+ /\b(?:monthly|per month|month(?:ly)? trend)\b/i.test(text)
141
+ ) {
142
+ const bucketColumn = `${dateColumn}_month`;
143
+ const filters = (request.filters ?? []).filter((filter) => filter.column !== dateColumn);
144
+ if (year) {
145
+ filters.push({ column: dateColumn, op: 'gte', value: `${year}-01-01` });
146
+ filters.push({ column: dateColumn, op: 'lt', value: `${Number(year) + 1}-01-01` });
147
+ }
148
+ return {
149
+ request: {
150
+ ...request,
151
+ // A model may also send the source column names. Once rows are
152
+ // grouped those names no longer exist (for example, Revenue becomes
153
+ // sum_Revenue), so retaining the projection would turn every result
154
+ // row into an empty object.
155
+ columns: undefined,
156
+ filters,
157
+ timeBucket: { column: dateColumn, grain: 'month' },
158
+ groupBy: [bucketColumn],
159
+ aggregations: measureColumns.map((column) => ({
160
+ column,
161
+ op: requestedOperation(column, request, text),
162
+ })),
163
+ sortBy: bucketColumn,
164
+ sortOrder: 'asc',
165
+ limit: 120,
166
+ },
167
+ chartTitle: `Monthly ${measureColumns.join(' and ')} trend${year ? ` for ${year}` : ''}`,
168
+ };
169
+ }
170
+
171
+ const groupBy = groupingColumn(text, dataset);
172
+ if (groupBy && requestedMeasures.length > 0) {
173
+ const aggregations = requestedMeasures.map((column) => ({
174
+ column,
175
+ op: requestedOperation(column, request, text),
176
+ }));
177
+ const sortOrder = requestedSortOrder(text);
178
+ return {
179
+ request: {
180
+ ...request,
181
+ columns: undefined,
182
+ groupBy: [groupBy],
183
+ aggregations,
184
+ ...(sortOrder
185
+ ? { sortBy: `${aggregations[0].op}_${aggregations[0].column}`, sortOrder }
186
+ : {}),
187
+ limit: Math.min(request.limit ?? 120, 120),
188
+ },
189
+ };
190
+ }
191
+
192
+ return {
193
+ request: normalizeDistinctCountIntent(text, normalizeAggregations(dataset, request)),
194
+ };
195
+ }
@@ -0,0 +1,122 @@
1
+ /**
2
+ * Route ordinary sheet questions through the structured query engine, while
3
+ * sending explicit joins and advanced analytics to the sandbox. This keeps
4
+ * common Excel/CSV questions independent from local Python availability.
5
+ */
6
+ export function tabularToolsForStep(input: {
7
+ stepNumber: number;
8
+ toolNames: string[];
9
+ requiresSandbox?: boolean;
10
+ }) {
11
+ if (input.stepNumber === 0) {
12
+ if (input.requiresSandbox && input.toolNames.includes('executeAnalysis')) {
13
+ return {
14
+ toolChoice: { type: 'tool' as const, toolName: 'executeAnalysis' },
15
+ activeTools: ['executeAnalysis'],
16
+ };
17
+ }
18
+ return {
19
+ toolChoice: { type: 'tool' as const, toolName: 'queryTabularData' },
20
+ activeTools: ['queryTabularData'],
21
+ };
22
+ }
23
+ if (input.stepNumber === 1) {
24
+ // The first evidence tool has already returned authoritative rows (and,
25
+ // for chart requests, queryTabularData has attached a normalized chart).
26
+ // Re-exposing presentation and analysis tools here lets some models emit
27
+ // a second, empty chart call or repeat the analysis instead of answering.
28
+ return { toolChoice: 'none' as const, activeTools: [] };
29
+ }
30
+ return { toolChoice: 'none' as const, activeTools: [] };
31
+ }
32
+
33
+ /** Stop once the host has attached a renderable chart to authoritative rows. */
34
+ export function hasCompletedTabularVisualizationStep(
35
+ steps: Array<{
36
+ toolResults?: readonly unknown[];
37
+ }>,
38
+ ) {
39
+ const lastStep = steps.at(-1);
40
+ return Boolean(
41
+ lastStep?.toolResults?.some((result) => {
42
+ if (!result || typeof result !== 'object' || !('toolName' in result)) return false;
43
+ if ((result as { toolName?: unknown }).toolName !== 'queryTabularData') return false;
44
+ const output = (result as { output?: unknown }).output;
45
+ return Boolean(
46
+ output &&
47
+ typeof output === 'object' &&
48
+ 'visualization' in output &&
49
+ (output as { visualization?: unknown }).visualization,
50
+ );
51
+ }),
52
+ );
53
+ }
54
+
55
+ /**
56
+ * Identify requests whose answer requires an out-of-core calculation rather
57
+ * than a single structured table query. This is phrased around analytical
58
+ * operations, so it works for any workbook and does not depend on sheet or
59
+ * column names.
60
+ */
61
+ export function requiresTabularSandbox(text: string): boolean {
62
+ return /\b(?:join|correlat(?:e|ion)|regression|cluster(?:ing)?|forecast(?:ing)?|statistical|sandbox|python|matplotlib|seaborn|scikit(?:-learn)?|custom\s+(?:analysis|chart|visuali[sz]ation)|across\s+(?:sheets?|tables?|datasets?|files?)|between\s+(?:sheets?|tables?|datasets?|files?))\b/i.test(
63
+ text,
64
+ );
65
+ }
66
+
67
+ /**
68
+ * A workspace can contain spreadsheets alongside PDFs, media, and notes.
69
+ * Only force the table path when the wording actually asks for tabular facts;
70
+ * otherwise normal source retrieval remains free to select the right modality.
71
+ */
72
+ export function isLikelyTabularQuestion(input: {
73
+ text: string;
74
+ columnNames: string[];
75
+ datasetNames: string[];
76
+ }) {
77
+ const text = input.text.trim().toLocaleLowerCase();
78
+ if (!text) return false;
79
+ if (
80
+ /\b(?:spreadsheet|excel|csv|worksheet|sheet|dataset|row|column|filter|sort|group(?:ed|ing)?|aggregate|average|median|total|sum|count|highest|lowest|largest|smallest|maximum|minimum|trend|distribution)\b/.test(
81
+ text,
82
+ ) ||
83
+ /\b(?:arithmetic|geometric|weighted)\s+mean\b|\bmean\s+(?:of|for|by)\b/.test(text)
84
+ ) {
85
+ return true;
86
+ }
87
+
88
+ // “Put it in a table” describes the requested answer format, not the
89
+ // source modality. Treat table as tabular only when the surrounding words
90
+ // make a data table explicit, so a media or document question can still
91
+ // retrieve its own source in a mixed workspace.
92
+ if (
93
+ /\b(?:data|database|pivot|lookup)\s+table\b|\btable\s+(?:data|rows?|columns?|schema|lookup)\b/.test(
94
+ text,
95
+ )
96
+ ) {
97
+ return true;
98
+ }
99
+
100
+ const mentionsDataset = input.datasetNames.some((name) => {
101
+ const stem = name
102
+ .replace(/\.(?:csv|xlsx|xls|json)$/i, '')
103
+ .trim()
104
+ .toLocaleLowerCase();
105
+ return stem.length >= 3 && text.includes(stem);
106
+ });
107
+ if (mentionsDataset) return true;
108
+
109
+ return input.columnNames.some((column) => {
110
+ const normalized = column.trim().toLocaleLowerCase();
111
+ if (normalized.length >= 4 && text.includes(normalized)) return true;
112
+ const terms = normalized.match(/[\p{L}\p{N}_]+/gu) ?? [];
113
+ const matchedTerms = terms.filter((term) => term.length >= 4 && text.includes(term));
114
+ if (matchedTerms.length >= 2) return true;
115
+
116
+ // Metrics in real workbooks often have long headers while the user asks
117
+ // with one distinctive word. A sufficiently specific header token is
118
+ // enough to choose the exact table tool without relying on dataset- or
119
+ // domain-specific routing rules.
120
+ return matchedTerms.some((term) => term.length >= 6);
121
+ });
122
+ }
@@ -0,0 +1,78 @@
1
+ import type { TabularDataset } from '@larkup/core/tabular-store';
2
+
3
+ export type TabularSandboxFile = { name: string; content: string };
4
+
5
+ export type TabularSandboxManifest = {
6
+ datasets: Array<{
7
+ id: string;
8
+ fileName: string;
9
+ file: string;
10
+ rowCount: number;
11
+ columns: string[];
12
+ }>;
13
+ };
14
+
15
+ function csvCell(value: unknown) {
16
+ const text = String(value ?? '');
17
+ return text.includes(',') || text.includes('"') || text.includes('\n')
18
+ ? `"${text.replace(/"/g, '""')}"`
19
+ : text;
20
+ }
21
+
22
+ /** Serialize a stored dataset with RFC 4180-compatible quoting for sandbox analysis. */
23
+ export function tabularDatasetToCsv(dataset: TabularDataset) {
24
+ const columns = dataset.columns.map((column) => column.name);
25
+ return [
26
+ columns.join(','),
27
+ ...dataset.rows.map((row) => columns.map((column) => csvCell(row[column])).join(',')),
28
+ ].join('\n');
29
+ }
30
+
31
+ /**
32
+ * Stage one or more selected datasets for a sandbox without exposing row data
33
+ * to the chat model. A single dataset keeps the long-standing `data.csv`
34
+ * contract; multiple datasets receive stable generic filenames plus a manifest
35
+ * that maps each source ID and sheet name to its CSV file.
36
+ */
37
+ export function createTabularSandboxFiles(datasets: TabularDataset[]): {
38
+ files: TabularSandboxFile[];
39
+ manifest?: TabularSandboxManifest;
40
+ } {
41
+ if (datasets.length === 0) return { files: [] };
42
+
43
+ const multiple = datasets.length > 1;
44
+ const manifest: TabularSandboxManifest = { datasets: [] };
45
+ const files = datasets.map((dataset, index) => {
46
+ const name = multiple ? `dataset-${index + 1}.csv` : 'data.csv';
47
+ manifest.datasets.push({
48
+ id: dataset.id,
49
+ fileName: dataset.fileName,
50
+ file: name,
51
+ rowCount: dataset.rowCount,
52
+ columns: dataset.columns.map((column) => column.name),
53
+ });
54
+ return { name, content: tabularDatasetToCsv(dataset) };
55
+ });
56
+
57
+ if (multiple) {
58
+ files.push({ name: 'datasets.json', content: JSON.stringify(manifest, null, 2) });
59
+ return { files, manifest };
60
+ }
61
+
62
+ return { files };
63
+ }
64
+
65
+ /**
66
+ * Recover a common single-sheet code pattern when every selected dataset was
67
+ * requested but the generated code still reads `data.csv`. Rewriting is safe
68
+ * only when the number of reads exactly matches the selected datasets.
69
+ */
70
+ export function remapLegacyMultiDatasetReads(code: string, datasetCount: number): string {
71
+ if (datasetCount < 2) return code;
72
+
73
+ const legacyRead = /pd\s*\.\s*read_(?:csv|excel)\(\s*(['"])data\.csv\1(?:\s*,[^)]*)?\s*\)/g;
74
+ if ([...code.matchAll(legacyRead)].length !== datasetCount) return code;
75
+
76
+ let datasetIndex = 0;
77
+ return code.replace(legacyRead, () => `pd.read_csv('dataset-${++datasetIndex}.csv')`);
78
+ }
@@ -0,0 +1,165 @@
1
+ import { formatChartLabel, normalizeChartConfig, numericValue } from '@larkup/core/chart-config';
2
+
3
+ export type TabularChartConfig = {
4
+ chartType: 'bar' | 'line';
5
+ title: string;
6
+ data: Record<string, unknown>[];
7
+ xAxisKey: string;
8
+ series: { dataKey: string; label: string }[];
9
+ showLegend: boolean;
10
+ xAxisLabel?: string;
11
+ yAxisLabel?: string;
12
+ };
13
+
14
+ type TabularVisualizationResult = {
15
+ visualization?: TabularChartConfig;
16
+ error?: unknown;
17
+ };
18
+
19
+ function formatChartCategory(value: string) {
20
+ const calendarMonth = value.match(/^(\d{4})-(\d{2})$/);
21
+ if (calendarMonth) {
22
+ return new Intl.DateTimeFormat('en-US', {
23
+ month: 'short',
24
+ year: 'numeric',
25
+ timeZone: 'UTC',
26
+ }).format(new Date(Date.UTC(Number(calendarMonth[1]), Number(calendarMonth[2]) - 1, 1)));
27
+ }
28
+ return formatChartLabel(value);
29
+ }
30
+
31
+ function formatSeriesLabel(value: string) {
32
+ return formatChartLabel(
33
+ value.replace(/^(?:sum|avg|countDistinct|count|min|max|median)[_\s-]+/i, ''),
34
+ );
35
+ }
36
+
37
+ export function requestsVisualization(text: string): boolean {
38
+ return /\b(chart|graph|plot|visuali[sz]e|distribution|breakdown|trends?|compare|show\s+me)\b/i.test(
39
+ text,
40
+ );
41
+ }
42
+
43
+ function normalizedTerms(value: string) {
44
+ return value.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
45
+ }
46
+
47
+ function columnIsMentioned(text: string, column: string) {
48
+ const requestTerms = new Set(normalizedTerms(text));
49
+ const columnTerms = normalizedTerms(column).filter((term) => term.length >= 3);
50
+ return columnTerms.some((term) => requestTerms.has(term));
51
+ }
52
+
53
+ function hasNumericValues(rows: Record<string, unknown>[], column: string) {
54
+ return rows.some((row) => numericValue(row[column]) !== undefined);
55
+ }
56
+
57
+ function hasVaryingValues(rows: Record<string, unknown>[], column: string) {
58
+ const values = new Set(
59
+ rows
60
+ .map((row) => row[column])
61
+ .filter((value) => value !== null && value !== undefined && String(value).trim())
62
+ .map((value) => String(value)),
63
+ );
64
+ return values.size > 1;
65
+ }
66
+
67
+ function isTemporalColumn(column: string, rows: Record<string, unknown>[]) {
68
+ if (/\b(?:year|date|month|quarter|week|day|time)\b/i.test(formatChartLabel(column))) return true;
69
+ return rows.some((row) => /^(?:19|20)\d{2}$/.test(String(row[column] ?? '').trim()));
70
+ }
71
+
72
+ /** Build a chart only from an already-queried, bounded tabular result. */
73
+ export function createTabularVisualization(
74
+ requestText: string | undefined,
75
+ result: { columns: string[]; rows: Record<string, unknown>[] },
76
+ ): TabularChartConfig | undefined {
77
+ if (!requestText || !requestsVisualization(requestText) || result.rows.length === 0) {
78
+ return undefined;
79
+ }
80
+
81
+ const wantsTimeSeries =
82
+ /\b(?:line(?:\s+\w+){0,2}\s+chart|trends?|over time|time series|year[ -]to[ -]year|available years?|monthly|quarterly|annually)\b/i.test(
83
+ requestText,
84
+ );
85
+ const numericColumns = result.columns.filter((column) => hasNumericValues(result.rows, column));
86
+ const temporalColumn = wantsTimeSeries
87
+ ? result.columns.find(
88
+ (column) => isTemporalColumn(column, result.rows) && hasVaryingValues(result.rows, column),
89
+ )
90
+ : undefined;
91
+ const categoryColumn = result.columns.find(
92
+ (column) =>
93
+ !numericColumns.includes(column) &&
94
+ hasVaryingValues(result.rows, column) &&
95
+ !/^_*empty(?:[_-]?\d+)?$/i.test(column.trim()),
96
+ );
97
+ const xAxisKey = temporalColumn ?? categoryColumn ?? result.columns[0] ?? '';
98
+ const seriesCandidates = numericColumns.filter((column) => column !== xAxisKey);
99
+ const mentionedSeries = seriesCandidates.filter((column) =>
100
+ columnIsMentioned(requestText, column),
101
+ );
102
+ const seriesColumns = (mentionedSeries.length > 0 ? mentionedSeries : seriesCandidates).slice(
103
+ 0,
104
+ 3,
105
+ );
106
+ const chartType = wantsTimeSeries ? 'line' : 'bar';
107
+
108
+ const candidate = normalizeChartConfig({
109
+ chartType,
110
+ title: 'Data chart',
111
+ data: result.rows.slice(0, 50),
112
+ xAxisKey,
113
+ series: seriesColumns.map((column) => ({ dataKey: column, label: column })),
114
+ });
115
+ if (candidate.error) return undefined;
116
+
117
+ return {
118
+ ...candidate,
119
+ chartType: candidate.chartType as 'bar' | 'line',
120
+ title: `${chartType === 'line' ? 'Trend' : 'Distribution'} by ${formatChartLabel(candidate.xAxisKey)}`,
121
+ series: candidate.series.map((series) => ({
122
+ ...series,
123
+ label:
124
+ !series.label || series.label === series.dataKey
125
+ ? formatSeriesLabel(series.dataKey)
126
+ : series.label,
127
+ })),
128
+ showLegend: candidate.showLegend ?? candidate.series.length > 1,
129
+ };
130
+ }
131
+
132
+ /**
133
+ * Keep the prose beside a rendered chart compact and deterministic. The chart
134
+ * already exposes every plotted row, so echoing those rows as assistant text
135
+ * is both noisy and a common source of provider-specific output failures.
136
+ */
137
+ export function formatTabularVisualizationAnswer(result: TabularVisualizationResult): string {
138
+ const chart = result.visualization;
139
+ if (!chart) {
140
+ return typeof result.error === 'string' && result.error.trim()
141
+ ? `I couldn’t create the requested chart: ${result.error.trim()}`
142
+ : 'I couldn’t create a chart from the available tabular result.';
143
+ }
144
+
145
+ const insights = chart.series.flatMap((series) => {
146
+ const points = chart.data.flatMap((row) => {
147
+ const value = numericValue(row[series.dataKey]);
148
+ const category = row[chart.xAxisKey];
149
+ return value === undefined || category === undefined || category === null
150
+ ? []
151
+ : [{ value, category: String(category) }];
152
+ });
153
+ if (points.length < 2) return [];
154
+ const peak = points.reduce((best, point) => (point.value > best.value ? point : best));
155
+ return `${series.label || formatSeriesLabel(series.dataKey)} peaks at ${new Intl.NumberFormat(
156
+ 'en-US',
157
+ { maximumFractionDigits: 2 },
158
+ ).format(peak.value)} in ${formatChartCategory(peak.category)}`;
159
+ });
160
+
161
+ const introduction = chart.title?.trim()
162
+ ? `Here’s the requested chart: ${chart.title.trim()}.`
163
+ : 'Here’s the requested chart.';
164
+ return insights.length > 0 ? `${introduction} ${insights.join('; ')}.` : introduction;
165
+ }