@bbigbang/core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (175) hide show
  1. package/dist/config.js +380 -0
  2. package/dist/execution/executionDispatcher.js +3810 -0
  3. package/dist/main.js +90 -0
  4. package/dist/nodeEventHistory.js +206 -0
  5. package/dist/scheduler/dreamLogic.js +50 -0
  6. package/dist/scheduler/dreamScheduler.js +65 -0
  7. package/dist/services/agentFileAccessService.js +1913 -0
  8. package/dist/services/agentRuntimeCleanupBroker.js +62 -0
  9. package/dist/services/agentSkillsBroker.js +118 -0
  10. package/dist/services/agentSkillsService.js +83 -0
  11. package/dist/services/agentWorkspaceBroker.js +937 -0
  12. package/dist/services/agentWorkspaceService.js +70 -0
  13. package/dist/services/appVersion.js +14 -0
  14. package/dist/services/auth.js +586 -0
  15. package/dist/services/claudeControlBroker.js +154 -0
  16. package/dist/services/claudeTranscriptBroker.js +100 -0
  17. package/dist/services/claudeTranscriptService.js +359 -0
  18. package/dist/services/codexAppServerBroker.js +155 -0
  19. package/dist/services/codexTranscriptBroker.js +98 -0
  20. package/dist/services/codexTranscriptService.js +961 -0
  21. package/dist/services/droidMissionBroker.js +124 -0
  22. package/dist/services/droidMissionImporter.js +630 -0
  23. package/dist/services/droidModelOptions.js +165 -0
  24. package/dist/services/hubServerRegistrationService.js +268 -0
  25. package/dist/services/libraryManifest.js +43 -0
  26. package/dist/services/libraryScaffold.js +26 -0
  27. package/dist/services/libraryService.js +2263 -0
  28. package/dist/services/memoryService.js +386 -0
  29. package/dist/services/missionEvidence.js +377 -0
  30. package/dist/services/missionService.js +2361 -0
  31. package/dist/services/missionTrace.js +158 -0
  32. package/dist/services/nativeMissionBriefParser.js +120 -0
  33. package/dist/services/nativeMissionOrchestrator.js +2045 -0
  34. package/dist/services/nativeMissionReportGenerator.js +227 -0
  35. package/dist/services/nativeMissionValidationRunner.js +452 -0
  36. package/dist/services/nativeMissionWorkerBroker.js +190 -0
  37. package/dist/services/nodeRegistry.js +34 -0
  38. package/dist/services/nodeStateReconciler.js +97 -0
  39. package/dist/services/panelMediaScanner.js +119 -0
  40. package/dist/services/persistentRuntimeJsonlClient.js +153 -0
  41. package/dist/services/platformAgentPolicy.js +180 -0
  42. package/dist/services/platformAgentService.js +2041 -0
  43. package/dist/services/projectAccessResolver.js +93 -0
  44. package/dist/services/projectService.js +392 -0
  45. package/dist/services/resourceSpaceService.js +140 -0
  46. package/dist/services/scenarioRuntimeService.js +1130 -0
  47. package/dist/services/suggestedPlannerService.js +868 -0
  48. package/dist/services/workbenchGitBroker.js +161 -0
  49. package/dist/services/workbenchGitService.js +69 -0
  50. package/dist/services/workbenchInspectBroker.js +65 -0
  51. package/dist/services/workbenchNodePathService.js +79 -0
  52. package/dist/services/workbenchRegistryService.js +240 -0
  53. package/dist/services/workbenchRootService.js +181 -0
  54. package/dist/services/workbenchTerminalBroker.js +378 -0
  55. package/dist/services/workspaceRunOwnership.js +60 -0
  56. package/dist/services/workspaceScaffold.js +105 -0
  57. package/dist/services/workspaceSessionRuntimeService.js +576 -0
  58. package/dist/services/workspaceSessionService.js +245 -0
  59. package/dist/services/workspaceToolActionRunner.js +1582 -0
  60. package/dist/services/workspaceToolErrors.js +10 -0
  61. package/dist/services/workspaceToolExecutionUtils.js +895 -0
  62. package/dist/services/workspaceToolLatestStateProjector.js +91 -0
  63. package/dist/services/workspaceToolManifest.js +572 -0
  64. package/dist/services/workspaceToolMutationQueue.js +43 -0
  65. package/dist/services/workspaceToolPanelProjection.js +460 -0
  66. package/dist/services/workspaceToolPromotion.js +255 -0
  67. package/dist/services/workspaceToolPromotionState.js +224 -0
  68. package/dist/services/workspaceToolPublishDiagnostics.js +189 -0
  69. package/dist/services/workspaceToolPublishIdentityResolver.js +146 -0
  70. package/dist/services/workspaceToolReadModel.js +378 -0
  71. package/dist/services/workspaceToolRunLedger.js +239 -0
  72. package/dist/services/workspaceToolService.js +3067 -0
  73. package/dist/services/workspaceToolSnapshotPanelSync.js +293 -0
  74. package/dist/services/workspaceToolTerminalLifecycle.js +283 -0
  75. package/dist/services/workspaceToolTypes.js +1 -0
  76. package/dist/services/workspaceToolUploadMaterializer.js +228 -0
  77. package/dist/web/actionCardRoutes.js +129 -0
  78. package/dist/web/actionCards.js +469 -0
  79. package/dist/web/activationContext.js +684 -0
  80. package/dist/web/agentChannelGuards.js +48 -0
  81. package/dist/web/agentMentionCooldowns.js +32 -0
  82. package/dist/web/agentReminders.js +1668 -0
  83. package/dist/web/agentRuntimePresence.js +197 -0
  84. package/dist/web/agentSelfState.js +494 -0
  85. package/dist/web/agentTaskLinks.js +26 -0
  86. package/dist/web/agentVisibility.js +79 -0
  87. package/dist/web/assets.js +95 -0
  88. package/dist/web/channelActivationPrompt.js +395 -0
  89. package/dist/web/channelMemoryNotes.js +127 -0
  90. package/dist/web/channelMentions.js +10 -0
  91. package/dist/web/channelMessageSequences.js +19 -0
  92. package/dist/web/channelSubscriptions.js +26 -0
  93. package/dist/web/clearedTaskRoots.js +10 -0
  94. package/dist/web/collaborationPromptGuidance.js +36 -0
  95. package/dist/web/collaborationSurfaceState.js +140 -0
  96. package/dist/web/contextBundleRanking.js +154 -0
  97. package/dist/web/contextBundleResolver.js +488 -0
  98. package/dist/web/conversationBuiltinSkillRoots.js +50 -0
  99. package/dist/web/conversationControls.js +232 -0
  100. package/dist/web/conversationHandoffs.js +612 -0
  101. package/dist/web/conversationManager.js +2511 -0
  102. package/dist/web/conversationSummaries.js +876 -0
  103. package/dist/web/conversationSurfaceKinds.js +17 -0
  104. package/dist/web/conversationTargets.js +173 -0
  105. package/dist/web/directActivationPrompt.js +122 -0
  106. package/dist/web/directReplyTargets.js +69 -0
  107. package/dist/web/directThreadResolver.js +129 -0
  108. package/dist/web/dmTaskHandoffPrompt.js +120 -0
  109. package/dist/web/dmTaskThreadStatusProjection.js +229 -0
  110. package/dist/web/ftsQuery.js +33 -0
  111. package/dist/web/internalAgentRouter.js +11341 -0
  112. package/dist/web/libraryCuratorScheduler.js +58 -0
  113. package/dist/web/libraryDocumentPromptGuidance.js +8 -0
  114. package/dist/web/messageCheckpoints.js +19 -0
  115. package/dist/web/nodeWsHandler.js +2495 -0
  116. package/dist/web/notificationRounds.js +1061 -0
  117. package/dist/web/panelActionMessages.js +108 -0
  118. package/dist/web/panelActivationPrompt.js +18 -0
  119. package/dist/web/panelAudit.js +273 -0
  120. package/dist/web/panelLifecycle.js +222 -0
  121. package/dist/web/panelMediaPolicy.js +43 -0
  122. package/dist/web/panelPathPolicy.js +63 -0
  123. package/dist/web/panelPreviews.js +175 -0
  124. package/dist/web/panelQueryHandles.js +2749 -0
  125. package/dist/web/panelRoutes.js +2147 -0
  126. package/dist/web/panels.js +904 -0
  127. package/dist/web/peerInboxAggregates.js +1247 -0
  128. package/dist/web/planApprovalState.js +92 -0
  129. package/dist/web/platformAgentScheduler.js +66 -0
  130. package/dist/web/proactiveOpportunities.js +452 -0
  131. package/dist/web/promptContextSections.js +242 -0
  132. package/dist/web/promptHistorySanitizer.js +26 -0
  133. package/dist/web/promptSlashCommands.js +158 -0
  134. package/dist/web/rollingConversationSummary.js +453 -0
  135. package/dist/web/routeHelpers.js +11 -0
  136. package/dist/web/routes/handoff.js +288 -0
  137. package/dist/web/routes/history.js +345 -0
  138. package/dist/web/routes/memory.js +258 -0
  139. package/dist/web/routes/selfState.js +171 -0
  140. package/dist/web/routes/workspace.js +154 -0
  141. package/dist/web/runSurfaceWatermarks.js +431 -0
  142. package/dist/web/runtimeCapabilities.js +48 -0
  143. package/dist/web/sameAgentHandoffs.js +494 -0
  144. package/dist/web/server.js +15567 -0
  145. package/dist/web/sharedCollaborationCapsules.js +163 -0
  146. package/dist/web/soloSessionRelay.js +42 -0
  147. package/dist/web/soloWsHandler.js +138 -0
  148. package/dist/web/suggestedPlannerScheduler.js +56 -0
  149. package/dist/web/surfaceActivationPolicy.js +108 -0
  150. package/dist/web/surfaceCollaborators.js +61 -0
  151. package/dist/web/surfaceSystemStatus.js +263 -0
  152. package/dist/web/targetParticipants.js +77 -0
  153. package/dist/web/taskEvents.js +49 -0
  154. package/dist/web/taskLifecycleMessages.js +165 -0
  155. package/dist/web/taskLoops.js +732 -0
  156. package/dist/web/taskMemoryNotes.js +224 -0
  157. package/dist/web/taskNumbers.js +16 -0
  158. package/dist/web/taskOwnerGuards.js +49 -0
  159. package/dist/web/taskParticipantResolver.js +42 -0
  160. package/dist/web/taskParticipants.js +97 -0
  161. package/dist/web/taskSourceDetails.js +20 -0
  162. package/dist/web/taskStateViews.js +210 -0
  163. package/dist/web/taskStatusTransitions.js +9 -0
  164. package/dist/web/taskThreadFollowups.js +599 -0
  165. package/dist/web/taskThreadRuntimeClosure.js +685 -0
  166. package/dist/web/taskUpdateDelivery.js +104 -0
  167. package/dist/web/threadReplyContentHeuristics.js +30 -0
  168. package/dist/web/threadRoots.js +61 -0
  169. package/dist/web/threadTaskBindings.js +365 -0
  170. package/dist/web/uiPanelPromptGuidance.js +27 -0
  171. package/dist/web/workspaceMemoryHints.js +143 -0
  172. package/dist/web/workspaceToolPromptGuidance.js +30 -0
  173. package/dist/web/wsHandler.js +397 -0
  174. package/dist/web/wsSink.js +116 -0
  175. package/package.json +54 -0
@@ -0,0 +1,2749 @@
1
+ import { createHash } from 'node:crypto';
2
+ import path from 'node:path';
3
+ import { validatePanelApiJsonlUrl, validatePanelApiJsonlUrlList } from '@bbigbang/protocol';
4
+ import { resolvePanelDatasetPath } from './panelPathPolicy.js';
5
+ import { aggregatePanelFieldRecords, getPanelFieldValue, } from './panels.js';
6
+ const QUERY_HANDLE_WORKSPACE_JSONL_PREFIX = 'workspace_jsonl:';
7
+ const QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX = 'workspace_jsonl_list:';
8
+ const QUERY_HANDLE_MAX_SOURCE_PATHS = 16;
9
+ const QUERY_HANDLE_READ_CHUNK_BYTES = 256 * 1024;
10
+ const QUERY_HANDLE_MAX_CHUNKS_PER_PAGE = 64;
11
+ const QUERY_HANDLE_MAX_CHUNKS_PER_SORT = 512;
12
+ const QUERY_HANDLE_INDEX_INTERVAL_ROWS = 200;
13
+ const API_JSONL_MAX_BYTES = 2 * 1024 * 1024;
14
+ const API_JSONL_MAX_ROWS = 10_000;
15
+ const API_JSONL_RANGE_CHUNK_BYTES = 256 * 1024;
16
+ const API_JSONL_MAX_RANGE_CHUNKS_PER_PAGE = 64;
17
+ const API_JSONL_MAX_RANGE_CHUNKS_PER_INDEX = 512;
18
+ /** TTL for panel_api_jsonl_response_cache entries (7 days). */
19
+ export const PANEL_API_JSONL_RESPONSE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
20
+ /** Per-panel size budget for panel_api_jsonl_response_cache (64 MiB). */
21
+ export const PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL = 64 * 1024 * 1024;
22
+ export class PanelQueryHandleError extends Error {
23
+ statusCode;
24
+ constructor(message, statusCode = 400) {
25
+ super(message);
26
+ this.name = 'PanelQueryHandleError';
27
+ this.statusCode = statusCode;
28
+ }
29
+ }
30
+ export function isPanelQueryHandleSource(panel) {
31
+ return panel.datasetSource?.kind === 'query_handle' || panel.datasetSource?.kind === 'api_jsonl';
32
+ }
33
+ export function parsePanelQueryHandle(handle) {
34
+ const trimmed = handle.trim();
35
+ if (trimmed.startsWith(QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX)) {
36
+ const rawList = trimmed.slice(QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX.length).trim();
37
+ let parsed;
38
+ try {
39
+ parsed = JSON.parse(rawList);
40
+ }
41
+ catch {
42
+ throw new PanelQueryHandleError('query_handle workspace_jsonl_list must be a JSON array of paths.', 400);
43
+ }
44
+ if (!Array.isArray(parsed) || parsed.length < 1 || parsed.length > QUERY_HANDLE_MAX_SOURCE_PATHS) {
45
+ throw new PanelQueryHandleError(`query_handle workspace_jsonl_list must contain 1-${QUERY_HANDLE_MAX_SOURCE_PATHS} paths.`, 400);
46
+ }
47
+ return {
48
+ paths: parsed.map((entry, index) => normalizeQueryHandleWorkspaceJsonlPath(entry, `query_handle workspace_jsonl_list path ${index + 1}`)),
49
+ };
50
+ }
51
+ if (!trimmed.startsWith(QUERY_HANDLE_WORKSPACE_JSONL_PREFIX)) {
52
+ throw new PanelQueryHandleError('Unsupported query_handle. Use workspace_jsonl:<path> or workspace_jsonl_list:["<path>",...] for query_handle panels.', 400);
53
+ }
54
+ return {
55
+ paths: [normalizeQueryHandleWorkspaceJsonlPath(trimmed.slice(QUERY_HANDLE_WORKSPACE_JSONL_PREFIX.length), 'query_handle workspace_jsonl path')],
56
+ };
57
+ }
58
+ function normalizeQueryHandleWorkspaceJsonlPath(rawPath, label) {
59
+ if (typeof rawPath !== 'string') {
60
+ throw new PanelQueryHandleError(`${label} must be a string path.`, 400);
61
+ }
62
+ const sourcePath = rawPath.trim();
63
+ if (!sourcePath
64
+ || sourcePath.includes('\0')
65
+ || /[\r\n]/u.test(sourcePath)) {
66
+ throw new PanelQueryHandleError(`${label} must be a non-empty path without control characters.`, 400);
67
+ }
68
+ if (!path.isAbsolute(sourcePath) && sourcePath.split('/').includes('..')) {
69
+ throw new PanelQueryHandleError(`${label} must not contain relative traversal.`, 400);
70
+ }
71
+ return sourcePath;
72
+ }
73
+ export function parsePanelApiJsonlUrl(url, options = {}) {
74
+ const validation = validatePanelApiJsonlUrl(url, options);
75
+ if (!validation.ok) {
76
+ throw new PanelQueryHandleError(validation.reason, 400);
77
+ }
78
+ return { url: validation.url };
79
+ }
80
+ export function parsePanelApiJsonlUrls(source, options = {}) {
81
+ const validation = validatePanelApiJsonlUrlList(source, options);
82
+ if (!validation.ok) {
83
+ throw new PanelQueryHandleError(validation.reason, 400);
84
+ }
85
+ return { urls: validation.urls, authProfile: validation.authProfile };
86
+ }
87
+ export async function listPanelQueryHandleRows(params) {
88
+ const source = params.panel.datasetSource;
89
+ if (source?.kind !== 'query_handle' && source?.kind !== 'api_jsonl') {
90
+ throw new PanelQueryHandleError('Panel does not use a dynamic dataset source.', 400);
91
+ }
92
+ const agent = params.conversationManager.getAgent(params.panel.agentId);
93
+ if (!agent?.nodeId) {
94
+ throw new PanelQueryHandleError('dynamic panel agent node is unavailable.', 409);
95
+ }
96
+ if (!params.workspaceBroker) {
97
+ throw new PanelQueryHandleError('Workspace broker unavailable.', 503);
98
+ }
99
+ if (source.kind === 'api_jsonl') {
100
+ const parsed = parsePanelApiJsonlUrls(source, {
101
+ allowedOrigins: params.apiJsonlAllowedOrigins,
102
+ allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
103
+ allowedAuthProfiles: params.apiJsonlAuthProfiles,
104
+ });
105
+ return listPanelApiJsonlRows({
106
+ db: params.db,
107
+ panel: params.panel,
108
+ conversationManager: params.conversationManager,
109
+ config: params.config,
110
+ apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
111
+ apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
112
+ apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
113
+ sourceUrls: parsed.urls,
114
+ authProfile: parsed.authProfile,
115
+ workspaceBroker: params.workspaceBroker,
116
+ nodeId: agent.nodeId,
117
+ datasetSchema: params.datasetSchema,
118
+ options: params.options,
119
+ limit: Math.min(Math.max(1, params.options.limit), 200),
120
+ });
121
+ }
122
+ if (!agent.workspacePath) {
123
+ throw new PanelQueryHandleError('query_handle panel agent workspace is unavailable.', 409);
124
+ }
125
+ const dataSources = resolveQueryHandleDataSources({
126
+ handle: source.handle,
127
+ workspaceRoot: agent.workspacePath,
128
+ config: params.config,
129
+ });
130
+ const limit = Math.min(Math.max(1, params.options.limit), 200);
131
+ if (params.options.sortField && params.options.sortDirection) {
132
+ return listSortedPanelQueryHandleRows({
133
+ db: params.db,
134
+ panel: params.panel,
135
+ conversationManager: params.conversationManager,
136
+ config: params.config,
137
+ indexContext: createQueryHandleIndexContext(params.db, params.panel.id, dataSources),
138
+ workspaceBroker: params.workspaceBroker,
139
+ nodeId: agent.nodeId,
140
+ workspaceRoot: agent.workspacePath,
141
+ dataSources,
142
+ datasetSchema: params.datasetSchema,
143
+ options: params.options,
144
+ limit,
145
+ });
146
+ }
147
+ return listUnsortedPanelQueryHandleRows({
148
+ db: params.db,
149
+ panel: params.panel,
150
+ conversationManager: params.conversationManager,
151
+ config: params.config,
152
+ indexContext: createQueryHandleIndexContext(params.db, params.panel.id, dataSources),
153
+ workspaceBroker: params.workspaceBroker,
154
+ nodeId: agent.nodeId,
155
+ workspaceRoot: agent.workspacePath,
156
+ dataSources,
157
+ datasetSchema: params.datasetSchema,
158
+ options: params.options,
159
+ limit,
160
+ });
161
+ }
162
+ export async function aggregatePanelQueryHandleRows(params) {
163
+ const source = params.panel.datasetSource;
164
+ if (source?.kind !== 'query_handle' && source?.kind !== 'api_jsonl') {
165
+ throw new PanelQueryHandleError('Panel does not use a dynamic dataset source.', 400);
166
+ }
167
+ const agent = params.conversationManager.getAgent(params.panel.agentId);
168
+ if (!agent?.nodeId) {
169
+ throw new PanelQueryHandleError('dynamic panel agent node is unavailable.', 409);
170
+ }
171
+ if (!params.workspaceBroker) {
172
+ throw new PanelQueryHandleError('Workspace broker unavailable.', 503);
173
+ }
174
+ if (source.kind === 'api_jsonl') {
175
+ const parsed = parsePanelApiJsonlUrls(source, {
176
+ allowedOrigins: params.apiJsonlAllowedOrigins,
177
+ allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
178
+ allowedAuthProfiles: params.apiJsonlAuthProfiles,
179
+ });
180
+ const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, parsed.urls, parsed.authProfile);
181
+ const sourceManifestKey = indexContext?.sourceManifestKey ?? apiJsonlSourceManifestKey(parsed.urls, parsed.authProfile);
182
+ const extraFieldNames = aggregateFieldIndexNames(params.options);
183
+ const hasBoundedResponseCache = hasApiJsonlResponseCacheForAllSources(params.db, {
184
+ panelId: params.panel.id,
185
+ sourceManifestKey,
186
+ sourceUrls: parsed.urls,
187
+ });
188
+ if (!hasBoundedResponseCache) {
189
+ const rangeMetadataKey = await ensureApiJsonlRangeFieldIndex({
190
+ db: params.db,
191
+ panel: params.panel,
192
+ sourceUrls: parsed.urls,
193
+ authProfile: parsed.authProfile,
194
+ workspaceBroker: params.workspaceBroker,
195
+ nodeId: agent.nodeId,
196
+ datasetSchema: params.datasetSchema,
197
+ indexContext,
198
+ extraFieldNames,
199
+ });
200
+ if (indexContext && rangeMetadataKey) {
201
+ const indexed = readAggregateFromQueryHandleFieldIndex({
202
+ db: params.db,
203
+ panel: params.panel,
204
+ sourceManifestKey: indexContext.sourceManifestKey,
205
+ metadataKey: rangeMetadataKey,
206
+ datasetSchema: params.datasetSchema,
207
+ options: params.options,
208
+ extraFieldNames,
209
+ sourceKind: 'api_jsonl_field_index',
210
+ });
211
+ if (indexed)
212
+ return indexed;
213
+ }
214
+ }
215
+ const sources = await fetchApiJsonlSources({
216
+ db: params.db,
217
+ panelId: params.panel.id,
218
+ sourceUrls: parsed.urls,
219
+ authProfile: parsed.authProfile,
220
+ sourceManifestKey,
221
+ workspaceBroker: params.workspaceBroker,
222
+ nodeId: agent.nodeId,
223
+ });
224
+ const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
225
+ const metadataKey = apiJsonlMetadataKey(sources);
226
+ if (indexContext) {
227
+ writeQueryHandleFieldIndexRows({
228
+ context: indexContext,
229
+ metadataKey,
230
+ datasetSchema: params.datasetSchema,
231
+ rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
232
+ extraFieldNames,
233
+ });
234
+ const indexed = readAggregateFromQueryHandleFieldIndex({
235
+ db: params.db,
236
+ panel: params.panel,
237
+ sourceManifestKey: indexContext.sourceManifestKey,
238
+ metadataKey,
239
+ datasetSchema: params.datasetSchema,
240
+ options: params.options,
241
+ extraFieldNames,
242
+ sourceKind: 'api_jsonl_field_index',
243
+ });
244
+ if (indexed)
245
+ return indexed;
246
+ }
247
+ return aggregatePanelFieldRecords(rows
248
+ .filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
249
+ .map((row) => row.fields), {
250
+ op: params.options.op,
251
+ field: params.options.field,
252
+ });
253
+ }
254
+ if (!agent.workspacePath) {
255
+ throw new PanelQueryHandleError('query_handle panel agent workspace is unavailable.', 409);
256
+ }
257
+ const dataSources = resolveQueryHandleDataSources({
258
+ handle: source.handle,
259
+ workspaceRoot: agent.workspacePath,
260
+ config: params.config,
261
+ });
262
+ const indexContext = createQueryHandleIndexContext(params.db, params.panel.id, dataSources);
263
+ const extraFieldNames = aggregateFieldIndexNames(params.options);
264
+ const metadataKey = indexContext
265
+ ? await loadQueryHandleIndexMetadataKey({
266
+ context: indexContext,
267
+ workspaceBroker: params.workspaceBroker,
268
+ nodeId: agent.nodeId,
269
+ workspaceRoot: agent.workspacePath,
270
+ dataSources,
271
+ })
272
+ : null;
273
+ if (indexContext && metadataKey) {
274
+ const indexed = readAggregateFromQueryHandleFieldIndex({
275
+ db: params.db,
276
+ panel: params.panel,
277
+ sourceManifestKey: indexContext.sourceManifestKey,
278
+ metadataKey,
279
+ datasetSchema: params.datasetSchema,
280
+ options: params.options,
281
+ extraFieldNames,
282
+ sourceKind: 'query_handle_field_index',
283
+ });
284
+ if (indexed)
285
+ return indexed;
286
+ }
287
+ const rows = await scanQueryHandleRowsForFieldIndex({
288
+ db: params.db,
289
+ panel: params.panel,
290
+ workspaceBroker: params.workspaceBroker,
291
+ nodeId: agent.nodeId,
292
+ workspaceRoot: agent.workspacePath,
293
+ dataSources,
294
+ datasetSchema: params.datasetSchema,
295
+ indexContext,
296
+ metadataKey,
297
+ extraFieldNames,
298
+ });
299
+ if (indexContext && metadataKey) {
300
+ const indexed = readAggregateFromQueryHandleFieldIndex({
301
+ db: params.db,
302
+ panel: params.panel,
303
+ sourceManifestKey: indexContext.sourceManifestKey,
304
+ metadataKey,
305
+ datasetSchema: params.datasetSchema,
306
+ options: params.options,
307
+ extraFieldNames,
308
+ sourceKind: 'query_handle_field_index',
309
+ });
310
+ if (indexed)
311
+ return indexed;
312
+ }
313
+ return aggregatePanelFieldRecords(rows
314
+ .filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
315
+ .map((row) => row.fields), {
316
+ op: params.options.op,
317
+ field: params.options.field,
318
+ });
319
+ }
320
+ async function listUnsortedPanelQueryHandleRows(params) {
321
+ if (params.indexContext && params.options.filterField && params.options.filterValue !== undefined) {
322
+ const metadataKey = await loadQueryHandleIndexMetadataKey({
323
+ context: params.indexContext,
324
+ workspaceBroker: params.workspaceBroker,
325
+ nodeId: params.nodeId,
326
+ workspaceRoot: params.workspaceRoot,
327
+ dataSources: params.dataSources,
328
+ });
329
+ if (metadataKey) {
330
+ const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
331
+ sourceKind: 'query_handle_field_index',
332
+ filterField: params.options.filterField,
333
+ filterValue: params.options.filterValue,
334
+ });
335
+ const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
336
+ db: params.db,
337
+ panel: params.panel,
338
+ conversationManager: params.conversationManager,
339
+ workspaceBroker: params.workspaceBroker,
340
+ config: params.config,
341
+ datasetSchema: params.datasetSchema,
342
+ sourceManifestKey: params.indexContext.sourceManifestKey,
343
+ metadataKey,
344
+ options: params.options,
345
+ position: cursor.position,
346
+ limit: params.limit,
347
+ });
348
+ if (indexed)
349
+ return indexed;
350
+ }
351
+ }
352
+ const cursor = parseQueryHandleCursor(params.options.cursor);
353
+ let sourceIndex = Math.min(Math.max(0, cursor.sourceIndex ?? 0), params.dataSources.length - 1);
354
+ if (!Number.isFinite(sourceIndex))
355
+ sourceIndex = 0;
356
+ const rows = [];
357
+ let offset = cursor.offset;
358
+ let rowIndex = cursor.rowIndex;
359
+ let chunks = 0;
360
+ const sourceSawRows = new Set();
361
+ if (offset > 0)
362
+ sourceSawRows.add(sourceIndex);
363
+ while (rows.length < params.limit && sourceIndex < params.dataSources.length) {
364
+ chunks += 1;
365
+ if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_PAGE) {
366
+ throw new PanelQueryHandleError('query_handle page scan exceeded the per-page work limit.', 413);
367
+ }
368
+ const chunk = await readQueryHandleChunk({
369
+ workspaceBroker: params.workspaceBroker,
370
+ nodeId: params.nodeId,
371
+ workspaceRoot: params.workspaceRoot,
372
+ dataSource: params.dataSources[sourceIndex],
373
+ offset,
374
+ limit: QUERY_HANDLE_READ_CHUNK_BYTES,
375
+ });
376
+ const content = chunk.content ?? '';
377
+ const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
378
+ if (parsed.completeText.length === 0 && chunk.hasMore) {
379
+ throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
380
+ }
381
+ const baseOffset = chunk.offset ?? offset;
382
+ let consumedBytes = 0;
383
+ for (const line of parsed.lines) {
384
+ const lineStartOffset = baseOffset + consumedBytes;
385
+ consumedBytes += line.byteLength;
386
+ const trimmed = line.text.trim();
387
+ if (!trimmed)
388
+ continue;
389
+ const rawRow = parseJsonlRow(trimmed, rowIndex);
390
+ const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
391
+ const isSourceStart = !sourceSawRows.has(sourceIndex);
392
+ sourceSawRows.add(sourceIndex);
393
+ recordQueryHandleCheckpoint(params.indexContext, {
394
+ sourceIndex,
395
+ chunk,
396
+ rowIndex,
397
+ byteOffset: lineStartOffset,
398
+ isSourceStart,
399
+ });
400
+ rowIndex += 1;
401
+ if (!matchesFilter(row.fields, params.options.filterField, params.options.filterValue)) {
402
+ continue;
403
+ }
404
+ rows.push(row);
405
+ if (rows.length >= params.limit)
406
+ break;
407
+ }
408
+ offset = baseOffset + consumedBytes;
409
+ if (offset >= chunk.size) {
410
+ sourceIndex += 1;
411
+ offset = 0;
412
+ }
413
+ }
414
+ return {
415
+ rows,
416
+ nextCursor: sourceIndex < params.dataSources.length
417
+ ? encodeQueryHandleCursor({ sourceIndex, offset, rowIndex })
418
+ : null,
419
+ };
420
+ }
421
+ async function listSortedPanelQueryHandleRows(params) {
422
+ const sortField = params.options.sortField;
423
+ const sortDirection = params.options.sortDirection;
424
+ if (!sortField || !sortDirection) {
425
+ throw new PanelQueryHandleError('query_handle sort requires both sort field and direction.', 400);
426
+ }
427
+ const queryKey = queryHandleViewIndexQueryKey(params.options);
428
+ const cursor = parseQueryHandleSortCursor(params.options.cursor, {
429
+ sortField,
430
+ sortDirection,
431
+ filterField: params.options.filterField,
432
+ filterValue: params.options.filterValue,
433
+ });
434
+ const metadataKey = params.indexContext
435
+ ? await loadQueryHandleIndexMetadataKey({
436
+ context: params.indexContext,
437
+ workspaceBroker: params.workspaceBroker,
438
+ nodeId: params.nodeId,
439
+ workspaceRoot: params.workspaceRoot,
440
+ dataSources: params.dataSources,
441
+ })
442
+ : null;
443
+ if (metadataKey) {
444
+ const cached = await readCachedQueryHandleViewIndexRows({
445
+ db: params.db,
446
+ panel: params.panel,
447
+ conversationManager: params.conversationManager,
448
+ workspaceBroker: params.workspaceBroker,
449
+ config: params.config,
450
+ datasetSchema: params.datasetSchema,
451
+ sourceManifestKey: params.indexContext.sourceManifestKey,
452
+ queryKey,
453
+ metadataKey,
454
+ position: cursor.position,
455
+ limit: params.limit,
456
+ });
457
+ if (cached)
458
+ return cached;
459
+ const indexed = await readQueryHandleRowsFromFieldIndex({
460
+ db: params.db,
461
+ panel: params.panel,
462
+ conversationManager: params.conversationManager,
463
+ workspaceBroker: params.workspaceBroker,
464
+ config: params.config,
465
+ datasetSchema: params.datasetSchema,
466
+ sourceManifestKey: params.indexContext.sourceManifestKey,
467
+ metadataKey,
468
+ options: params.options,
469
+ position: cursor.position,
470
+ limit: params.limit,
471
+ });
472
+ if (indexed)
473
+ return indexed;
474
+ }
475
+ const rows = [];
476
+ const fieldIndexRows = [];
477
+ let rowIndex = 0;
478
+ let chunks = 0;
479
+ const sourceSawRows = new Set();
480
+ for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
481
+ const dataSource = params.dataSources[sourceIndex];
482
+ let offset = 0;
483
+ let hasMore = true;
484
+ while (hasMore) {
485
+ chunks += 1;
486
+ if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_SORT) {
487
+ throw new PanelQueryHandleError('query_handle sorted scan exceeded the per-request work limit.', 413);
488
+ }
489
+ const chunk = await readQueryHandleChunk({
490
+ workspaceBroker: params.workspaceBroker,
491
+ nodeId: params.nodeId,
492
+ workspaceRoot: params.workspaceRoot,
493
+ dataSource,
494
+ offset,
495
+ limit: QUERY_HANDLE_READ_CHUNK_BYTES,
496
+ });
497
+ const content = chunk.content ?? '';
498
+ const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
499
+ if (parsed.completeText.length === 0 && chunk.hasMore) {
500
+ throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
501
+ }
502
+ const baseOffset = chunk.offset ?? offset;
503
+ let consumedBytes = 0;
504
+ for (const line of parsed.lines) {
505
+ const lineStartOffset = baseOffset + consumedBytes;
506
+ consumedBytes += line.byteLength;
507
+ const trimmed = line.text.trim();
508
+ if (!trimmed)
509
+ continue;
510
+ const rawRow = parseJsonlRow(trimmed, rowIndex);
511
+ const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
512
+ const isSourceStart = !sourceSawRows.has(sourceIndex);
513
+ sourceSawRows.add(sourceIndex);
514
+ recordQueryHandleCheckpoint(params.indexContext, {
515
+ sourceIndex,
516
+ chunk,
517
+ rowIndex,
518
+ byteOffset: lineStartOffset,
519
+ isSourceStart,
520
+ });
521
+ rowIndex += 1;
522
+ fieldIndexRows.push({ rowIndex: row.rowIndex, fields: row.fields });
523
+ if (!matchesFilter(row.fields, params.options.filterField, params.options.filterValue)) {
524
+ continue;
525
+ }
526
+ rows.push(row);
527
+ }
528
+ offset = baseOffset + consumedBytes;
529
+ hasMore = offset < chunk.size;
530
+ if (parsed.lines.length === 0 && !hasMore)
531
+ break;
532
+ }
533
+ }
534
+ rows.sort((left, right) => comparePanelRowsBySortField(left, right, sortField, sortDirection, params.options.sortFieldType));
535
+ if (metadataKey && params.indexContext) {
536
+ writeQueryHandleViewIndexRows({
537
+ context: params.indexContext,
538
+ queryKey,
539
+ metadataKey,
540
+ rowIndices: rows.map((row) => row.rowIndex),
541
+ });
542
+ writeQueryHandleFieldIndexRows({
543
+ context: params.indexContext,
544
+ metadataKey,
545
+ datasetSchema: params.datasetSchema,
546
+ rows: fieldIndexRows,
547
+ });
548
+ }
549
+ const start = Math.min(cursor.position, rows.length);
550
+ const resultRows = rows.slice(start, start + params.limit);
551
+ const nextPosition = start + resultRows.length;
552
+ return {
553
+ rows: resultRows,
554
+ nextCursor: nextPosition < rows.length
555
+ ? encodeQueryHandleSortCursor({
556
+ position: nextPosition,
557
+ sortField,
558
+ sortDirection,
559
+ filterField: params.options.filterField,
560
+ filterValue: params.options.filterValue,
561
+ })
562
+ : null,
563
+ };
564
+ }
565
+ async function listPanelApiJsonlRows(params) {
566
+ const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, params.sourceUrls, params.authProfile);
567
+ const selectRowsByIndicesFromDynamicSource = async (rowIndices) => (await listPanelQueryHandleRowsByIndices({
568
+ db: params.db,
569
+ panel: params.panel,
570
+ conversationManager: params.conversationManager,
571
+ workspaceBroker: params.workspaceBroker,
572
+ config: params.config,
573
+ apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
574
+ apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
575
+ apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
576
+ datasetSchema: params.datasetSchema,
577
+ rowIndices,
578
+ })).rows;
579
+ if (!params.options.sortField && !params.options.filterField && !isPositionCursor(params.options.cursor)) {
580
+ const ranged = await listPanelApiJsonlRowsByRange({
581
+ ...params,
582
+ indexContext,
583
+ });
584
+ if (ranged)
585
+ return ranged;
586
+ }
587
+ const sourceManifestKey = indexContext?.sourceManifestKey ?? apiJsonlSourceManifestKey(params.sourceUrls, params.authProfile);
588
+ const hasBoundedResponseCache = hasApiJsonlResponseCacheForAllSources(params.db, {
589
+ panelId: params.panel.id,
590
+ sourceManifestKey,
591
+ sourceUrls: params.sourceUrls,
592
+ });
593
+ if (!hasBoundedResponseCache && ((params.options.sortField && params.options.sortDirection) || (params.options.filterField && params.options.filterValue !== undefined))) {
594
+ const rangeMetadataKey = await ensureApiJsonlRangeFieldIndex({
595
+ db: params.db,
596
+ panel: params.panel,
597
+ sourceUrls: params.sourceUrls,
598
+ authProfile: params.authProfile,
599
+ workspaceBroker: params.workspaceBroker,
600
+ nodeId: params.nodeId,
601
+ datasetSchema: params.datasetSchema,
602
+ indexContext,
603
+ extraFieldNames: [],
604
+ });
605
+ if (indexContext && rangeMetadataKey && params.options.sortField && params.options.sortDirection) {
606
+ const cursor = parseQueryHandleSortCursor(params.options.cursor, {
607
+ sortField: params.options.sortField,
608
+ sortDirection: params.options.sortDirection,
609
+ filterField: params.options.filterField,
610
+ filterValue: params.options.filterValue,
611
+ });
612
+ const indexed = await readQueryHandleRowsFromFieldIndex({
613
+ db: params.db,
614
+ panel: params.panel,
615
+ conversationManager: params.conversationManager,
616
+ workspaceBroker: params.workspaceBroker,
617
+ config: params.config,
618
+ datasetSchema: params.datasetSchema,
619
+ sourceManifestKey: indexContext.sourceManifestKey,
620
+ metadataKey: rangeMetadataKey,
621
+ options: params.options,
622
+ position: cursor.position,
623
+ limit: params.limit,
624
+ selectRowsByIndices: selectRowsByIndicesFromDynamicSource,
625
+ });
626
+ if (indexed)
627
+ return indexed;
628
+ }
629
+ if (indexContext && rangeMetadataKey && params.options.filterField && params.options.filterValue !== undefined) {
630
+ const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
631
+ sourceKind: 'api_jsonl_field_index',
632
+ filterField: params.options.filterField,
633
+ filterValue: params.options.filterValue,
634
+ });
635
+ const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
636
+ db: params.db,
637
+ panel: params.panel,
638
+ conversationManager: params.conversationManager,
639
+ workspaceBroker: params.workspaceBroker,
640
+ config: params.config,
641
+ datasetSchema: params.datasetSchema,
642
+ sourceManifestKey: indexContext.sourceManifestKey,
643
+ metadataKey: rangeMetadataKey,
644
+ options: params.options,
645
+ position: cursor.position,
646
+ limit: params.limit,
647
+ sourceKind: 'api_jsonl_field_index',
648
+ selectRowsByIndices: selectRowsByIndicesFromDynamicSource,
649
+ });
650
+ if (indexed)
651
+ return indexed;
652
+ }
653
+ }
654
+ const sources = await fetchApiJsonlSources({
655
+ db: params.db,
656
+ panelId: params.panel.id,
657
+ sourceUrls: params.sourceUrls,
658
+ authProfile: params.authProfile,
659
+ sourceManifestKey,
660
+ workspaceBroker: params.workspaceBroker,
661
+ nodeId: params.nodeId,
662
+ });
663
+ const metadataKey = apiJsonlMetadataKey(sources);
664
+ const selectRowsByIndices = async (rowIndices) => selectApiJsonlRowsByIndices(sources, params.datasetSchema, rowIndices);
665
+ if (params.options.sortField && params.options.sortDirection) {
666
+ const queryKey = queryHandleViewIndexQueryKey(params.options);
667
+ const cursor = parseQueryHandleSortCursor(params.options.cursor, {
668
+ sortField: params.options.sortField,
669
+ sortDirection: params.options.sortDirection,
670
+ filterField: params.options.filterField,
671
+ filterValue: params.options.filterValue,
672
+ });
673
+ if (indexContext) {
674
+ const cached = await readCachedQueryHandleViewIndexRows({
675
+ db: params.db,
676
+ panel: params.panel,
677
+ conversationManager: params.conversationManager,
678
+ workspaceBroker: params.workspaceBroker,
679
+ config: params.config,
680
+ datasetSchema: params.datasetSchema,
681
+ sourceManifestKey: indexContext.sourceManifestKey,
682
+ queryKey,
683
+ metadataKey,
684
+ position: cursor.position,
685
+ limit: params.limit,
686
+ selectRowsByIndices,
687
+ });
688
+ if (cached)
689
+ return cached;
690
+ const indexed = await readQueryHandleRowsFromFieldIndex({
691
+ db: params.db,
692
+ panel: params.panel,
693
+ conversationManager: params.conversationManager,
694
+ workspaceBroker: params.workspaceBroker,
695
+ config: params.config,
696
+ datasetSchema: params.datasetSchema,
697
+ sourceManifestKey: indexContext.sourceManifestKey,
698
+ metadataKey,
699
+ options: params.options,
700
+ position: cursor.position,
701
+ limit: params.limit,
702
+ selectRowsByIndices,
703
+ });
704
+ if (indexed)
705
+ return indexed;
706
+ }
707
+ const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
708
+ const filteredRows = params.options.filterField && params.options.filterValue !== undefined
709
+ ? rows.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
710
+ : rows;
711
+ filteredRows.sort((left, right) => comparePanelRowsBySortField(left, right, params.options.sortField, params.options.sortDirection, params.options.sortFieldType));
712
+ if (indexContext) {
713
+ writeQueryHandleViewIndexRows({
714
+ context: indexContext,
715
+ queryKey,
716
+ metadataKey,
717
+ rowIndices: filteredRows.map((row) => row.rowIndex),
718
+ });
719
+ writeQueryHandleFieldIndexRows({
720
+ context: indexContext,
721
+ metadataKey,
722
+ datasetSchema: params.datasetSchema,
723
+ rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
724
+ });
725
+ }
726
+ return sliceRowsByPositionCursor(filteredRows, cursor.position, params.limit, (position) => encodeQueryHandleSortCursor({
727
+ position,
728
+ sortField: params.options.sortField,
729
+ sortDirection: params.options.sortDirection,
730
+ filterField: params.options.filterField,
731
+ filterValue: params.options.filterValue,
732
+ }));
733
+ }
734
+ if (params.options.filterField && params.options.filterValue !== undefined) {
735
+ const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
736
+ sourceKind: 'api_jsonl_field_index',
737
+ filterField: params.options.filterField,
738
+ filterValue: params.options.filterValue,
739
+ });
740
+ if (indexContext) {
741
+ const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
742
+ db: params.db,
743
+ panel: params.panel,
744
+ conversationManager: params.conversationManager,
745
+ workspaceBroker: params.workspaceBroker,
746
+ config: params.config,
747
+ datasetSchema: params.datasetSchema,
748
+ sourceManifestKey: indexContext.sourceManifestKey,
749
+ metadataKey,
750
+ options: params.options,
751
+ position: cursor.position,
752
+ limit: params.limit,
753
+ sourceKind: 'api_jsonl_field_index',
754
+ selectRowsByIndices,
755
+ });
756
+ if (indexed)
757
+ return indexed;
758
+ }
759
+ const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
760
+ if (indexContext) {
761
+ writeQueryHandleFieldIndexRows({
762
+ context: indexContext,
763
+ metadataKey,
764
+ datasetSchema: params.datasetSchema,
765
+ rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
766
+ });
767
+ }
768
+ const filteredRows = rows.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue));
769
+ return sliceRowsByPositionCursor(filteredRows, cursor.position, params.limit, (position) => encodeQueryHandlePositionCursor({
770
+ position,
771
+ sourceKind: 'api_jsonl_field_index',
772
+ filterField: params.options.filterField,
773
+ filterValue: params.options.filterValue,
774
+ }));
775
+ }
776
+ const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
777
+ const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
778
+ sourceKind: 'api_jsonl',
779
+ filterField: params.options.filterField,
780
+ filterValue: params.options.filterValue,
781
+ });
782
+ return sliceRowsByPositionCursor(rows, cursor.position, params.limit, (position) => encodeQueryHandlePositionCursor({
783
+ position,
784
+ sourceKind: 'api_jsonl',
785
+ filterField: params.options.filterField,
786
+ filterValue: params.options.filterValue,
787
+ }));
788
+ }
789
+ async function listPanelApiJsonlRowsByRange(params) {
790
+ const cursor = parseApiJsonlRangeCursor(params.options.cursor);
791
+ let sourceIndex = Math.min(Math.max(0, cursor.sourceIndex), params.sourceUrls.length - 1);
792
+ if (!Number.isFinite(sourceIndex))
793
+ sourceIndex = 0;
794
+ let offset = cursor.offset;
795
+ let rowIndex = cursor.rowIndex;
796
+ const rows = [];
797
+ let chunks = 0;
798
+ const sourceSawRows = new Set();
799
+ if (offset > 0)
800
+ sourceSawRows.add(sourceIndex);
801
+ let currentValidator = cursor.url === params.sourceUrls[sourceIndex]
802
+ ? {
803
+ url: cursor.url,
804
+ etag: cursor.etag ?? null,
805
+ lastModified: cursor.lastModified ?? null,
806
+ totalSize: cursor.totalSize ?? null,
807
+ }
808
+ : null;
809
+ while (rows.length < params.limit && sourceIndex < params.sourceUrls.length) {
810
+ chunks += 1;
811
+ if (chunks > API_JSONL_MAX_RANGE_CHUNKS_PER_PAGE) {
812
+ throw new PanelQueryHandleError('api_jsonl ranged page scan exceeded the per-page work limit.', 413);
813
+ }
814
+ const url = params.sourceUrls[sourceIndex];
815
+ const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
816
+ maxBytes: API_JSONL_MAX_BYTES,
817
+ ...(params.authProfile ? { authProfile: params.authProfile } : {}),
818
+ rangeStart: offset,
819
+ rangeLength: API_JSONL_RANGE_CHUNK_BYTES,
820
+ ...(currentValidator?.url === url ? withApiJsonlIfRange(currentValidator) : {}),
821
+ });
822
+ if (result.status !== 206
823
+ || result.partialContent !== true
824
+ || typeof result.rangeStart !== 'number'
825
+ || typeof result.rangeEnd !== 'number'
826
+ || result.rangeStart !== offset) {
827
+ return null;
828
+ }
829
+ if (Buffer.byteLength(result.content, 'utf8') > API_JSONL_MAX_BYTES) {
830
+ throw new PanelQueryHandleError(`api_jsonl response exceeds byte limit (${API_JSONL_MAX_BYTES}).`, 413);
831
+ }
832
+ currentValidator = {
833
+ url,
834
+ etag: result.etag ?? null,
835
+ lastModified: result.lastModified ?? null,
836
+ totalSize: result.totalSize ?? null,
837
+ };
838
+ const checkpointChunk = apiJsonlRangeCheckpointChunk(result);
839
+ const parsed = parseCompleteJsonlLines(result.content, Boolean(result.hasMore));
840
+ if (parsed.completeText.length === 0 && result.hasMore) {
841
+ throw new PanelQueryHandleError('api_jsonl JSONL row exceeds the per-row size limit.', 413);
842
+ }
843
+ const baseOffset = result.rangeStart;
844
+ let consumedBytes = 0;
845
+ let stoppedForLimit = false;
846
+ let lastConsumedLineIndex = -1;
847
+ for (const [lineIndex, line] of parsed.lines.entries()) {
848
+ const lineStartOffset = baseOffset + consumedBytes;
849
+ consumedBytes += line.byteLength;
850
+ lastConsumedLineIndex = lineIndex;
851
+ const trimmed = line.text.trim();
852
+ if (!trimmed)
853
+ continue;
854
+ if (rowIndex >= API_JSONL_MAX_ROWS) {
855
+ throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
856
+ }
857
+ const rawRow = parseJsonlRow(trimmed, rowIndex);
858
+ const isSourceStart = !sourceSawRows.has(sourceIndex);
859
+ sourceSawRows.add(sourceIndex);
860
+ recordApiJsonlRangeCheckpoint(params.indexContext, {
861
+ sourceIndex,
862
+ chunk: checkpointChunk,
863
+ rowIndex,
864
+ byteOffset: lineStartOffset,
865
+ isSourceStart,
866
+ });
867
+ rows.push(buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema));
868
+ rowIndex += 1;
869
+ if (rows.length >= params.limit) {
870
+ stoppedForLimit = true;
871
+ break;
872
+ }
873
+ }
874
+ offset = baseOffset + consumedBytes;
875
+ const consumedAllParsedLines = lastConsumedLineIndex >= parsed.lines.length - 1;
876
+ if (result.hasMore !== true && (!stoppedForLimit || consumedAllParsedLines)) {
877
+ sourceIndex += 1;
878
+ offset = 0;
879
+ currentValidator = null;
880
+ }
881
+ }
882
+ if (sourceIndex >= params.sourceUrls.length) {
883
+ return { rows, nextCursor: null };
884
+ }
885
+ const nextUrl = params.sourceUrls[sourceIndex];
886
+ return {
887
+ rows,
888
+ nextCursor: encodeApiJsonlRangeCursor({
889
+ sourceKind: 'api_jsonl_range',
890
+ sourceIndex,
891
+ offset,
892
+ rowIndex,
893
+ url: nextUrl,
894
+ ...(currentValidator?.url === nextUrl
895
+ ? {
896
+ etag: currentValidator.etag ?? null,
897
+ lastModified: currentValidator.lastModified ?? null,
898
+ totalSize: currentValidator.totalSize ?? null,
899
+ }
900
+ : {}),
901
+ }),
902
+ };
903
+ }
904
+ async function fetchApiJsonlSources(params) {
905
+ const sources = [];
906
+ let totalRows = 0;
907
+ if (params.db) {
908
+ pruneApiJsonlResponseCache(params.db, params.panelId, params.sourceManifestKey);
909
+ }
910
+ for (const [sourceIndex, url] of params.sourceUrls.entries()) {
911
+ const cached = params.db
912
+ ? loadApiJsonlResponseCache(params.db, {
913
+ panelId: params.panelId,
914
+ sourceManifestKey: params.sourceManifestKey,
915
+ sourceIndex,
916
+ url,
917
+ })
918
+ : null;
919
+ const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
920
+ maxBytes: API_JSONL_MAX_BYTES,
921
+ ...(params.authProfile ? { authProfile: params.authProfile } : {}),
922
+ ...(cached?.etag ? { ifNoneMatch: cached.etag } : {}),
923
+ ...(cached?.lastModified ? { ifModifiedSince: cached.lastModified } : {}),
924
+ });
925
+ if (result.notModified || result.status === 304) {
926
+ if (!cached) {
927
+ throw new PanelQueryHandleError('api_jsonl endpoint returned 304 but no cached response is available.', 502);
928
+ }
929
+ totalRows += cached.rowCount;
930
+ if (totalRows > API_JSONL_MAX_ROWS) {
931
+ throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
932
+ }
933
+ const refreshed = {
934
+ ...cached,
935
+ etag: result.etag ?? cached.etag,
936
+ lastModified: result.lastModified ?? cached.lastModified,
937
+ fetchedFromCache: true,
938
+ };
939
+ if (params.db && (refreshed.etag !== cached.etag || refreshed.lastModified !== cached.lastModified)) {
940
+ upsertApiJsonlResponseCache(params.db, {
941
+ panelId: params.panelId,
942
+ sourceManifestKey: params.sourceManifestKey,
943
+ sourceIndex,
944
+ url,
945
+ source: refreshed,
946
+ });
947
+ }
948
+ sources.push(refreshed);
949
+ continue;
950
+ }
951
+ const byteLength = Buffer.byteLength(result.content, 'utf8');
952
+ if (byteLength > API_JSONL_MAX_BYTES) {
953
+ throw new PanelQueryHandleError(`api_jsonl response exceeds byte limit (${API_JSONL_MAX_BYTES}).`, 413);
954
+ }
955
+ const rowCount = countApiJsonlRows(result.content);
956
+ totalRows += rowCount;
957
+ if (totalRows > API_JSONL_MAX_ROWS) {
958
+ throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
959
+ }
960
+ const source = {
961
+ url,
962
+ content: result.content,
963
+ byteLength,
964
+ contentHash: createHash('sha256').update(result.content).digest('hex'),
965
+ rowCount,
966
+ contentType: result.contentType,
967
+ etag: result.etag ?? null,
968
+ lastModified: result.lastModified ?? null,
969
+ fetchedFromCache: false,
970
+ };
971
+ sources.push(source);
972
+ if (params.db) {
973
+ upsertApiJsonlResponseCache(params.db, {
974
+ panelId: params.panelId,
975
+ sourceManifestKey: params.sourceManifestKey,
976
+ sourceIndex,
977
+ url,
978
+ source,
979
+ });
980
+ }
981
+ }
982
+ return sources;
983
+ }
984
+ function countApiJsonlRows(content) {
985
+ return content.split('\n').filter((line) => line.trim() !== '').length;
986
+ }
987
+ function createApiJsonlIndexContext(db, panelId, sourceUrls, authProfile) {
988
+ if (!db || sourceUrls.length < 1)
989
+ return null;
990
+ const sourceKeys = sourceUrls.map((url) => `api_jsonl:${url}`);
991
+ return {
992
+ db,
993
+ panelId,
994
+ sourceManifestKey: apiJsonlSourceManifestKey(sourceUrls, authProfile),
995
+ sourceKeys,
996
+ metadataBySourceIndex: new Map(),
997
+ };
998
+ }
999
+ export function apiJsonlSourceManifestKey(sourceUrls, authProfile) {
1000
+ return JSON.stringify({ kind: 'api_jsonl', authProfile, urls: sourceUrls });
1001
+ }
1002
+ function apiJsonlMetadataKey(sources) {
1003
+ return JSON.stringify(sources.map((source) => ({
1004
+ url: source.url,
1005
+ byteLength: source.byteLength,
1006
+ rowCount: source.rowCount,
1007
+ sha256: source.contentHash,
1008
+ })));
1009
+ }
1010
+ function loadApiJsonlResponseCache(db, params) {
1011
+ const row = db.prepare(`SELECT url,
1012
+ content,
1013
+ byte_length as byteLength,
1014
+ content_hash as contentHash,
1015
+ row_count as rowCount,
1016
+ content_type as contentType,
1017
+ etag,
1018
+ last_modified as lastModified
1019
+ FROM panel_api_jsonl_response_cache
1020
+ WHERE panel_id = ?
1021
+ AND source_manifest_key = ?
1022
+ AND source_index = ?
1023
+ AND url = ?
1024
+ LIMIT 1`).get(params.panelId, params.sourceManifestKey, params.sourceIndex, params.url);
1025
+ return row ? { ...row, fetchedFromCache: true } : null;
1026
+ }
1027
+ function hasApiJsonlResponseCacheForAllSources(db, params) {
1028
+ if (!db || params.sourceUrls.length === 0)
1029
+ return false;
1030
+ const lookup = db.prepare(`SELECT 1
1031
+ FROM panel_api_jsonl_response_cache
1032
+ WHERE panel_id = ?
1033
+ AND source_manifest_key = ?
1034
+ AND source_index = ?
1035
+ AND url = ?
1036
+ LIMIT 1`);
1037
+ return params.sourceUrls.every((url, sourceIndex) => Boolean(lookup.get(params.panelId, params.sourceManifestKey, sourceIndex, url)));
1038
+ }
1039
+ function upsertApiJsonlResponseCache(db, params) {
1040
+ const now = Date.now();
1041
+ db.prepare(`INSERT INTO panel_api_jsonl_response_cache(
1042
+ panel_id, source_manifest_key, source_index, url,
1043
+ etag, last_modified, content_type, content, content_hash,
1044
+ byte_length, row_count, fetched_at, updated_at
1045
+ )
1046
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
1047
+ ON CONFLICT(panel_id, source_manifest_key, source_index, url)
1048
+ DO UPDATE SET
1049
+ etag = excluded.etag,
1050
+ last_modified = excluded.last_modified,
1051
+ content_type = excluded.content_type,
1052
+ content = excluded.content,
1053
+ content_hash = excluded.content_hash,
1054
+ byte_length = excluded.byte_length,
1055
+ row_count = excluded.row_count,
1056
+ updated_at = excluded.updated_at`).run(params.panelId, params.sourceManifestKey, params.sourceIndex, params.url, params.source.etag, params.source.lastModified, params.source.contentType, params.source.content, params.source.contentHash, params.source.byteLength, params.source.rowCount, now, now);
1057
+ }
1058
+ export function pruneApiJsonlResponseCache(db, panelId, sourceManifestKey) {
1059
+ const now = Date.now();
1060
+ const ttlCutoff = now - PANEL_API_JSONL_RESPONSE_CACHE_TTL_MS;
1061
+ const evictedKeys = new Set();
1062
+ // Source-churn eviction: any manifest key other than the active one is stale.
1063
+ const nonActiveKeys = db.prepare(`SELECT DISTINCT source_manifest_key
1064
+ FROM panel_api_jsonl_response_cache
1065
+ WHERE panel_id = ?
1066
+ AND source_manifest_key != ?`).all(panelId, sourceManifestKey);
1067
+ for (const row of nonActiveKeys) {
1068
+ evictedKeys.add(row.source_manifest_key);
1069
+ }
1070
+ if (nonActiveKeys.length > 0) {
1071
+ db.prepare(`DELETE FROM panel_api_jsonl_response_cache
1072
+ WHERE panel_id = ?
1073
+ AND source_manifest_key != ?`).run(panelId, sourceManifestKey);
1074
+ }
1075
+ // TTL eviction for the active manifest key (whole-source granularity).
1076
+ const staleActiveKey = db.prepare(`SELECT source_manifest_key
1077
+ FROM panel_api_jsonl_response_cache
1078
+ WHERE panel_id = ?
1079
+ AND source_manifest_key = ?
1080
+ GROUP BY source_manifest_key
1081
+ HAVING MAX(updated_at) < ?`).get(panelId, sourceManifestKey, ttlCutoff);
1082
+ if (staleActiveKey) {
1083
+ evictedKeys.add(staleActiveKey.source_manifest_key);
1084
+ db.prepare(`DELETE FROM panel_api_jsonl_response_cache
1085
+ WHERE panel_id = ?
1086
+ AND source_manifest_key = ?`).run(panelId, staleActiveKey.source_manifest_key);
1087
+ }
1088
+ // Size-budget eviction, preserving the active manifest key until last.
1089
+ let totalBytes = db.prepare(`SELECT COALESCE(SUM(byte_length), 0) as total
1090
+ FROM panel_api_jsonl_response_cache
1091
+ WHERE panel_id = ?`).get(panelId).total;
1092
+ if (totalBytes > PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL) {
1093
+ const candidates = db.prepare(`SELECT source_manifest_key, SUM(byte_length) as byte_length
1094
+ FROM panel_api_jsonl_response_cache
1095
+ WHERE panel_id = ?
1096
+ GROUP BY source_manifest_key
1097
+ ORDER BY CASE WHEN source_manifest_key = ? THEN 1 ELSE 0 END ASC,
1098
+ MAX(updated_at) ASC,
1099
+ source_manifest_key ASC`).all(panelId, sourceManifestKey);
1100
+ for (const candidate of candidates) {
1101
+ if (totalBytes <= PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL)
1102
+ break;
1103
+ if (evictedKeys.has(candidate.source_manifest_key))
1104
+ continue;
1105
+ db.prepare(`DELETE FROM panel_api_jsonl_response_cache
1106
+ WHERE panel_id = ?
1107
+ AND source_manifest_key = ?`).run(panelId, candidate.source_manifest_key);
1108
+ totalBytes -= candidate.byte_length;
1109
+ evictedKeys.add(candidate.source_manifest_key);
1110
+ }
1111
+ }
1112
+ if (evictedKeys.size === 0)
1113
+ return;
1114
+ const keys = Array.from(evictedKeys);
1115
+ const placeholders = keys.map(() => '?').join(',');
1116
+ db.transaction(() => {
1117
+ db.prepare(`DELETE FROM panel_query_handle_view_index_meta
1118
+ WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
1119
+ db.prepare(`DELETE FROM panel_query_handle_view_index_rows
1120
+ WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
1121
+ db.prepare(`DELETE FROM panel_query_handle_field_index_meta
1122
+ WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
1123
+ db.prepare(`DELETE FROM panel_query_handle_field_index_rows
1124
+ WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
1125
+ })();
1126
+ }
1127
+ function parseApiJsonlRowsFromSources(sources, datasetSchema) {
1128
+ let rows = [];
1129
+ for (const source of sources) {
1130
+ rows = rows.concat(parseApiJsonlRows(source.content, datasetSchema, rows.length));
1131
+ }
1132
+ return rows;
1133
+ }
1134
+ async function selectApiJsonlRowsByIndices(sources, datasetSchema, rowIndices) {
1135
+ const targetIndices = new Set(rowIndices);
1136
+ if (targetIndices.size === 0)
1137
+ return [];
1138
+ const found = new Map();
1139
+ let globalRowIndex = 0;
1140
+ for (const source of sources) {
1141
+ for (const line of source.content.split('\n')) {
1142
+ const trimmed = line.trim();
1143
+ if (!trimmed)
1144
+ continue;
1145
+ if (targetIndices.has(globalRowIndex)) {
1146
+ const rawRow = parseJsonlRow(trimmed, globalRowIndex);
1147
+ found.set(globalRowIndex, buildPanelRowFromQueryRow(rawRow, globalRowIndex, datasetSchema));
1148
+ if (found.size === targetIndices.size) {
1149
+ return rowIndices
1150
+ .map((rowIndex) => found.get(rowIndex))
1151
+ .filter((row) => Boolean(row));
1152
+ }
1153
+ }
1154
+ globalRowIndex += 1;
1155
+ }
1156
+ }
1157
+ return rowIndices
1158
+ .map((rowIndex) => found.get(rowIndex))
1159
+ .filter((row) => Boolean(row));
1160
+ }
1161
+ function parseApiJsonlRows(content, datasetSchema, startRowIndex = 0) {
1162
+ const lines = content.split('\n').filter((line) => line.trim() !== '');
1163
+ if (startRowIndex + lines.length > API_JSONL_MAX_ROWS) {
1164
+ throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
1165
+ }
1166
+ return lines.map((line, rowIndex) => {
1167
+ const globalRowIndex = startRowIndex + rowIndex;
1168
+ const rawRow = parseJsonlRow(line.trim(), globalRowIndex);
1169
+ return buildPanelRowFromQueryRow(rawRow, globalRowIndex, datasetSchema);
1170
+ });
1171
+ }
1172
+ function sliceRowsByPositionCursor(rows, startPosition, limit, encodeCursor) {
1173
+ const start = Math.min(Math.max(0, startPosition), rows.length);
1174
+ const resultRows = rows.slice(start, start + limit);
1175
+ const nextPosition = start + resultRows.length;
1176
+ return {
1177
+ rows: resultRows,
1178
+ nextCursor: nextPosition < rows.length ? encodeCursor(nextPosition) : null,
1179
+ };
1180
+ }
1181
+ export async function listPanelQueryHandleRowsByIndices(params) {
1182
+ const uniqueIndices = Array.from(new Set(params.rowIndices.filter((rowIndex) => Number.isInteger(rowIndex) && rowIndex >= 0)));
1183
+ if (uniqueIndices.length === 0) {
1184
+ return { rows: [], nextCursor: null };
1185
+ }
1186
+ const targetIndices = new Set(uniqueIndices);
1187
+ const maxTargetIndex = Math.max(...uniqueIndices);
1188
+ const found = new Map();
1189
+ let cursor = await getIndexedCursorForDynamicRowIndices(params, Math.min(...uniqueIndices));
1190
+ while (found.size < targetIndices.size) {
1191
+ const result = await listPanelQueryHandleRows({
1192
+ db: params.db,
1193
+ panel: params.panel,
1194
+ conversationManager: params.conversationManager,
1195
+ workspaceBroker: params.workspaceBroker,
1196
+ config: params.config,
1197
+ apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
1198
+ apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
1199
+ apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
1200
+ datasetSchema: params.datasetSchema,
1201
+ options: {
1202
+ cursor,
1203
+ limit: 200,
1204
+ },
1205
+ });
1206
+ for (const row of result.rows) {
1207
+ if (targetIndices.has(row.rowIndex)) {
1208
+ found.set(row.rowIndex, row);
1209
+ }
1210
+ }
1211
+ const lastRowIndex = result.rows[result.rows.length - 1]?.rowIndex;
1212
+ if (!result.nextCursor || lastRowIndex === undefined || lastRowIndex >= maxTargetIndex) {
1213
+ break;
1214
+ }
1215
+ cursor = result.nextCursor;
1216
+ }
1217
+ return {
1218
+ rows: params.rowIndices
1219
+ .map((rowIndex) => found.get(rowIndex))
1220
+ .filter((row) => Boolean(row)),
1221
+ nextCursor: null,
1222
+ };
1223
+ }
1224
+ function resolveQueryHandleDataSources(params) {
1225
+ const parsed = parsePanelQueryHandle(params.handle);
1226
+ return parsed.paths.map((sourcePath) => resolveQueryHandleDataSourcePath({
1227
+ sourcePath,
1228
+ workspaceRoot: params.workspaceRoot,
1229
+ config: params.config,
1230
+ }));
1231
+ }
1232
+ function resolveQueryHandleDataSourcePath(params) {
1233
+ if (!path.isAbsolute(params.sourcePath)) {
1234
+ return {
1235
+ kind: 'workspace_relative',
1236
+ relativePath: params.sourcePath,
1237
+ };
1238
+ }
1239
+ const resolved = resolvePanelDatasetPath(params.workspaceRoot, params.sourcePath, params.config);
1240
+ if (!resolved.ok) {
1241
+ throw new PanelQueryHandleError(resolved.reason, 403);
1242
+ }
1243
+ return {
1244
+ kind: 'absolute',
1245
+ absolutePath: resolved.absolutePath,
1246
+ allowedRoot: resolved.allowedRoot,
1247
+ };
1248
+ }
1249
+ function createQueryHandleIndexContext(db, panelId, dataSources) {
1250
+ if (!db || dataSources.length < 1)
1251
+ return null;
1252
+ const sourceKeys = dataSources.map((dataSource) => queryHandleSourceKey(dataSource));
1253
+ return {
1254
+ db,
1255
+ panelId,
1256
+ sourceManifestKey: JSON.stringify(sourceKeys),
1257
+ sourceKeys,
1258
+ metadataBySourceIndex: new Map(),
1259
+ };
1260
+ }
1261
+ async function getIndexedCursorForDynamicRowIndices(params, minTargetIndex) {
1262
+ if (params.panel.datasetSource?.kind === 'api_jsonl') {
1263
+ return getIndexedCursorForApiJsonlRowIndices(params, minTargetIndex);
1264
+ }
1265
+ return getIndexedCursorForQueryHandleRowIndices(params, minTargetIndex);
1266
+ }
1267
+ async function getIndexedCursorForQueryHandleRowIndices(params, minTargetIndex) {
1268
+ if (!params.db || minTargetIndex <= 0 || params.panel.datasetSource?.kind !== 'query_handle') {
1269
+ return null;
1270
+ }
1271
+ const agent = params.conversationManager.getAgent(params.panel.agentId);
1272
+ if (!agent?.nodeId || !agent.workspacePath || !params.workspaceBroker) {
1273
+ return null;
1274
+ }
1275
+ const dataSources = resolveQueryHandleDataSources({
1276
+ handle: params.panel.datasetSource.handle,
1277
+ workspaceRoot: agent.workspacePath,
1278
+ config: params.config,
1279
+ });
1280
+ const indexContext = createQueryHandleIndexContext(params.db, params.panel.id, dataSources);
1281
+ if (!indexContext)
1282
+ return null;
1283
+ const existing = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
1284
+ if (!existing)
1285
+ return null;
1286
+ const sourceRows = listIndexedQueryHandleSources(indexContext);
1287
+ if (!hasIndexedRowsForEveryQueryHandleSource(indexContext, sourceRows))
1288
+ return null;
1289
+ for (let sourceIndex = 0; sourceIndex < dataSources.length; sourceIndex += 1) {
1290
+ const metadataChunk = await readQueryHandleChunk({
1291
+ workspaceBroker: params.workspaceBroker,
1292
+ nodeId: agent.nodeId,
1293
+ workspaceRoot: agent.workspacePath,
1294
+ dataSource: dataSources[sourceIndex],
1295
+ offset: 0,
1296
+ limit: 1,
1297
+ });
1298
+ if (!ensureQueryHandleIndexMetadataFresh(indexContext, sourceIndex, metadataChunk)) {
1299
+ return null;
1300
+ }
1301
+ }
1302
+ const fresh = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
1303
+ if (!fresh)
1304
+ return null;
1305
+ return encodeQueryHandleCursor({
1306
+ sourceIndex: fresh.sourceIndex,
1307
+ offset: fresh.byteOffset,
1308
+ rowIndex: fresh.rowIndex,
1309
+ });
1310
+ }
1311
+ async function getIndexedCursorForApiJsonlRowIndices(params, minTargetIndex) {
1312
+ if (!params.db || minTargetIndex <= 0 || params.panel.datasetSource?.kind !== 'api_jsonl') {
1313
+ return null;
1314
+ }
1315
+ const agent = params.conversationManager.getAgent(params.panel.agentId);
1316
+ if (!agent?.nodeId || !params.workspaceBroker)
1317
+ return null;
1318
+ const parsed = parsePanelApiJsonlUrls(params.panel.datasetSource, {
1319
+ allowedOrigins: params.apiJsonlAllowedOrigins,
1320
+ allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
1321
+ allowedAuthProfiles: params.apiJsonlAuthProfiles,
1322
+ });
1323
+ const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, parsed.urls, parsed.authProfile);
1324
+ if (!indexContext)
1325
+ return null;
1326
+ const existing = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
1327
+ if (!existing)
1328
+ return null;
1329
+ const sourceRows = listIndexedQueryHandleSources(indexContext);
1330
+ if (!hasIndexedRowsThroughSource(indexContext, sourceRows, existing.sourceIndex))
1331
+ return null;
1332
+ const sourceMetadata = new Map();
1333
+ for (let sourceIndex = 0; sourceIndex <= existing.sourceIndex; sourceIndex += 1) {
1334
+ const metadata = await readApiJsonlRangeMetadata({
1335
+ workspaceBroker: params.workspaceBroker,
1336
+ nodeId: agent.nodeId,
1337
+ url: parsed.urls[sourceIndex],
1338
+ authProfile: parsed.authProfile,
1339
+ });
1340
+ if (!metadata)
1341
+ return null;
1342
+ sourceMetadata.set(sourceIndex, metadata);
1343
+ if (!ensureQueryHandleIndexMetadataFresh(indexContext, sourceIndex, metadata.chunk)) {
1344
+ return null;
1345
+ }
1346
+ }
1347
+ const fresh = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
1348
+ if (!fresh)
1349
+ return null;
1350
+ const metadata = sourceMetadata.get(fresh.sourceIndex);
1351
+ if (!metadata)
1352
+ return null;
1353
+ return encodeApiJsonlRangeCursor({
1354
+ sourceKind: 'api_jsonl_range',
1355
+ sourceIndex: fresh.sourceIndex,
1356
+ offset: fresh.byteOffset,
1357
+ rowIndex: fresh.rowIndex,
1358
+ url: parsed.urls[fresh.sourceIndex],
1359
+ etag: metadata.etag,
1360
+ lastModified: metadata.lastModified,
1361
+ totalSize: metadata.totalSize,
1362
+ });
1363
+ }
1364
+ function queryHandleSourceKey(dataSource) {
1365
+ return dataSource.kind === 'workspace_relative'
1366
+ ? `workspace:${dataSource.relativePath}`
1367
+ : `absolute:${dataSource.absolutePath}`;
1368
+ }
1369
+ function selectNearestQueryHandleCheckpoint(context, targetRowIndex) {
1370
+ const row = context.db.prepare(`SELECT source_index as sourceIndex, row_index as rowIndex, byte_offset as byteOffset
1371
+ FROM panel_query_handle_index
1372
+ WHERE panel_id = ?
1373
+ AND source_manifest_key = ?
1374
+ AND row_index <= ?
1375
+ ORDER BY row_index DESC
1376
+ LIMIT 1`).get(context.panelId, context.sourceManifestKey, targetRowIndex);
1377
+ if (!row
1378
+ || !Number.isInteger(row.sourceIndex)
1379
+ || !Number.isInteger(row.rowIndex)
1380
+ || !Number.isInteger(row.byteOffset)
1381
+ || context.sourceKeys[row.sourceIndex] == null) {
1382
+ return null;
1383
+ }
1384
+ return row;
1385
+ }
1386
+ function listIndexedQueryHandleSources(context) {
1387
+ return context.db.prepare(`SELECT source_index as sourceIndex, source_key as sourceKey
1388
+ FROM panel_query_handle_index
1389
+ WHERE panel_id = ?
1390
+ AND source_manifest_key = ?
1391
+ GROUP BY source_index, source_key
1392
+ ORDER BY source_index ASC`).all(context.panelId, context.sourceManifestKey);
1393
+ }
1394
+ function hasIndexedRowsForEveryQueryHandleSource(context, rows) {
1395
+ const bySourceIndex = new Map(rows.map((row) => [row.sourceIndex, row.sourceKey]));
1396
+ return context.sourceKeys.every((sourceKey, sourceIndex) => bySourceIndex.get(sourceIndex) === sourceKey);
1397
+ }
1398
+ function hasIndexedRowsThroughSource(context, rows, maxSourceIndex) {
1399
+ const bySourceIndex = new Map(rows.map((row) => [row.sourceIndex, row.sourceKey]));
1400
+ for (let sourceIndex = 0; sourceIndex <= maxSourceIndex; sourceIndex += 1) {
1401
+ if (bySourceIndex.get(sourceIndex) !== context.sourceKeys[sourceIndex])
1402
+ return false;
1403
+ }
1404
+ return true;
1405
+ }
1406
+ async function readApiJsonlRangeMetadata(params) {
1407
+ const result = await params.workspaceBroker.fetchHttpText(params.nodeId, params.url, {
1408
+ maxBytes: API_JSONL_MAX_BYTES,
1409
+ ...(params.authProfile ? { authProfile: params.authProfile } : {}),
1410
+ rangeStart: 0,
1411
+ rangeLength: 1,
1412
+ });
1413
+ if (result.status !== 206
1414
+ || result.partialContent !== true
1415
+ || result.rangeStart !== 0) {
1416
+ return null;
1417
+ }
1418
+ const chunk = apiJsonlRangeCheckpointChunk(result);
1419
+ if (!chunk || chunk.modifiedAt == null)
1420
+ return null;
1421
+ return {
1422
+ chunk,
1423
+ etag: result.etag ?? null,
1424
+ lastModified: result.lastModified ?? null,
1425
+ totalSize: chunk.size,
1426
+ };
1427
+ }
1428
+ async function ensureApiJsonlRangeFieldIndex(params) {
1429
+ if (!params.db || !params.indexContext)
1430
+ return null;
1431
+ const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
1432
+ if (!fieldSchemaKey)
1433
+ return null;
1434
+ const sourceMetadata = [];
1435
+ for (const [sourceIndex, url] of params.sourceUrls.entries()) {
1436
+ const metadata = await readApiJsonlRangeMetadata({
1437
+ workspaceBroker: params.workspaceBroker,
1438
+ nodeId: params.nodeId,
1439
+ url,
1440
+ authProfile: params.authProfile,
1441
+ });
1442
+ if (!metadata)
1443
+ return null;
1444
+ sourceMetadata.push(metadata);
1445
+ if (!ensureQueryHandleIndexMetadataFresh(params.indexContext, sourceIndex, metadata.chunk)) {
1446
+ return null;
1447
+ }
1448
+ }
1449
+ const metadataKey = apiJsonlRangeMetadataKey(params.sourceUrls, sourceMetadata);
1450
+ const existing = params.db.prepare(`SELECT 1
1451
+ FROM panel_query_handle_field_index_meta
1452
+ WHERE panel_id = ?
1453
+ AND source_manifest_key = ?
1454
+ AND metadata_key = ?
1455
+ AND field_schema_key = ?
1456
+ LIMIT 1`).get(params.panel.id, params.indexContext.sourceManifestKey, metadataKey, fieldSchemaKey);
1457
+ if (existing)
1458
+ return metadataKey;
1459
+ const rows = await scanApiJsonlRowsForFieldIndexByRange({
1460
+ panel: params.panel,
1461
+ sourceUrls: params.sourceUrls,
1462
+ authProfile: params.authProfile,
1463
+ workspaceBroker: params.workspaceBroker,
1464
+ nodeId: params.nodeId,
1465
+ datasetSchema: params.datasetSchema,
1466
+ indexContext: params.indexContext,
1467
+ sourceMetadata,
1468
+ });
1469
+ if (!rows)
1470
+ return null;
1471
+ writeQueryHandleFieldIndexRows({
1472
+ context: params.indexContext,
1473
+ metadataKey,
1474
+ datasetSchema: params.datasetSchema,
1475
+ rows,
1476
+ extraFieldNames: params.extraFieldNames,
1477
+ });
1478
+ return metadataKey;
1479
+ }
1480
+ function apiJsonlRangeMetadataKey(sourceUrls, metadata) {
1481
+ return JSON.stringify(metadata.map((source, sourceIndex) => ({
1482
+ url: sourceUrls[sourceIndex],
1483
+ byteLength: source.totalSize,
1484
+ etag: source.etag,
1485
+ lastModified: source.lastModified,
1486
+ })));
1487
+ }
1488
+ async function scanApiJsonlRowsForFieldIndexByRange(params) {
1489
+ void params.panel;
1490
+ const rows = [];
1491
+ let rowIndex = 0;
1492
+ let chunks = 0;
1493
+ const sourceSawRows = new Set();
1494
+ for (const [sourceIndex, url] of params.sourceUrls.entries()) {
1495
+ const metadata = params.sourceMetadata[sourceIndex];
1496
+ if (!metadata)
1497
+ return null;
1498
+ let offset = 0;
1499
+ let hasMore = true;
1500
+ const currentValidator = {
1501
+ url,
1502
+ etag: metadata.etag,
1503
+ lastModified: metadata.lastModified,
1504
+ totalSize: metadata.totalSize,
1505
+ };
1506
+ while (hasMore) {
1507
+ chunks += 1;
1508
+ if (chunks > API_JSONL_MAX_RANGE_CHUNKS_PER_INDEX) {
1509
+ throw new PanelQueryHandleError('api_jsonl range index scan exceeded the per-request work limit.', 413);
1510
+ }
1511
+ const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
1512
+ maxBytes: API_JSONL_MAX_BYTES,
1513
+ ...(params.authProfile ? { authProfile: params.authProfile } : {}),
1514
+ rangeStart: offset,
1515
+ rangeLength: API_JSONL_RANGE_CHUNK_BYTES,
1516
+ ...withApiJsonlIfRange(currentValidator),
1517
+ });
1518
+ if (result.status !== 206
1519
+ || result.partialContent !== true
1520
+ || typeof result.rangeStart !== 'number'
1521
+ || typeof result.rangeEnd !== 'number'
1522
+ || result.rangeStart !== offset) {
1523
+ return null;
1524
+ }
1525
+ const checkpointChunk = apiJsonlRangeCheckpointChunk(result);
1526
+ if (!checkpointChunk || !ensureQueryHandleIndexMetadataFresh(params.indexContext, sourceIndex, checkpointChunk)) {
1527
+ return null;
1528
+ }
1529
+ const parsed = parseCompleteJsonlLines(result.content, Boolean(result.hasMore));
1530
+ if (parsed.completeText.length === 0 && result.hasMore) {
1531
+ throw new PanelQueryHandleError('api_jsonl JSONL row exceeds the per-row size limit.', 413);
1532
+ }
1533
+ const baseOffset = result.rangeStart;
1534
+ let consumedBytes = 0;
1535
+ for (const line of parsed.lines) {
1536
+ const lineStartOffset = baseOffset + consumedBytes;
1537
+ consumedBytes += line.byteLength;
1538
+ const trimmed = line.text.trim();
1539
+ if (!trimmed)
1540
+ continue;
1541
+ if (rowIndex >= API_JSONL_MAX_ROWS) {
1542
+ throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
1543
+ }
1544
+ const rawRow = parseJsonlRow(trimmed, rowIndex);
1545
+ const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
1546
+ const isSourceStart = !sourceSawRows.has(sourceIndex);
1547
+ sourceSawRows.add(sourceIndex);
1548
+ recordApiJsonlRangeCheckpoint(params.indexContext, {
1549
+ sourceIndex,
1550
+ chunk: checkpointChunk,
1551
+ rowIndex,
1552
+ byteOffset: lineStartOffset,
1553
+ isSourceStart,
1554
+ });
1555
+ rows.push({ rowIndex: row.rowIndex, fields: row.fields });
1556
+ rowIndex += 1;
1557
+ }
1558
+ offset = baseOffset + consumedBytes;
1559
+ hasMore = result.hasMore === true;
1560
+ if (parsed.lines.length === 0 && !hasMore)
1561
+ break;
1562
+ }
1563
+ }
1564
+ return rows;
1565
+ }
1566
+ function queryHandleViewIndexQueryKey(options) {
1567
+ return JSON.stringify({
1568
+ sortField: options.sortField ?? null,
1569
+ sortDirection: options.sortDirection ?? null,
1570
+ sortFieldType: options.sortFieldType ?? null,
1571
+ filterField: options.filterField ?? null,
1572
+ filterValue: options.filterValue ?? null,
1573
+ });
1574
+ }
1575
+ function parseQueryHandleViewIndexCursorRequest(queryKey) {
1576
+ try {
1577
+ const parsed = JSON.parse(queryKey);
1578
+ if (typeof parsed.sortField !== 'string')
1579
+ return null;
1580
+ if (parsed.sortDirection !== 'asc' && parsed.sortDirection !== 'desc')
1581
+ return null;
1582
+ return {
1583
+ sortField: parsed.sortField,
1584
+ sortDirection: parsed.sortDirection,
1585
+ ...(typeof parsed.filterField === 'string' ? { filterField: parsed.filterField } : {}),
1586
+ ...(parsed.filterValue !== null && parsed.filterValue !== undefined ? { filterValue: parsed.filterValue } : {}),
1587
+ };
1588
+ }
1589
+ catch {
1590
+ return null;
1591
+ }
1592
+ }
1593
+ async function loadQueryHandleIndexMetadataKey(params) {
1594
+ for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
1595
+ const existing = params.context.metadataBySourceIndex.get(sourceIndex);
1596
+ if (existing)
1597
+ continue;
1598
+ const metadataChunk = await readQueryHandleChunk({
1599
+ workspaceBroker: params.workspaceBroker,
1600
+ nodeId: params.nodeId,
1601
+ workspaceRoot: params.workspaceRoot,
1602
+ dataSource: params.dataSources[sourceIndex],
1603
+ offset: 0,
1604
+ limit: 1,
1605
+ });
1606
+ if (!ensureQueryHandleIndexMetadataFresh(params.context, sourceIndex, metadataChunk)) {
1607
+ return null;
1608
+ }
1609
+ }
1610
+ const metadata = params.context.sourceKeys.map((sourceKey, sourceIndex) => {
1611
+ const sourceMetadata = params.context.metadataBySourceIndex.get(sourceIndex);
1612
+ return sourceMetadata ? {
1613
+ sourceKey,
1614
+ fileSize: sourceMetadata.fileSize,
1615
+ modifiedAt: sourceMetadata.modifiedAt,
1616
+ } : null;
1617
+ });
1618
+ return metadata.every(Boolean) ? JSON.stringify(metadata) : null;
1619
+ }
1620
+ async function readCachedQueryHandleViewIndexRows(params) {
1621
+ if (!params.db)
1622
+ return null;
1623
+ const meta = params.db.prepare(`SELECT total_count as totalCount
1624
+ FROM panel_query_handle_view_index_meta
1625
+ WHERE panel_id = ?
1626
+ AND source_manifest_key = ?
1627
+ AND query_key = ?
1628
+ AND metadata_key = ?`).get(params.panel.id, params.sourceManifestKey, params.queryKey, params.metadataKey);
1629
+ if (!meta || !Number.isInteger(meta.totalCount) || meta.totalCount < 0)
1630
+ return null;
1631
+ const start = Math.min(Math.max(0, params.position), meta.totalCount);
1632
+ const expectedRows = Math.min(params.limit, Math.max(0, meta.totalCount - start));
1633
+ if (expectedRows === 0) {
1634
+ return { rows: [], nextCursor: null };
1635
+ }
1636
+ const indexRows = params.db.prepare(`SELECT row_index as rowIndex
1637
+ FROM panel_query_handle_view_index_rows
1638
+ WHERE panel_id = ?
1639
+ AND source_manifest_key = ?
1640
+ AND query_key = ?
1641
+ AND metadata_key = ?
1642
+ AND position >= ?
1643
+ ORDER BY position ASC
1644
+ LIMIT ?`).all(params.panel.id, params.sourceManifestKey, params.queryKey, params.metadataKey, start, expectedRows);
1645
+ if (indexRows.length !== expectedRows || indexRows.some((row) => !Number.isInteger(row.rowIndex))) {
1646
+ return null;
1647
+ }
1648
+ const rowIndices = indexRows.map((row) => row.rowIndex);
1649
+ const selectedRows = params.selectRowsByIndices
1650
+ ? await params.selectRowsByIndices(rowIndices)
1651
+ : (await listPanelQueryHandleRowsByIndices({
1652
+ db: params.db,
1653
+ panel: params.panel,
1654
+ conversationManager: params.conversationManager,
1655
+ workspaceBroker: params.workspaceBroker,
1656
+ config: params.config,
1657
+ datasetSchema: params.datasetSchema,
1658
+ rowIndices,
1659
+ })).rows;
1660
+ if (selectedRows.length !== indexRows.length)
1661
+ return null;
1662
+ const nextPosition = start + selectedRows.length;
1663
+ const cursorRequest = parseQueryHandleViewIndexCursorRequest(params.queryKey);
1664
+ if (!cursorRequest)
1665
+ return null;
1666
+ return {
1667
+ rows: selectedRows,
1668
+ nextCursor: nextPosition < meta.totalCount
1669
+ ? encodeQueryHandleSortCursor({
1670
+ position: nextPosition,
1671
+ ...cursorRequest,
1672
+ })
1673
+ : null,
1674
+ };
1675
+ }
1676
+ function writeQueryHandleViewIndexRows(params) {
1677
+ const now = Date.now();
1678
+ const deleteMeta = params.context.db.prepare(`DELETE FROM panel_query_handle_view_index_meta
1679
+ WHERE panel_id = ?
1680
+ AND source_manifest_key = ?
1681
+ AND query_key = ?`);
1682
+ const deleteRows = params.context.db.prepare(`DELETE FROM panel_query_handle_view_index_rows
1683
+ WHERE panel_id = ?
1684
+ AND source_manifest_key = ?
1685
+ AND query_key = ?`);
1686
+ const insertMeta = params.context.db.prepare(`INSERT INTO panel_query_handle_view_index_meta(
1687
+ panel_id, source_manifest_key, query_key, metadata_key,
1688
+ total_count, created_at, updated_at
1689
+ )
1690
+ VALUES(?, ?, ?, ?, ?, ?, ?)`);
1691
+ const insertRow = params.context.db.prepare(`INSERT INTO panel_query_handle_view_index_rows(
1692
+ panel_id, source_manifest_key, query_key, metadata_key,
1693
+ position, row_index, created_at
1694
+ )
1695
+ VALUES(?, ?, ?, ?, ?, ?, ?)`);
1696
+ params.context.db.transaction(() => {
1697
+ deleteRows.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey);
1698
+ deleteMeta.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey);
1699
+ insertMeta.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey, params.metadataKey, params.rowIndices.length, now, now);
1700
+ params.rowIndices.forEach((rowIndex, position) => {
1701
+ insertRow.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey, params.metadataKey, position, rowIndex, now);
1702
+ });
1703
+ })();
1704
+ }
1705
+ function aggregateFieldIndexNames(options) {
1706
+ return Array.from(new Set([
1707
+ ...(options.field ? [options.field] : []),
1708
+ ...(options.filterField ? [options.filterField] : []),
1709
+ ]));
1710
+ }
1711
+ async function scanQueryHandleRowsForFieldIndex(params) {
1712
+ const rows = [];
1713
+ let rowIndex = 0;
1714
+ let chunks = 0;
1715
+ const sourceSawRows = new Set();
1716
+ for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
1717
+ const dataSource = params.dataSources[sourceIndex];
1718
+ let offset = 0;
1719
+ let hasMore = true;
1720
+ while (hasMore) {
1721
+ chunks += 1;
1722
+ if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_SORT) {
1723
+ throw new PanelQueryHandleError('query_handle aggregate scan exceeded the per-request work limit.', 413);
1724
+ }
1725
+ const chunk = await readQueryHandleChunk({
1726
+ workspaceBroker: params.workspaceBroker,
1727
+ nodeId: params.nodeId,
1728
+ workspaceRoot: params.workspaceRoot,
1729
+ dataSource,
1730
+ offset,
1731
+ limit: QUERY_HANDLE_READ_CHUNK_BYTES,
1732
+ });
1733
+ const content = chunk.content ?? '';
1734
+ const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
1735
+ if (parsed.completeText.length === 0 && chunk.hasMore) {
1736
+ throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
1737
+ }
1738
+ const baseOffset = chunk.offset ?? offset;
1739
+ let consumedBytes = 0;
1740
+ for (const line of parsed.lines) {
1741
+ const lineStartOffset = baseOffset + consumedBytes;
1742
+ consumedBytes += line.byteLength;
1743
+ const trimmed = line.text.trim();
1744
+ if (!trimmed)
1745
+ continue;
1746
+ const rawRow = parseJsonlRow(trimmed, rowIndex);
1747
+ const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
1748
+ const isSourceStart = !sourceSawRows.has(sourceIndex);
1749
+ sourceSawRows.add(sourceIndex);
1750
+ recordQueryHandleCheckpoint(params.indexContext, {
1751
+ sourceIndex,
1752
+ chunk,
1753
+ rowIndex,
1754
+ byteOffset: lineStartOffset,
1755
+ isSourceStart,
1756
+ });
1757
+ rows.push({ rowIndex: row.rowIndex, fields: row.fields });
1758
+ rowIndex += 1;
1759
+ }
1760
+ offset = baseOffset + consumedBytes;
1761
+ hasMore = offset < chunk.size;
1762
+ if (parsed.lines.length === 0 && !hasMore)
1763
+ break;
1764
+ }
1765
+ }
1766
+ if (params.indexContext && params.metadataKey) {
1767
+ writeQueryHandleFieldIndexRows({
1768
+ context: params.indexContext,
1769
+ metadataKey: params.metadataKey,
1770
+ datasetSchema: params.datasetSchema,
1771
+ rows,
1772
+ extraFieldNames: params.extraFieldNames,
1773
+ });
1774
+ }
1775
+ return rows;
1776
+ }
1777
+ function queryHandleFieldIndexFields(datasetSchema, extraFieldNames = []) {
1778
+ const extraFields = new Set(extraFieldNames);
1779
+ return (datasetSchema?.fields ?? []).filter((field) => field.filterable || field.sortable || extraFields.has(field.name));
1780
+ }
1781
+ function queryHandleFieldIndexSchemaKey(datasetSchema, extraFieldNames = []) {
1782
+ const fields = queryHandleFieldIndexFields(datasetSchema, extraFieldNames);
1783
+ if (fields.length === 0)
1784
+ return null;
1785
+ return JSON.stringify(fields.map((field) => ({
1786
+ name: field.name,
1787
+ type: field.type,
1788
+ filterable: Boolean(field.filterable),
1789
+ sortable: Boolean(field.sortable),
1790
+ })));
1791
+ }
1792
+ function readAggregateFromQueryHandleFieldIndex(params) {
1793
+ if (!params.db)
1794
+ return null;
1795
+ void params.sourceKind;
1796
+ const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
1797
+ if (!fieldSchemaKey)
1798
+ return null;
1799
+ const meta = params.db.prepare(`SELECT row_count as rowCount
1800
+ FROM panel_query_handle_field_index_meta
1801
+ WHERE panel_id = ?
1802
+ AND source_manifest_key = ?
1803
+ AND metadata_key = ?
1804
+ AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
1805
+ if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
1806
+ return null;
1807
+ const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
1808
+ if (params.options.filterField && !filterField)
1809
+ return null;
1810
+ const filter = filterField && params.options.filterValue !== undefined
1811
+ ? normalizeQueryHandleIndexedValue(filterField, params.options.filterValue)
1812
+ : null;
1813
+ if (filterField && !filter?.hasValue) {
1814
+ return emptyAggregateResult(params.options);
1815
+ }
1816
+ const totalCount = filterField && filter
1817
+ ? countRowsMatchingQueryHandleFieldFilter({
1818
+ db: params.db,
1819
+ panelId: params.panel.id,
1820
+ sourceManifestKey: params.sourceManifestKey,
1821
+ metadataKey: params.metadataKey,
1822
+ fieldSchemaKey,
1823
+ filterField,
1824
+ filter,
1825
+ })
1826
+ : meta.rowCount;
1827
+ if (params.options.op === 'count' && !params.options.field) {
1828
+ return {
1829
+ op: params.options.op,
1830
+ field: null,
1831
+ value: totalCount,
1832
+ count: totalCount,
1833
+ totalCount,
1834
+ hasValue: true,
1835
+ };
1836
+ }
1837
+ if (!params.options.field)
1838
+ return emptyAggregateResult(params.options, totalCount);
1839
+ const aggregateField = (params.datasetSchema?.fields ?? []).find((field) => field.name === params.options.field);
1840
+ if (!aggregateField)
1841
+ return null;
1842
+ if (!filterField && params.options.filterValue === undefined) {
1843
+ const summary = readAggregateFromQueryHandleAggregateIndex({
1844
+ db: params.db,
1845
+ panelId: params.panel.id,
1846
+ sourceManifestKey: params.sourceManifestKey,
1847
+ metadataKey: params.metadataKey,
1848
+ fieldSchemaKey,
1849
+ aggregateField,
1850
+ op: params.options.op,
1851
+ });
1852
+ if (summary) {
1853
+ return {
1854
+ op: params.options.op,
1855
+ field: aggregateField.name,
1856
+ value: summary.value,
1857
+ count: summary.count,
1858
+ totalCount,
1859
+ hasValue: params.options.op === 'count' || summary.value !== null,
1860
+ };
1861
+ }
1862
+ }
1863
+ const aggregate = queryAggregateFieldIndexRows({
1864
+ db: params.db,
1865
+ panelId: params.panel.id,
1866
+ sourceManifestKey: params.sourceManifestKey,
1867
+ metadataKey: params.metadataKey,
1868
+ fieldSchemaKey,
1869
+ aggregateField,
1870
+ filterField,
1871
+ filter,
1872
+ op: params.options.op,
1873
+ });
1874
+ return {
1875
+ op: params.options.op,
1876
+ field: aggregateField.name,
1877
+ value: aggregate.value,
1878
+ count: aggregate.count,
1879
+ totalCount,
1880
+ hasValue: params.options.op === 'count' || aggregate.value !== null,
1881
+ };
1882
+ }
1883
+ function readAggregateFromQueryHandleAggregateIndex(params) {
1884
+ const row = params.db.prepare(`SELECT value_count as valueCount,
1885
+ non_empty_count as nonEmptyCount,
1886
+ numeric_count as numericCount,
1887
+ numeric_sum as numericSum,
1888
+ numeric_min as numericMin,
1889
+ numeric_max as numericMax
1890
+ FROM panel_query_handle_field_aggregate_index
1891
+ WHERE panel_id = ?
1892
+ AND source_manifest_key = ?
1893
+ AND metadata_key = ?
1894
+ AND field_schema_key = ?
1895
+ AND field_name = ?
1896
+ LIMIT 1`).get(params.panelId, params.sourceManifestKey, params.metadataKey, params.fieldSchemaKey, params.aggregateField.name);
1897
+ if (!row)
1898
+ return null;
1899
+ if (params.op === 'count') {
1900
+ const count = params.aggregateField.type === 'string' ? row.nonEmptyCount : row.valueCount;
1901
+ return { value: count, count };
1902
+ }
1903
+ if (params.aggregateField.type !== 'number' || row.numericCount <= 0) {
1904
+ return { value: null, count: 0 };
1905
+ }
1906
+ if (params.op === 'sum')
1907
+ return { value: row.numericSum, count: row.numericCount };
1908
+ if (params.op === 'avg')
1909
+ return { value: row.numericSum / row.numericCount, count: row.numericCount };
1910
+ if (params.op === 'min')
1911
+ return { value: row.numericMin, count: row.numericCount };
1912
+ return { value: row.numericMax, count: row.numericCount };
1913
+ }
1914
+ function emptyAggregateResult(options, totalCount = 0) {
1915
+ if (options.op === 'count') {
1916
+ return {
1917
+ op: options.op,
1918
+ field: options.field ?? null,
1919
+ value: 0,
1920
+ count: 0,
1921
+ totalCount,
1922
+ hasValue: true,
1923
+ };
1924
+ }
1925
+ return {
1926
+ op: options.op,
1927
+ field: options.field ?? null,
1928
+ value: null,
1929
+ count: 0,
1930
+ totalCount,
1931
+ hasValue: false,
1932
+ };
1933
+ }
1934
+ function countRowsMatchingQueryHandleFieldFilter(params) {
1935
+ const filterColumn = queryHandleFieldValueColumn(params.filterField);
1936
+ const filterValue = filterColumn === 'string_value'
1937
+ ? params.filter.stringValue
1938
+ : filterColumn === 'number_value'
1939
+ ? params.filter.numberValue
1940
+ : params.filter.booleanValue;
1941
+ const row = params.db.prepare(`SELECT COUNT(*) as totalCount
1942
+ FROM panel_query_handle_field_index_rows
1943
+ WHERE panel_id = ?
1944
+ AND source_manifest_key = ?
1945
+ AND metadata_key = ?
1946
+ AND field_schema_key = ?
1947
+ AND field_name = ?
1948
+ AND has_value = 1
1949
+ AND ${filterColumn} = ?`).get(params.panelId, params.sourceManifestKey, params.metadataKey, params.fieldSchemaKey, params.filterField.name, filterValue);
1950
+ return row?.totalCount ?? 0;
1951
+ }
1952
+ function queryAggregateFieldIndexRows(params) {
1953
+ const filterColumn = params.filterField ? queryHandleFieldValueColumn(params.filterField) : null;
1954
+ const filterValue = params.filterField && params.filter && filterColumn
1955
+ ? filterColumn === 'string_value'
1956
+ ? params.filter.stringValue
1957
+ : filterColumn === 'number_value'
1958
+ ? params.filter.numberValue
1959
+ : params.filter.booleanValue
1960
+ : null;
1961
+ const filterJoin = params.filterField
1962
+ ? `JOIN panel_query_handle_field_index_rows f
1963
+ ON f.panel_id = a.panel_id
1964
+ AND f.source_manifest_key = a.source_manifest_key
1965
+ AND f.metadata_key = a.metadata_key
1966
+ AND f.field_schema_key = a.field_schema_key
1967
+ AND f.row_index = a.row_index
1968
+ AND f.field_name = ?`
1969
+ : '';
1970
+ const filterWhere = params.filterField && filterColumn
1971
+ ? `AND f.has_value = 1 AND f.${filterColumn} = ?`
1972
+ : '';
1973
+ const filterParams = params.filterField && filterValue !== null
1974
+ ? [params.filterField.name, filterValue]
1975
+ : [];
1976
+ const nonEmptyPredicate = params.aggregateField.type === 'string'
1977
+ ? "a.has_value = 1 AND trim(a.string_value) != ''"
1978
+ : 'a.has_value = 1';
1979
+ const aggregatePredicate = params.op === 'count'
1980
+ ? nonEmptyPredicate
1981
+ : "a.has_value = 1 AND a.field_type = 'number'";
1982
+ const aggregateExpression = params.op === 'count'
1983
+ ? 'COUNT(*)'
1984
+ : params.op === 'avg'
1985
+ ? 'AVG(a.number_value)'
1986
+ : `${params.op.toUpperCase()}(a.number_value)`;
1987
+ const sql = `
1988
+ SELECT ${aggregateExpression} as value, COUNT(*) as count
1989
+ FROM panel_query_handle_field_index_rows a
1990
+ ${filterJoin}
1991
+ WHERE a.panel_id = ?
1992
+ AND a.source_manifest_key = ?
1993
+ AND a.metadata_key = ?
1994
+ AND a.field_schema_key = ?
1995
+ AND a.field_name = ?
1996
+ AND ${aggregatePredicate}
1997
+ ${filterWhere}`;
1998
+ const paramsForQuery = params.filterField
1999
+ ? [
2000
+ ...filterParams.slice(0, 1),
2001
+ params.panelId,
2002
+ params.sourceManifestKey,
2003
+ params.metadataKey,
2004
+ params.fieldSchemaKey,
2005
+ params.aggregateField.name,
2006
+ ...filterParams.slice(1),
2007
+ ]
2008
+ : [
2009
+ params.panelId,
2010
+ params.sourceManifestKey,
2011
+ params.metadataKey,
2012
+ params.fieldSchemaKey,
2013
+ params.aggregateField.name,
2014
+ ];
2015
+ const row = params.db.prepare(sql).get(...paramsForQuery);
2016
+ const value = typeof row?.value === 'number' && Number.isFinite(row.value) ? row.value : null;
2017
+ return {
2018
+ value: params.op === 'count' ? (row?.count ?? 0) : value,
2019
+ count: row?.count ?? 0,
2020
+ };
2021
+ }
2022
+ function getQueryHandleIndexedField(datasetSchema, fieldName, purpose) {
2023
+ if (!fieldName)
2024
+ return null;
2025
+ const field = (datasetSchema?.fields ?? []).find((candidate) => candidate.name === fieldName);
2026
+ if (!field)
2027
+ return null;
2028
+ if (purpose === 'filter' && !field.filterable)
2029
+ return null;
2030
+ if (purpose === 'sort' && !field.sortable)
2031
+ return null;
2032
+ return field;
2033
+ }
2034
+ function queryHandleFieldValueColumn(field) {
2035
+ if (field.type === 'number')
2036
+ return 'number_value';
2037
+ if (field.type === 'boolean')
2038
+ return 'boolean_value';
2039
+ return 'string_value';
2040
+ }
2041
+ function normalizeQueryHandleIndexedValue(field, value) {
2042
+ if (field.type === 'number') {
2043
+ return typeof value === 'number' && Number.isFinite(value)
2044
+ ? { hasValue: 1, stringValue: '', numberValue: value, booleanValue: 0 }
2045
+ : { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
2046
+ }
2047
+ if (field.type === 'boolean') {
2048
+ return typeof value === 'boolean'
2049
+ ? { hasValue: 1, stringValue: '', numberValue: 0, booleanValue: value ? 1 : 0 }
2050
+ : { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
2051
+ }
2052
+ return typeof value === 'string'
2053
+ ? { hasValue: 1, stringValue: value, numberValue: 0, booleanValue: 0 }
2054
+ : { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
2055
+ }
2056
+ async function readQueryHandleRowsFromFieldIndex(params) {
2057
+ if (!params.db || !params.options.sortField || !params.options.sortDirection)
2058
+ return null;
2059
+ const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema);
2060
+ if (!fieldSchemaKey)
2061
+ return null;
2062
+ const sortField = getQueryHandleIndexedField(params.datasetSchema, params.options.sortField, 'sort');
2063
+ if (!sortField)
2064
+ return null;
2065
+ const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
2066
+ if (params.options.filterField && !filterField)
2067
+ return null;
2068
+ const meta = params.db.prepare(`SELECT row_count as rowCount
2069
+ FROM panel_query_handle_field_index_meta
2070
+ WHERE panel_id = ?
2071
+ AND source_manifest_key = ?
2072
+ AND metadata_key = ?
2073
+ AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
2074
+ if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
2075
+ return null;
2076
+ const sortColumn = queryHandleFieldValueColumn(sortField);
2077
+ const sortDirection = params.options.sortDirection === 'desc' ? 'DESC' : 'ASC';
2078
+ const tieDirection = params.options.sortDirection === 'desc' ? 'DESC' : 'ASC';
2079
+ const start = Math.max(0, params.position);
2080
+ const filter = filterField && params.options.filterValue !== undefined
2081
+ ? normalizeQueryHandleIndexedValue(filterField, params.options.filterValue)
2082
+ : null;
2083
+ if (filterField && !filter?.hasValue)
2084
+ return { rows: [], nextCursor: null };
2085
+ const filterColumn = filterField ? queryHandleFieldValueColumn(filterField) : null;
2086
+ const baseParams = [
2087
+ params.panel.id,
2088
+ params.sourceManifestKey,
2089
+ params.metadataKey,
2090
+ fieldSchemaKey,
2091
+ sortField.name,
2092
+ ];
2093
+ const filterJoin = filterField
2094
+ ? `JOIN panel_query_handle_field_index_rows f
2095
+ ON f.panel_id = s.panel_id
2096
+ AND f.source_manifest_key = s.source_manifest_key
2097
+ AND f.metadata_key = s.metadata_key
2098
+ AND f.field_schema_key = s.field_schema_key
2099
+ AND f.row_index = s.row_index
2100
+ AND f.field_name = ?`
2101
+ : '';
2102
+ const filterWhere = filterField && filterColumn && filter
2103
+ ? `AND f.has_value = 1 AND f.${filterColumn} = ?`
2104
+ : '';
2105
+ const filterParams = filterField && filterColumn && filter
2106
+ ? [filterField.name, filterColumn === 'string_value' ? filter.stringValue : filterColumn === 'number_value' ? filter.numberValue : filter.booleanValue]
2107
+ : [];
2108
+ const countSql = `
2109
+ SELECT COUNT(*) as totalCount
2110
+ FROM panel_query_handle_field_index_rows s
2111
+ ${filterJoin}
2112
+ WHERE s.panel_id = ?
2113
+ AND s.source_manifest_key = ?
2114
+ AND s.metadata_key = ?
2115
+ AND s.field_schema_key = ?
2116
+ AND s.field_name = ?
2117
+ ${filterWhere}`;
2118
+ const countParams = filterField
2119
+ ? [...filterParams.slice(0, 1), ...baseParams, ...filterParams.slice(1)]
2120
+ : baseParams;
2121
+ const count = params.db.prepare(countSql).get(...countParams);
2122
+ const totalCount = count?.totalCount ?? 0;
2123
+ if (totalCount <= start)
2124
+ return { rows: [], nextCursor: null };
2125
+ const rowSql = `
2126
+ SELECT s.row_index as rowIndex
2127
+ FROM panel_query_handle_field_index_rows s
2128
+ ${filterJoin}
2129
+ WHERE s.panel_id = ?
2130
+ AND s.source_manifest_key = ?
2131
+ AND s.metadata_key = ?
2132
+ AND s.field_schema_key = ?
2133
+ AND s.field_name = ?
2134
+ ${filterWhere}
2135
+ ORDER BY s.${sortColumn} ${sortDirection}, s.row_index ${tieDirection}
2136
+ LIMIT ? OFFSET ?`;
2137
+ const rowParams = filterField
2138
+ ? [...filterParams.slice(0, 1), ...baseParams, ...filterParams.slice(1), params.limit, start]
2139
+ : [...baseParams, params.limit, start];
2140
+ const indexRows = params.db.prepare(rowSql).all(...rowParams);
2141
+ if (indexRows.some((row) => !Number.isInteger(row.rowIndex)))
2142
+ return null;
2143
+ const rowIndices = indexRows.map((row) => row.rowIndex);
2144
+ const selectedRows = params.selectRowsByIndices
2145
+ ? await params.selectRowsByIndices(rowIndices)
2146
+ : (await listPanelQueryHandleRowsByIndices({
2147
+ db: params.db,
2148
+ panel: params.panel,
2149
+ conversationManager: params.conversationManager,
2150
+ workspaceBroker: params.workspaceBroker,
2151
+ config: params.config,
2152
+ datasetSchema: params.datasetSchema,
2153
+ rowIndices,
2154
+ })).rows;
2155
+ if (selectedRows.length !== indexRows.length)
2156
+ return null;
2157
+ const nextPosition = start + selectedRows.length;
2158
+ return {
2159
+ rows: selectedRows,
2160
+ nextCursor: nextPosition < totalCount
2161
+ ? encodeQueryHandleSortCursor({
2162
+ position: nextPosition,
2163
+ sortField: sortField.name,
2164
+ sortDirection: params.options.sortDirection,
2165
+ filterField: filterField?.name,
2166
+ filterValue: params.options.filterValue,
2167
+ })
2168
+ : null,
2169
+ };
2170
+ }
2171
+ async function readFilteredQueryHandleRowsFromFieldIndex(params) {
2172
+ if (!params.db || !params.options.filterField || params.options.filterValue === undefined)
2173
+ return null;
2174
+ const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema);
2175
+ if (!fieldSchemaKey)
2176
+ return null;
2177
+ const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
2178
+ if (!filterField)
2179
+ return null;
2180
+ const meta = params.db.prepare(`SELECT row_count as rowCount
2181
+ FROM panel_query_handle_field_index_meta
2182
+ WHERE panel_id = ?
2183
+ AND source_manifest_key = ?
2184
+ AND metadata_key = ?
2185
+ AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
2186
+ if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
2187
+ return null;
2188
+ const filter = normalizeQueryHandleIndexedValue(filterField, params.options.filterValue);
2189
+ if (!filter.hasValue)
2190
+ return { rows: [], nextCursor: null };
2191
+ const filterColumn = queryHandleFieldValueColumn(filterField);
2192
+ const filterValue = filterColumn === 'string_value'
2193
+ ? filter.stringValue
2194
+ : filterColumn === 'number_value'
2195
+ ? filter.numberValue
2196
+ : filter.booleanValue;
2197
+ const start = Math.max(0, params.position);
2198
+ const baseParams = [
2199
+ params.panel.id,
2200
+ params.sourceManifestKey,
2201
+ params.metadataKey,
2202
+ fieldSchemaKey,
2203
+ filterField.name,
2204
+ filterValue,
2205
+ ];
2206
+ const count = params.db.prepare(`SELECT COUNT(*) as totalCount
2207
+ FROM panel_query_handle_field_index_rows
2208
+ WHERE panel_id = ?
2209
+ AND source_manifest_key = ?
2210
+ AND metadata_key = ?
2211
+ AND field_schema_key = ?
2212
+ AND field_name = ?
2213
+ AND has_value = 1
2214
+ AND ${filterColumn} = ?`).get(...baseParams);
2215
+ const totalCount = count?.totalCount ?? 0;
2216
+ if (totalCount <= start)
2217
+ return { rows: [], nextCursor: null };
2218
+ const indexRows = params.db.prepare(`SELECT row_index as rowIndex
2219
+ FROM panel_query_handle_field_index_rows
2220
+ WHERE panel_id = ?
2221
+ AND source_manifest_key = ?
2222
+ AND metadata_key = ?
2223
+ AND field_schema_key = ?
2224
+ AND field_name = ?
2225
+ AND has_value = 1
2226
+ AND ${filterColumn} = ?
2227
+ ORDER BY row_index ASC
2228
+ LIMIT ? OFFSET ?`).all(...baseParams, params.limit, start);
2229
+ if (indexRows.some((row) => !Number.isInteger(row.rowIndex)))
2230
+ return null;
2231
+ const rowIndices = indexRows.map((row) => row.rowIndex);
2232
+ const selectedRows = params.selectRowsByIndices
2233
+ ? await params.selectRowsByIndices(rowIndices)
2234
+ : (await listPanelQueryHandleRowsByIndices({
2235
+ db: params.db,
2236
+ panel: params.panel,
2237
+ conversationManager: params.conversationManager,
2238
+ workspaceBroker: params.workspaceBroker,
2239
+ config: params.config,
2240
+ datasetSchema: params.datasetSchema,
2241
+ rowIndices,
2242
+ })).rows;
2243
+ if (selectedRows.length !== indexRows.length)
2244
+ return null;
2245
+ const nextPosition = start + selectedRows.length;
2246
+ return {
2247
+ rows: selectedRows,
2248
+ nextCursor: nextPosition < totalCount
2249
+ ? encodeQueryHandlePositionCursor({
2250
+ position: nextPosition,
2251
+ sourceKind: params.sourceKind ?? 'query_handle_field_index',
2252
+ filterField: filterField.name,
2253
+ filterValue: params.options.filterValue,
2254
+ })
2255
+ : null,
2256
+ };
2257
+ }
2258
+ function writeQueryHandleFieldIndexRows(params) {
2259
+ const fields = queryHandleFieldIndexFields(params.datasetSchema, params.extraFieldNames);
2260
+ const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
2261
+ if (!fieldSchemaKey || fields.length === 0)
2262
+ return;
2263
+ const now = Date.now();
2264
+ const deleteMeta = params.context.db.prepare(`DELETE FROM panel_query_handle_field_index_meta
2265
+ WHERE panel_id = ?
2266
+ AND source_manifest_key = ?
2267
+ AND field_schema_key = ?`);
2268
+ const deleteRows = params.context.db.prepare(`DELETE FROM panel_query_handle_field_index_rows
2269
+ WHERE panel_id = ?
2270
+ AND source_manifest_key = ?
2271
+ AND field_schema_key = ?`);
2272
+ const deleteAggregates = params.context.db.prepare(`DELETE FROM panel_query_handle_field_aggregate_index
2273
+ WHERE panel_id = ?
2274
+ AND source_manifest_key = ?
2275
+ AND field_schema_key = ?`);
2276
+ const insertMeta = params.context.db.prepare(`INSERT INTO panel_query_handle_field_index_meta(
2277
+ panel_id, source_manifest_key, metadata_key, field_schema_key,
2278
+ row_count, created_at, updated_at
2279
+ )
2280
+ VALUES(?, ?, ?, ?, ?, ?, ?)`);
2281
+ const insertRow = params.context.db.prepare(`INSERT INTO panel_query_handle_field_index_rows(
2282
+ panel_id, source_manifest_key, metadata_key, field_schema_key,
2283
+ field_name, row_index, field_type, has_value,
2284
+ string_value, number_value, boolean_value, created_at
2285
+ )
2286
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
2287
+ const insertAggregate = params.context.db.prepare(`INSERT INTO panel_query_handle_field_aggregate_index(
2288
+ panel_id, source_manifest_key, metadata_key, field_schema_key,
2289
+ field_name, field_type, value_count, non_empty_count,
2290
+ numeric_count, numeric_sum, numeric_min, numeric_max, created_at
2291
+ )
2292
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
2293
+ const aggregateSummaries = buildQueryHandleFieldAggregateSummaries({
2294
+ datasetSchema: params.datasetSchema,
2295
+ rows: params.rows,
2296
+ extraFieldNames: params.extraFieldNames,
2297
+ });
2298
+ params.context.db.transaction(() => {
2299
+ deleteRows.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
2300
+ deleteAggregates.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
2301
+ deleteMeta.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
2302
+ insertMeta.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, params.rows.length, now, now);
2303
+ for (const row of params.rows) {
2304
+ for (const field of fields) {
2305
+ const indexedValue = normalizeQueryHandleIndexedValue(field, getPanelFieldValue(row.fields, field.name));
2306
+ insertRow.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, field.name, row.rowIndex, field.type, indexedValue.hasValue, indexedValue.stringValue, indexedValue.numberValue, indexedValue.booleanValue, now);
2307
+ }
2308
+ }
2309
+ for (const summary of aggregateSummaries) {
2310
+ insertAggregate.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, summary.fieldName, summary.fieldType, summary.valueCount, summary.nonEmptyCount, summary.numericCount, summary.numericSum, summary.numericMin, summary.numericMax, now);
2311
+ }
2312
+ })();
2313
+ }
2314
+ function buildQueryHandleFieldAggregateSummaries(params) {
2315
+ return queryHandleFieldIndexFields(params.datasetSchema, params.extraFieldNames).map((field) => {
2316
+ const summary = {
2317
+ fieldName: field.name,
2318
+ fieldType: field.type,
2319
+ valueCount: 0,
2320
+ nonEmptyCount: 0,
2321
+ numericCount: 0,
2322
+ numericSum: 0,
2323
+ numericMin: null,
2324
+ numericMax: null,
2325
+ };
2326
+ for (const row of params.rows) {
2327
+ const indexedValue = normalizeQueryHandleIndexedValue(field, getPanelFieldValue(row.fields, field.name));
2328
+ if (!indexedValue.hasValue)
2329
+ continue;
2330
+ summary.valueCount += 1;
2331
+ if (field.type === 'string') {
2332
+ if (indexedValue.stringValue.trim() !== '')
2333
+ summary.nonEmptyCount += 1;
2334
+ }
2335
+ else {
2336
+ summary.nonEmptyCount += 1;
2337
+ }
2338
+ if (field.type === 'number') {
2339
+ summary.numericCount += 1;
2340
+ summary.numericSum += indexedValue.numberValue;
2341
+ summary.numericMin = summary.numericMin === null
2342
+ ? indexedValue.numberValue
2343
+ : Math.min(summary.numericMin, indexedValue.numberValue);
2344
+ summary.numericMax = summary.numericMax === null
2345
+ ? indexedValue.numberValue
2346
+ : Math.max(summary.numericMax, indexedValue.numberValue);
2347
+ }
2348
+ }
2349
+ return summary;
2350
+ });
2351
+ }
2352
+ function recordApiJsonlRangeCheckpoint(context, params) {
2353
+ if (!params.chunk || params.chunk.modifiedAt == null)
2354
+ return;
2355
+ recordQueryHandleCheckpoint(context, {
2356
+ sourceIndex: params.sourceIndex,
2357
+ chunk: params.chunk,
2358
+ rowIndex: params.rowIndex,
2359
+ byteOffset: params.byteOffset,
2360
+ isSourceStart: params.isSourceStart,
2361
+ });
2362
+ }
2363
+ function recordQueryHandleCheckpoint(context, params) {
2364
+ if (!context || context.sourceKeys[params.sourceIndex] == null)
2365
+ return;
2366
+ if (!params.isSourceStart
2367
+ && params.rowIndex !== 0
2368
+ && params.rowIndex % QUERY_HANDLE_INDEX_INTERVAL_ROWS !== 0) {
2369
+ return;
2370
+ }
2371
+ if (!ensureQueryHandleIndexMetadataFresh(context, params.sourceIndex, params.chunk))
2372
+ return;
2373
+ const metadata = context.metadataBySourceIndex.get(params.sourceIndex);
2374
+ if (!metadata)
2375
+ return;
2376
+ const now = Date.now();
2377
+ context.db.prepare(`INSERT OR REPLACE INTO panel_query_handle_index(
2378
+ panel_id, source_manifest_key, source_index, source_key, file_size, modified_at,
2379
+ row_index, byte_offset, created_at, updated_at
2380
+ )
2381
+ VALUES(
2382
+ ?, ?, ?, ?, ?, ?,
2383
+ ?, ?,
2384
+ COALESCE((
2385
+ SELECT created_at FROM panel_query_handle_index
2386
+ WHERE panel_id = ? AND source_manifest_key = ? AND source_index = ? AND source_key = ? AND row_index = ?
2387
+ ), ?),
2388
+ ?
2389
+ )`).run(context.panelId, context.sourceManifestKey, params.sourceIndex, context.sourceKeys[params.sourceIndex], metadata.fileSize, metadata.modifiedAt, params.rowIndex, params.byteOffset, context.panelId, context.sourceManifestKey, params.sourceIndex, context.sourceKeys[params.sourceIndex], params.rowIndex, now, now);
2390
+ }
2391
+ function apiJsonlRangeCheckpointChunk(result) {
2392
+ const totalSize = normalizeApiJsonlTotalSize(result.totalSize);
2393
+ if (totalSize == null)
2394
+ return null;
2395
+ return {
2396
+ content: result.content,
2397
+ size: totalSize,
2398
+ modifiedAt: apiJsonlValidatorFingerprint(result),
2399
+ offset: result.rangeStart,
2400
+ hasMore: result.hasMore,
2401
+ };
2402
+ }
2403
+ function normalizeApiJsonlTotalSize(totalSize) {
2404
+ return typeof totalSize === 'number' && Number.isSafeInteger(totalSize) && totalSize >= 0
2405
+ ? totalSize
2406
+ : null;
2407
+ }
2408
+ function apiJsonlValidatorFingerprint(source) {
2409
+ const etag = typeof source.etag === 'string' ? source.etag.trim() : '';
2410
+ if (etag)
2411
+ return hashStringToSafeInteger(`etag:${etag}`);
2412
+ const lastModified = typeof source.lastModified === 'string' ? source.lastModified.trim() : '';
2413
+ if (lastModified)
2414
+ return hashStringToSafeInteger(`last-modified:${lastModified}`);
2415
+ return null;
2416
+ }
2417
+ function hashStringToSafeInteger(value) {
2418
+ return createHash('sha256').update(value).digest().readUIntBE(0, 6);
2419
+ }
2420
+ function ensureQueryHandleIndexMetadataFresh(context, sourceIndex, chunk) {
2421
+ const existingMetadata = context.metadataBySourceIndex.get(sourceIndex);
2422
+ if (existingMetadata)
2423
+ return true;
2424
+ const sourceKey = context.sourceKeys[sourceIndex];
2425
+ if (sourceKey == null)
2426
+ return false;
2427
+ const fileSize = chunk.size;
2428
+ const modifiedAt = normalizeModifiedAt(chunk.modifiedAt);
2429
+ context.db.prepare(`DELETE FROM panel_query_handle_index
2430
+ WHERE panel_id = ?
2431
+ AND source_manifest_key = ?
2432
+ AND source_index = ?
2433
+ AND source_key != ?`).run(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey);
2434
+ const mismatch = modifiedAt == null
2435
+ ? context.db.prepare(`SELECT 1
2436
+ FROM panel_query_handle_index
2437
+ WHERE panel_id = ?
2438
+ AND source_manifest_key = ?
2439
+ AND source_index = ?
2440
+ AND source_key = ?
2441
+ AND file_size != ?
2442
+ LIMIT 1`).get(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey, fileSize)
2443
+ : context.db.prepare(`SELECT 1
2444
+ FROM panel_query_handle_index
2445
+ WHERE panel_id = ?
2446
+ AND source_manifest_key = ?
2447
+ AND source_index = ?
2448
+ AND source_key = ?
2449
+ AND (file_size != ? OR modified_at IS NULL OR modified_at != ?)
2450
+ LIMIT 1`).get(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey, fileSize, modifiedAt);
2451
+ if (mismatch) {
2452
+ context.db.prepare(`DELETE FROM panel_query_handle_index
2453
+ WHERE panel_id = ?
2454
+ AND source_manifest_key = ?
2455
+ AND source_index = ?`).run(context.panelId, context.sourceManifestKey, sourceIndex);
2456
+ }
2457
+ context.metadataBySourceIndex.set(sourceIndex, { fileSize, modifiedAt });
2458
+ return true;
2459
+ }
2460
+ function normalizeModifiedAt(modifiedAt) {
2461
+ return typeof modifiedAt === 'number' && Number.isFinite(modifiedAt)
2462
+ ? Math.trunc(modifiedAt)
2463
+ : null;
2464
+ }
2465
+ async function readQueryHandleChunk(params) {
2466
+ try {
2467
+ if (params.dataSource.kind === 'workspace_relative') {
2468
+ return await params.workspaceBroker.readFile(params.nodeId, params.workspaceRoot, params.dataSource.relativePath, {
2469
+ scaffold: false,
2470
+ offset: params.offset,
2471
+ limit: params.limit,
2472
+ });
2473
+ }
2474
+ const exact = await params.workspaceBroker.readExactPath(params.nodeId, params.dataSource.absolutePath, {
2475
+ allowedRoot: params.dataSource.allowedRoot,
2476
+ offset: params.offset,
2477
+ limit: params.limit,
2478
+ });
2479
+ if (exact.kind !== 'file') {
2480
+ throw new PanelQueryHandleError('query_handle workspace_jsonl path must point to a file.', 400);
2481
+ }
2482
+ return exact;
2483
+ }
2484
+ catch (error) {
2485
+ if (error instanceof PanelQueryHandleError)
2486
+ throw error;
2487
+ const message = error instanceof Error ? error.message : String(error);
2488
+ if (message.includes('not_found') || message.includes('ENOENT') || message.includes('No such file')) {
2489
+ const path = params.dataSource.kind === 'workspace_relative'
2490
+ ? params.dataSource.relativePath
2491
+ : params.dataSource.absolutePath;
2492
+ throw new PanelQueryHandleError(`query_handle source not found: ${path}`, 404);
2493
+ }
2494
+ throw error;
2495
+ }
2496
+ }
2497
+ function parseCompleteJsonlLines(content, hasMore) {
2498
+ if (!hasMore) {
2499
+ return { completeText: content, lines: splitJsonlLineSegments(content) };
2500
+ }
2501
+ const lastNewline = content.lastIndexOf('\n');
2502
+ if (lastNewline < 0)
2503
+ return { completeText: '', lines: [] };
2504
+ const completeText = content.slice(0, lastNewline + 1);
2505
+ return {
2506
+ completeText,
2507
+ lines: splitJsonlLineSegments(completeText),
2508
+ };
2509
+ }
2510
+ function splitJsonlLineSegments(text) {
2511
+ const segments = [];
2512
+ let start = 0;
2513
+ while (start < text.length) {
2514
+ const newlineIndex = text.indexOf('\n', start);
2515
+ const end = newlineIndex >= 0 ? newlineIndex + 1 : text.length;
2516
+ const segment = text.slice(start, end);
2517
+ const line = segment.endsWith('\n') ? segment.slice(0, -1) : segment;
2518
+ segments.push({
2519
+ text: line.endsWith('\r') ? line.slice(0, -1) : line,
2520
+ byteLength: Buffer.byteLength(segment, 'utf8'),
2521
+ });
2522
+ start = end;
2523
+ }
2524
+ return segments;
2525
+ }
2526
+ function parseJsonlRow(line, rowIndex) {
2527
+ try {
2528
+ const parsed = JSON.parse(line);
2529
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
2530
+ throw new Error('row must be an object');
2531
+ }
2532
+ return parsed;
2533
+ }
2534
+ catch (error) {
2535
+ throw new PanelQueryHandleError(`Invalid dynamic panel JSONL row at index ${rowIndex}: ${String(error?.message ?? error)}`, 400);
2536
+ }
2537
+ }
2538
+ function buildPanelRowFromQueryRow(rawRow, rowIndex, datasetSchema) {
2539
+ const fields = {};
2540
+ for (const field of datasetSchema?.fields ?? []) {
2541
+ const value = getPanelFieldValue(rawRow, field.name);
2542
+ if (!isFieldValueValid(field, value)) {
2543
+ throw new PanelQueryHandleError(`Invalid dynamic panel row ${rowIndex}: field "${field.name}" must be a ${field.type}.`, 400);
2544
+ }
2545
+ fields[field.name] = value;
2546
+ }
2547
+ const media = {};
2548
+ for (const slot of datasetSchema?.mediaSlots ?? []) {
2549
+ const slotValue = rawRow[slot.name];
2550
+ if (typeof slotValue === 'string') {
2551
+ media[slot.name] = { kind: slot.kind, value: slotValue };
2552
+ }
2553
+ }
2554
+ const rowId = typeof rawRow.row_id === 'string' && rawRow.row_id.trim()
2555
+ ? rawRow.row_id.trim()
2556
+ : typeof rawRow.rowId === 'string' && rawRow.rowId.trim()
2557
+ ? rawRow.rowId.trim()
2558
+ : null;
2559
+ return {
2560
+ rowIndex,
2561
+ rowId,
2562
+ fields,
2563
+ media,
2564
+ };
2565
+ }
2566
+ function isFieldValueValid(field, value) {
2567
+ if (value === undefined || value === null)
2568
+ return true;
2569
+ if (field.type === 'number')
2570
+ return typeof value === 'number' && Number.isFinite(value);
2571
+ return typeof value === field.type;
2572
+ }
2573
+ function matchesFilter(fields, filterField, filterValue) {
2574
+ if (!filterField || filterValue === undefined)
2575
+ return true;
2576
+ return getPanelFieldValue(fields, filterField) === filterValue;
2577
+ }
2578
+ function parseQueryHandleCursor(cursor) {
2579
+ if (!cursor)
2580
+ return { sourceIndex: 0, offset: 0, rowIndex: 0 };
2581
+ try {
2582
+ const parsed = decodeCursorObject(cursor);
2583
+ const sourceIndex = typeof parsed.sourceIndex === 'number' && Number.isInteger(parsed.sourceIndex) && parsed.sourceIndex >= 0
2584
+ ? parsed.sourceIndex
2585
+ : 0;
2586
+ const offset = typeof parsed.offset === 'number' && Number.isInteger(parsed.offset) && parsed.offset >= 0
2587
+ ? parsed.offset
2588
+ : 0;
2589
+ const rowIndex = typeof parsed.rowIndex === 'number' && Number.isInteger(parsed.rowIndex) && parsed.rowIndex >= 0
2590
+ ? parsed.rowIndex
2591
+ : 0;
2592
+ return { sourceIndex, offset, rowIndex };
2593
+ }
2594
+ catch {
2595
+ return { sourceIndex: 0, offset: 0, rowIndex: 0 };
2596
+ }
2597
+ }
2598
+ function encodeQueryHandleCursor(cursor) {
2599
+ return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
2600
+ }
2601
+ function parseQueryHandleSortCursor(cursor, request) {
2602
+ if (!cursor)
2603
+ return { position: 0 };
2604
+ try {
2605
+ const parsed = decodeCursorObject(cursor);
2606
+ const sameSort = parsed.sortField === request.sortField
2607
+ && parsed.sortDirection === request.sortDirection
2608
+ && parsed.filterField === request.filterField
2609
+ && parsed.filterValue === request.filterValue;
2610
+ if (!sameSort)
2611
+ return { position: 0 };
2612
+ const position = typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0
2613
+ ? parsed.position
2614
+ : 0;
2615
+ return { position };
2616
+ }
2617
+ catch {
2618
+ return { position: 0 };
2619
+ }
2620
+ }
2621
+ function encodeQueryHandleSortCursor(cursor) {
2622
+ return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
2623
+ }
2624
+ function parseQueryHandlePositionCursor(cursor, request) {
2625
+ if (!cursor)
2626
+ return { position: 0 };
2627
+ try {
2628
+ const parsed = decodeCursorObject(cursor);
2629
+ if (request.sourceKind === 'api_jsonl'
2630
+ && parsed.sourceKind === 'api_jsonl_range'
2631
+ && typeof parsed.rowIndex === 'number'
2632
+ && Number.isInteger(parsed.rowIndex)
2633
+ && parsed.rowIndex >= 0) {
2634
+ return { position: parsed.rowIndex };
2635
+ }
2636
+ const sameRequest = parsed.sourceKind === request.sourceKind
2637
+ && parsed.filterField === request.filterField
2638
+ && parsed.filterValue === request.filterValue;
2639
+ if (!sameRequest)
2640
+ return { position: 0 };
2641
+ const position = typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0
2642
+ ? parsed.position
2643
+ : 0;
2644
+ return { position };
2645
+ }
2646
+ catch {
2647
+ return { position: 0 };
2648
+ }
2649
+ }
2650
+ function encodeQueryHandlePositionCursor(cursor) {
2651
+ return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
2652
+ }
2653
+ function parseApiJsonlRangeCursor(cursor) {
2654
+ if (!cursor)
2655
+ return defaultApiJsonlRangeCursor();
2656
+ try {
2657
+ const parsed = decodeCursorObject(cursor);
2658
+ if (parsed.sourceKind !== 'api_jsonl_range')
2659
+ return defaultApiJsonlRangeCursor();
2660
+ return {
2661
+ sourceKind: 'api_jsonl_range',
2662
+ sourceIndex: typeof parsed.sourceIndex === 'number' && Number.isInteger(parsed.sourceIndex) && parsed.sourceIndex >= 0
2663
+ ? parsed.sourceIndex
2664
+ : 0,
2665
+ offset: typeof parsed.offset === 'number' && Number.isInteger(parsed.offset) && parsed.offset >= 0
2666
+ ? parsed.offset
2667
+ : 0,
2668
+ rowIndex: typeof parsed.rowIndex === 'number' && Number.isInteger(parsed.rowIndex) && parsed.rowIndex >= 0
2669
+ ? parsed.rowIndex
2670
+ : 0,
2671
+ ...(typeof parsed.url === 'string' && parsed.url ? { url: parsed.url } : {}),
2672
+ etag: typeof parsed.etag === 'string' && parsed.etag ? parsed.etag : null,
2673
+ lastModified: typeof parsed.lastModified === 'string' && parsed.lastModified ? parsed.lastModified : null,
2674
+ totalSize: typeof parsed.totalSize === 'number' && Number.isSafeInteger(parsed.totalSize) && parsed.totalSize >= 0
2675
+ ? parsed.totalSize
2676
+ : null,
2677
+ };
2678
+ }
2679
+ catch {
2680
+ return defaultApiJsonlRangeCursor();
2681
+ }
2682
+ }
2683
+ function defaultApiJsonlRangeCursor() {
2684
+ return {
2685
+ sourceKind: 'api_jsonl_range',
2686
+ sourceIndex: 0,
2687
+ offset: 0,
2688
+ rowIndex: 0,
2689
+ etag: null,
2690
+ lastModified: null,
2691
+ totalSize: null,
2692
+ };
2693
+ }
2694
+ function encodeApiJsonlRangeCursor(cursor) {
2695
+ return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
2696
+ }
2697
+ function isPositionCursor(cursor) {
2698
+ if (!cursor)
2699
+ return false;
2700
+ try {
2701
+ const parsed = decodeCursorObject(cursor);
2702
+ return typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0;
2703
+ }
2704
+ catch {
2705
+ return false;
2706
+ }
2707
+ }
2708
+ function withApiJsonlIfRange(cursor) {
2709
+ if (typeof cursor.etag === 'string' && cursor.etag.trim())
2710
+ return { ifRange: cursor.etag.trim() };
2711
+ if (typeof cursor.lastModified === 'string' && cursor.lastModified.trim())
2712
+ return { ifRange: cursor.lastModified.trim() };
2713
+ return {};
2714
+ }
2715
+ function decodeCursorObject(cursor) {
2716
+ const parsed = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8'));
2717
+ return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
2718
+ ? parsed
2719
+ : {};
2720
+ }
2721
+ function comparePanelRowsBySortField(left, right, sortField, sortDirection, sortFieldType) {
2722
+ const leftValue = getPanelFieldValue(left.fields, sortField);
2723
+ const rightValue = getPanelFieldValue(right.fields, sortField);
2724
+ const primary = compareSortValues(leftValue, rightValue, sortFieldType);
2725
+ if (primary !== 0) {
2726
+ return sortDirection === 'asc' ? primary : -primary;
2727
+ }
2728
+ const tieBreaker = left.rowIndex - right.rowIndex;
2729
+ return sortDirection === 'asc' ? tieBreaker : -tieBreaker;
2730
+ }
2731
+ function compareSortValues(left, right, sortFieldType) {
2732
+ if (sortFieldType === 'number') {
2733
+ return normalizedSortNumber(left) - normalizedSortNumber(right);
2734
+ }
2735
+ if (sortFieldType === 'boolean') {
2736
+ return Number(left === true) - Number(right === true);
2737
+ }
2738
+ return normalizedSortString(left).localeCompare(normalizedSortString(right));
2739
+ }
2740
+ function normalizedSortNumber(value) {
2741
+ return typeof value === 'number' && Number.isFinite(value) ? value : 0;
2742
+ }
2743
+ function normalizedSortString(value) {
2744
+ if (typeof value === 'string')
2745
+ return value;
2746
+ if (typeof value === 'number' || typeof value === 'boolean')
2747
+ return String(value);
2748
+ return '';
2749
+ }