@bbigbang/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config.js +380 -0
- package/dist/execution/executionDispatcher.js +3810 -0
- package/dist/main.js +90 -0
- package/dist/nodeEventHistory.js +206 -0
- package/dist/scheduler/dreamLogic.js +50 -0
- package/dist/scheduler/dreamScheduler.js +65 -0
- package/dist/services/agentFileAccessService.js +1913 -0
- package/dist/services/agentRuntimeCleanupBroker.js +62 -0
- package/dist/services/agentSkillsBroker.js +118 -0
- package/dist/services/agentSkillsService.js +83 -0
- package/dist/services/agentWorkspaceBroker.js +937 -0
- package/dist/services/agentWorkspaceService.js +70 -0
- package/dist/services/appVersion.js +14 -0
- package/dist/services/auth.js +586 -0
- package/dist/services/claudeControlBroker.js +154 -0
- package/dist/services/claudeTranscriptBroker.js +100 -0
- package/dist/services/claudeTranscriptService.js +359 -0
- package/dist/services/codexAppServerBroker.js +155 -0
- package/dist/services/codexTranscriptBroker.js +98 -0
- package/dist/services/codexTranscriptService.js +961 -0
- package/dist/services/droidMissionBroker.js +124 -0
- package/dist/services/droidMissionImporter.js +630 -0
- package/dist/services/droidModelOptions.js +165 -0
- package/dist/services/hubServerRegistrationService.js +268 -0
- package/dist/services/libraryManifest.js +43 -0
- package/dist/services/libraryScaffold.js +26 -0
- package/dist/services/libraryService.js +2263 -0
- package/dist/services/memoryService.js +386 -0
- package/dist/services/missionEvidence.js +377 -0
- package/dist/services/missionService.js +2361 -0
- package/dist/services/missionTrace.js +158 -0
- package/dist/services/nativeMissionBriefParser.js +120 -0
- package/dist/services/nativeMissionOrchestrator.js +2045 -0
- package/dist/services/nativeMissionReportGenerator.js +227 -0
- package/dist/services/nativeMissionValidationRunner.js +452 -0
- package/dist/services/nativeMissionWorkerBroker.js +190 -0
- package/dist/services/nodeRegistry.js +34 -0
- package/dist/services/nodeStateReconciler.js +97 -0
- package/dist/services/panelMediaScanner.js +119 -0
- package/dist/services/persistentRuntimeJsonlClient.js +153 -0
- package/dist/services/platformAgentPolicy.js +180 -0
- package/dist/services/platformAgentService.js +2041 -0
- package/dist/services/projectAccessResolver.js +93 -0
- package/dist/services/projectService.js +392 -0
- package/dist/services/resourceSpaceService.js +140 -0
- package/dist/services/scenarioRuntimeService.js +1130 -0
- package/dist/services/suggestedPlannerService.js +868 -0
- package/dist/services/workbenchGitBroker.js +161 -0
- package/dist/services/workbenchGitService.js +69 -0
- package/dist/services/workbenchInspectBroker.js +65 -0
- package/dist/services/workbenchNodePathService.js +79 -0
- package/dist/services/workbenchRegistryService.js +240 -0
- package/dist/services/workbenchRootService.js +181 -0
- package/dist/services/workbenchTerminalBroker.js +378 -0
- package/dist/services/workspaceRunOwnership.js +60 -0
- package/dist/services/workspaceScaffold.js +105 -0
- package/dist/services/workspaceSessionRuntimeService.js +576 -0
- package/dist/services/workspaceSessionService.js +245 -0
- package/dist/services/workspaceToolActionRunner.js +1582 -0
- package/dist/services/workspaceToolErrors.js +10 -0
- package/dist/services/workspaceToolExecutionUtils.js +895 -0
- package/dist/services/workspaceToolLatestStateProjector.js +91 -0
- package/dist/services/workspaceToolManifest.js +572 -0
- package/dist/services/workspaceToolMutationQueue.js +43 -0
- package/dist/services/workspaceToolPanelProjection.js +460 -0
- package/dist/services/workspaceToolPromotion.js +255 -0
- package/dist/services/workspaceToolPromotionState.js +224 -0
- package/dist/services/workspaceToolPublishDiagnostics.js +189 -0
- package/dist/services/workspaceToolPublishIdentityResolver.js +146 -0
- package/dist/services/workspaceToolReadModel.js +378 -0
- package/dist/services/workspaceToolRunLedger.js +239 -0
- package/dist/services/workspaceToolService.js +3067 -0
- package/dist/services/workspaceToolSnapshotPanelSync.js +293 -0
- package/dist/services/workspaceToolTerminalLifecycle.js +283 -0
- package/dist/services/workspaceToolTypes.js +1 -0
- package/dist/services/workspaceToolUploadMaterializer.js +228 -0
- package/dist/web/actionCardRoutes.js +129 -0
- package/dist/web/actionCards.js +469 -0
- package/dist/web/activationContext.js +684 -0
- package/dist/web/agentChannelGuards.js +48 -0
- package/dist/web/agentMentionCooldowns.js +32 -0
- package/dist/web/agentReminders.js +1668 -0
- package/dist/web/agentRuntimePresence.js +197 -0
- package/dist/web/agentSelfState.js +494 -0
- package/dist/web/agentTaskLinks.js +26 -0
- package/dist/web/agentVisibility.js +79 -0
- package/dist/web/assets.js +95 -0
- package/dist/web/channelActivationPrompt.js +395 -0
- package/dist/web/channelMemoryNotes.js +127 -0
- package/dist/web/channelMentions.js +10 -0
- package/dist/web/channelMessageSequences.js +19 -0
- package/dist/web/channelSubscriptions.js +26 -0
- package/dist/web/clearedTaskRoots.js +10 -0
- package/dist/web/collaborationPromptGuidance.js +36 -0
- package/dist/web/collaborationSurfaceState.js +140 -0
- package/dist/web/contextBundleRanking.js +154 -0
- package/dist/web/contextBundleResolver.js +488 -0
- package/dist/web/conversationBuiltinSkillRoots.js +50 -0
- package/dist/web/conversationControls.js +232 -0
- package/dist/web/conversationHandoffs.js +612 -0
- package/dist/web/conversationManager.js +2511 -0
- package/dist/web/conversationSummaries.js +876 -0
- package/dist/web/conversationSurfaceKinds.js +17 -0
- package/dist/web/conversationTargets.js +173 -0
- package/dist/web/directActivationPrompt.js +122 -0
- package/dist/web/directReplyTargets.js +69 -0
- package/dist/web/directThreadResolver.js +129 -0
- package/dist/web/dmTaskHandoffPrompt.js +120 -0
- package/dist/web/dmTaskThreadStatusProjection.js +229 -0
- package/dist/web/ftsQuery.js +33 -0
- package/dist/web/internalAgentRouter.js +11341 -0
- package/dist/web/libraryCuratorScheduler.js +58 -0
- package/dist/web/libraryDocumentPromptGuidance.js +8 -0
- package/dist/web/messageCheckpoints.js +19 -0
- package/dist/web/nodeWsHandler.js +2495 -0
- package/dist/web/notificationRounds.js +1061 -0
- package/dist/web/panelActionMessages.js +108 -0
- package/dist/web/panelActivationPrompt.js +18 -0
- package/dist/web/panelAudit.js +273 -0
- package/dist/web/panelLifecycle.js +222 -0
- package/dist/web/panelMediaPolicy.js +43 -0
- package/dist/web/panelPathPolicy.js +63 -0
- package/dist/web/panelPreviews.js +175 -0
- package/dist/web/panelQueryHandles.js +2749 -0
- package/dist/web/panelRoutes.js +2147 -0
- package/dist/web/panels.js +904 -0
- package/dist/web/peerInboxAggregates.js +1247 -0
- package/dist/web/planApprovalState.js +92 -0
- package/dist/web/platformAgentScheduler.js +66 -0
- package/dist/web/proactiveOpportunities.js +452 -0
- package/dist/web/promptContextSections.js +242 -0
- package/dist/web/promptHistorySanitizer.js +26 -0
- package/dist/web/promptSlashCommands.js +158 -0
- package/dist/web/rollingConversationSummary.js +453 -0
- package/dist/web/routeHelpers.js +11 -0
- package/dist/web/routes/handoff.js +288 -0
- package/dist/web/routes/history.js +345 -0
- package/dist/web/routes/memory.js +258 -0
- package/dist/web/routes/selfState.js +171 -0
- package/dist/web/routes/workspace.js +154 -0
- package/dist/web/runSurfaceWatermarks.js +431 -0
- package/dist/web/runtimeCapabilities.js +48 -0
- package/dist/web/sameAgentHandoffs.js +494 -0
- package/dist/web/server.js +15567 -0
- package/dist/web/sharedCollaborationCapsules.js +163 -0
- package/dist/web/soloSessionRelay.js +42 -0
- package/dist/web/soloWsHandler.js +138 -0
- package/dist/web/suggestedPlannerScheduler.js +56 -0
- package/dist/web/surfaceActivationPolicy.js +108 -0
- package/dist/web/surfaceCollaborators.js +61 -0
- package/dist/web/surfaceSystemStatus.js +263 -0
- package/dist/web/targetParticipants.js +77 -0
- package/dist/web/taskEvents.js +49 -0
- package/dist/web/taskLifecycleMessages.js +165 -0
- package/dist/web/taskLoops.js +732 -0
- package/dist/web/taskMemoryNotes.js +224 -0
- package/dist/web/taskNumbers.js +16 -0
- package/dist/web/taskOwnerGuards.js +49 -0
- package/dist/web/taskParticipantResolver.js +42 -0
- package/dist/web/taskParticipants.js +97 -0
- package/dist/web/taskSourceDetails.js +20 -0
- package/dist/web/taskStateViews.js +210 -0
- package/dist/web/taskStatusTransitions.js +9 -0
- package/dist/web/taskThreadFollowups.js +599 -0
- package/dist/web/taskThreadRuntimeClosure.js +685 -0
- package/dist/web/taskUpdateDelivery.js +104 -0
- package/dist/web/threadReplyContentHeuristics.js +30 -0
- package/dist/web/threadRoots.js +61 -0
- package/dist/web/threadTaskBindings.js +365 -0
- package/dist/web/uiPanelPromptGuidance.js +27 -0
- package/dist/web/workspaceMemoryHints.js +143 -0
- package/dist/web/workspaceToolPromptGuidance.js +30 -0
- package/dist/web/wsHandler.js +397 -0
- package/dist/web/wsSink.js +116 -0
- package/package.json +54 -0
|
@@ -0,0 +1,2749 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { validatePanelApiJsonlUrl, validatePanelApiJsonlUrlList } from '@bbigbang/protocol';
|
|
4
|
+
import { resolvePanelDatasetPath } from './panelPathPolicy.js';
|
|
5
|
+
import { aggregatePanelFieldRecords, getPanelFieldValue, } from './panels.js';
|
|
6
|
+
const QUERY_HANDLE_WORKSPACE_JSONL_PREFIX = 'workspace_jsonl:';
|
|
7
|
+
const QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX = 'workspace_jsonl_list:';
|
|
8
|
+
const QUERY_HANDLE_MAX_SOURCE_PATHS = 16;
|
|
9
|
+
const QUERY_HANDLE_READ_CHUNK_BYTES = 256 * 1024;
|
|
10
|
+
const QUERY_HANDLE_MAX_CHUNKS_PER_PAGE = 64;
|
|
11
|
+
const QUERY_HANDLE_MAX_CHUNKS_PER_SORT = 512;
|
|
12
|
+
const QUERY_HANDLE_INDEX_INTERVAL_ROWS = 200;
|
|
13
|
+
const API_JSONL_MAX_BYTES = 2 * 1024 * 1024;
|
|
14
|
+
const API_JSONL_MAX_ROWS = 10_000;
|
|
15
|
+
const API_JSONL_RANGE_CHUNK_BYTES = 256 * 1024;
|
|
16
|
+
const API_JSONL_MAX_RANGE_CHUNKS_PER_PAGE = 64;
|
|
17
|
+
const API_JSONL_MAX_RANGE_CHUNKS_PER_INDEX = 512;
|
|
18
|
+
/** TTL for panel_api_jsonl_response_cache entries (7 days). */
|
|
19
|
+
export const PANEL_API_JSONL_RESPONSE_CACHE_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
20
|
+
/** Per-panel size budget for panel_api_jsonl_response_cache (64 MiB). */
|
|
21
|
+
export const PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL = 64 * 1024 * 1024;
|
|
22
|
+
export class PanelQueryHandleError extends Error {
|
|
23
|
+
statusCode;
|
|
24
|
+
constructor(message, statusCode = 400) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.name = 'PanelQueryHandleError';
|
|
27
|
+
this.statusCode = statusCode;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
export function isPanelQueryHandleSource(panel) {
|
|
31
|
+
return panel.datasetSource?.kind === 'query_handle' || panel.datasetSource?.kind === 'api_jsonl';
|
|
32
|
+
}
|
|
33
|
+
export function parsePanelQueryHandle(handle) {
|
|
34
|
+
const trimmed = handle.trim();
|
|
35
|
+
if (trimmed.startsWith(QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX)) {
|
|
36
|
+
const rawList = trimmed.slice(QUERY_HANDLE_WORKSPACE_JSONL_LIST_PREFIX.length).trim();
|
|
37
|
+
let parsed;
|
|
38
|
+
try {
|
|
39
|
+
parsed = JSON.parse(rawList);
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
throw new PanelQueryHandleError('query_handle workspace_jsonl_list must be a JSON array of paths.', 400);
|
|
43
|
+
}
|
|
44
|
+
if (!Array.isArray(parsed) || parsed.length < 1 || parsed.length > QUERY_HANDLE_MAX_SOURCE_PATHS) {
|
|
45
|
+
throw new PanelQueryHandleError(`query_handle workspace_jsonl_list must contain 1-${QUERY_HANDLE_MAX_SOURCE_PATHS} paths.`, 400);
|
|
46
|
+
}
|
|
47
|
+
return {
|
|
48
|
+
paths: parsed.map((entry, index) => normalizeQueryHandleWorkspaceJsonlPath(entry, `query_handle workspace_jsonl_list path ${index + 1}`)),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
if (!trimmed.startsWith(QUERY_HANDLE_WORKSPACE_JSONL_PREFIX)) {
|
|
52
|
+
throw new PanelQueryHandleError('Unsupported query_handle. Use workspace_jsonl:<path> or workspace_jsonl_list:["<path>",...] for query_handle panels.', 400);
|
|
53
|
+
}
|
|
54
|
+
return {
|
|
55
|
+
paths: [normalizeQueryHandleWorkspaceJsonlPath(trimmed.slice(QUERY_HANDLE_WORKSPACE_JSONL_PREFIX.length), 'query_handle workspace_jsonl path')],
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
function normalizeQueryHandleWorkspaceJsonlPath(rawPath, label) {
|
|
59
|
+
if (typeof rawPath !== 'string') {
|
|
60
|
+
throw new PanelQueryHandleError(`${label} must be a string path.`, 400);
|
|
61
|
+
}
|
|
62
|
+
const sourcePath = rawPath.trim();
|
|
63
|
+
if (!sourcePath
|
|
64
|
+
|| sourcePath.includes('\0')
|
|
65
|
+
|| /[\r\n]/u.test(sourcePath)) {
|
|
66
|
+
throw new PanelQueryHandleError(`${label} must be a non-empty path without control characters.`, 400);
|
|
67
|
+
}
|
|
68
|
+
if (!path.isAbsolute(sourcePath) && sourcePath.split('/').includes('..')) {
|
|
69
|
+
throw new PanelQueryHandleError(`${label} must not contain relative traversal.`, 400);
|
|
70
|
+
}
|
|
71
|
+
return sourcePath;
|
|
72
|
+
}
|
|
73
|
+
export function parsePanelApiJsonlUrl(url, options = {}) {
|
|
74
|
+
const validation = validatePanelApiJsonlUrl(url, options);
|
|
75
|
+
if (!validation.ok) {
|
|
76
|
+
throw new PanelQueryHandleError(validation.reason, 400);
|
|
77
|
+
}
|
|
78
|
+
return { url: validation.url };
|
|
79
|
+
}
|
|
80
|
+
export function parsePanelApiJsonlUrls(source, options = {}) {
|
|
81
|
+
const validation = validatePanelApiJsonlUrlList(source, options);
|
|
82
|
+
if (!validation.ok) {
|
|
83
|
+
throw new PanelQueryHandleError(validation.reason, 400);
|
|
84
|
+
}
|
|
85
|
+
return { urls: validation.urls, authProfile: validation.authProfile };
|
|
86
|
+
}
|
|
87
|
+
export async function listPanelQueryHandleRows(params) {
|
|
88
|
+
const source = params.panel.datasetSource;
|
|
89
|
+
if (source?.kind !== 'query_handle' && source?.kind !== 'api_jsonl') {
|
|
90
|
+
throw new PanelQueryHandleError('Panel does not use a dynamic dataset source.', 400);
|
|
91
|
+
}
|
|
92
|
+
const agent = params.conversationManager.getAgent(params.panel.agentId);
|
|
93
|
+
if (!agent?.nodeId) {
|
|
94
|
+
throw new PanelQueryHandleError('dynamic panel agent node is unavailable.', 409);
|
|
95
|
+
}
|
|
96
|
+
if (!params.workspaceBroker) {
|
|
97
|
+
throw new PanelQueryHandleError('Workspace broker unavailable.', 503);
|
|
98
|
+
}
|
|
99
|
+
if (source.kind === 'api_jsonl') {
|
|
100
|
+
const parsed = parsePanelApiJsonlUrls(source, {
|
|
101
|
+
allowedOrigins: params.apiJsonlAllowedOrigins,
|
|
102
|
+
allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
103
|
+
allowedAuthProfiles: params.apiJsonlAuthProfiles,
|
|
104
|
+
});
|
|
105
|
+
return listPanelApiJsonlRows({
|
|
106
|
+
db: params.db,
|
|
107
|
+
panel: params.panel,
|
|
108
|
+
conversationManager: params.conversationManager,
|
|
109
|
+
config: params.config,
|
|
110
|
+
apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
|
|
111
|
+
apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
112
|
+
apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
|
|
113
|
+
sourceUrls: parsed.urls,
|
|
114
|
+
authProfile: parsed.authProfile,
|
|
115
|
+
workspaceBroker: params.workspaceBroker,
|
|
116
|
+
nodeId: agent.nodeId,
|
|
117
|
+
datasetSchema: params.datasetSchema,
|
|
118
|
+
options: params.options,
|
|
119
|
+
limit: Math.min(Math.max(1, params.options.limit), 200),
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
if (!agent.workspacePath) {
|
|
123
|
+
throw new PanelQueryHandleError('query_handle panel agent workspace is unavailable.', 409);
|
|
124
|
+
}
|
|
125
|
+
const dataSources = resolveQueryHandleDataSources({
|
|
126
|
+
handle: source.handle,
|
|
127
|
+
workspaceRoot: agent.workspacePath,
|
|
128
|
+
config: params.config,
|
|
129
|
+
});
|
|
130
|
+
const limit = Math.min(Math.max(1, params.options.limit), 200);
|
|
131
|
+
if (params.options.sortField && params.options.sortDirection) {
|
|
132
|
+
return listSortedPanelQueryHandleRows({
|
|
133
|
+
db: params.db,
|
|
134
|
+
panel: params.panel,
|
|
135
|
+
conversationManager: params.conversationManager,
|
|
136
|
+
config: params.config,
|
|
137
|
+
indexContext: createQueryHandleIndexContext(params.db, params.panel.id, dataSources),
|
|
138
|
+
workspaceBroker: params.workspaceBroker,
|
|
139
|
+
nodeId: agent.nodeId,
|
|
140
|
+
workspaceRoot: agent.workspacePath,
|
|
141
|
+
dataSources,
|
|
142
|
+
datasetSchema: params.datasetSchema,
|
|
143
|
+
options: params.options,
|
|
144
|
+
limit,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
return listUnsortedPanelQueryHandleRows({
|
|
148
|
+
db: params.db,
|
|
149
|
+
panel: params.panel,
|
|
150
|
+
conversationManager: params.conversationManager,
|
|
151
|
+
config: params.config,
|
|
152
|
+
indexContext: createQueryHandleIndexContext(params.db, params.panel.id, dataSources),
|
|
153
|
+
workspaceBroker: params.workspaceBroker,
|
|
154
|
+
nodeId: agent.nodeId,
|
|
155
|
+
workspaceRoot: agent.workspacePath,
|
|
156
|
+
dataSources,
|
|
157
|
+
datasetSchema: params.datasetSchema,
|
|
158
|
+
options: params.options,
|
|
159
|
+
limit,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
export async function aggregatePanelQueryHandleRows(params) {
|
|
163
|
+
const source = params.panel.datasetSource;
|
|
164
|
+
if (source?.kind !== 'query_handle' && source?.kind !== 'api_jsonl') {
|
|
165
|
+
throw new PanelQueryHandleError('Panel does not use a dynamic dataset source.', 400);
|
|
166
|
+
}
|
|
167
|
+
const agent = params.conversationManager.getAgent(params.panel.agentId);
|
|
168
|
+
if (!agent?.nodeId) {
|
|
169
|
+
throw new PanelQueryHandleError('dynamic panel agent node is unavailable.', 409);
|
|
170
|
+
}
|
|
171
|
+
if (!params.workspaceBroker) {
|
|
172
|
+
throw new PanelQueryHandleError('Workspace broker unavailable.', 503);
|
|
173
|
+
}
|
|
174
|
+
if (source.kind === 'api_jsonl') {
|
|
175
|
+
const parsed = parsePanelApiJsonlUrls(source, {
|
|
176
|
+
allowedOrigins: params.apiJsonlAllowedOrigins,
|
|
177
|
+
allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
178
|
+
allowedAuthProfiles: params.apiJsonlAuthProfiles,
|
|
179
|
+
});
|
|
180
|
+
const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, parsed.urls, parsed.authProfile);
|
|
181
|
+
const sourceManifestKey = indexContext?.sourceManifestKey ?? apiJsonlSourceManifestKey(parsed.urls, parsed.authProfile);
|
|
182
|
+
const extraFieldNames = aggregateFieldIndexNames(params.options);
|
|
183
|
+
const hasBoundedResponseCache = hasApiJsonlResponseCacheForAllSources(params.db, {
|
|
184
|
+
panelId: params.panel.id,
|
|
185
|
+
sourceManifestKey,
|
|
186
|
+
sourceUrls: parsed.urls,
|
|
187
|
+
});
|
|
188
|
+
if (!hasBoundedResponseCache) {
|
|
189
|
+
const rangeMetadataKey = await ensureApiJsonlRangeFieldIndex({
|
|
190
|
+
db: params.db,
|
|
191
|
+
panel: params.panel,
|
|
192
|
+
sourceUrls: parsed.urls,
|
|
193
|
+
authProfile: parsed.authProfile,
|
|
194
|
+
workspaceBroker: params.workspaceBroker,
|
|
195
|
+
nodeId: agent.nodeId,
|
|
196
|
+
datasetSchema: params.datasetSchema,
|
|
197
|
+
indexContext,
|
|
198
|
+
extraFieldNames,
|
|
199
|
+
});
|
|
200
|
+
if (indexContext && rangeMetadataKey) {
|
|
201
|
+
const indexed = readAggregateFromQueryHandleFieldIndex({
|
|
202
|
+
db: params.db,
|
|
203
|
+
panel: params.panel,
|
|
204
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
205
|
+
metadataKey: rangeMetadataKey,
|
|
206
|
+
datasetSchema: params.datasetSchema,
|
|
207
|
+
options: params.options,
|
|
208
|
+
extraFieldNames,
|
|
209
|
+
sourceKind: 'api_jsonl_field_index',
|
|
210
|
+
});
|
|
211
|
+
if (indexed)
|
|
212
|
+
return indexed;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
const sources = await fetchApiJsonlSources({
|
|
216
|
+
db: params.db,
|
|
217
|
+
panelId: params.panel.id,
|
|
218
|
+
sourceUrls: parsed.urls,
|
|
219
|
+
authProfile: parsed.authProfile,
|
|
220
|
+
sourceManifestKey,
|
|
221
|
+
workspaceBroker: params.workspaceBroker,
|
|
222
|
+
nodeId: agent.nodeId,
|
|
223
|
+
});
|
|
224
|
+
const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
|
|
225
|
+
const metadataKey = apiJsonlMetadataKey(sources);
|
|
226
|
+
if (indexContext) {
|
|
227
|
+
writeQueryHandleFieldIndexRows({
|
|
228
|
+
context: indexContext,
|
|
229
|
+
metadataKey,
|
|
230
|
+
datasetSchema: params.datasetSchema,
|
|
231
|
+
rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
|
|
232
|
+
extraFieldNames,
|
|
233
|
+
});
|
|
234
|
+
const indexed = readAggregateFromQueryHandleFieldIndex({
|
|
235
|
+
db: params.db,
|
|
236
|
+
panel: params.panel,
|
|
237
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
238
|
+
metadataKey,
|
|
239
|
+
datasetSchema: params.datasetSchema,
|
|
240
|
+
options: params.options,
|
|
241
|
+
extraFieldNames,
|
|
242
|
+
sourceKind: 'api_jsonl_field_index',
|
|
243
|
+
});
|
|
244
|
+
if (indexed)
|
|
245
|
+
return indexed;
|
|
246
|
+
}
|
|
247
|
+
return aggregatePanelFieldRecords(rows
|
|
248
|
+
.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
|
|
249
|
+
.map((row) => row.fields), {
|
|
250
|
+
op: params.options.op,
|
|
251
|
+
field: params.options.field,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
if (!agent.workspacePath) {
|
|
255
|
+
throw new PanelQueryHandleError('query_handle panel agent workspace is unavailable.', 409);
|
|
256
|
+
}
|
|
257
|
+
const dataSources = resolveQueryHandleDataSources({
|
|
258
|
+
handle: source.handle,
|
|
259
|
+
workspaceRoot: agent.workspacePath,
|
|
260
|
+
config: params.config,
|
|
261
|
+
});
|
|
262
|
+
const indexContext = createQueryHandleIndexContext(params.db, params.panel.id, dataSources);
|
|
263
|
+
const extraFieldNames = aggregateFieldIndexNames(params.options);
|
|
264
|
+
const metadataKey = indexContext
|
|
265
|
+
? await loadQueryHandleIndexMetadataKey({
|
|
266
|
+
context: indexContext,
|
|
267
|
+
workspaceBroker: params.workspaceBroker,
|
|
268
|
+
nodeId: agent.nodeId,
|
|
269
|
+
workspaceRoot: agent.workspacePath,
|
|
270
|
+
dataSources,
|
|
271
|
+
})
|
|
272
|
+
: null;
|
|
273
|
+
if (indexContext && metadataKey) {
|
|
274
|
+
const indexed = readAggregateFromQueryHandleFieldIndex({
|
|
275
|
+
db: params.db,
|
|
276
|
+
panel: params.panel,
|
|
277
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
278
|
+
metadataKey,
|
|
279
|
+
datasetSchema: params.datasetSchema,
|
|
280
|
+
options: params.options,
|
|
281
|
+
extraFieldNames,
|
|
282
|
+
sourceKind: 'query_handle_field_index',
|
|
283
|
+
});
|
|
284
|
+
if (indexed)
|
|
285
|
+
return indexed;
|
|
286
|
+
}
|
|
287
|
+
const rows = await scanQueryHandleRowsForFieldIndex({
|
|
288
|
+
db: params.db,
|
|
289
|
+
panel: params.panel,
|
|
290
|
+
workspaceBroker: params.workspaceBroker,
|
|
291
|
+
nodeId: agent.nodeId,
|
|
292
|
+
workspaceRoot: agent.workspacePath,
|
|
293
|
+
dataSources,
|
|
294
|
+
datasetSchema: params.datasetSchema,
|
|
295
|
+
indexContext,
|
|
296
|
+
metadataKey,
|
|
297
|
+
extraFieldNames,
|
|
298
|
+
});
|
|
299
|
+
if (indexContext && metadataKey) {
|
|
300
|
+
const indexed = readAggregateFromQueryHandleFieldIndex({
|
|
301
|
+
db: params.db,
|
|
302
|
+
panel: params.panel,
|
|
303
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
304
|
+
metadataKey,
|
|
305
|
+
datasetSchema: params.datasetSchema,
|
|
306
|
+
options: params.options,
|
|
307
|
+
extraFieldNames,
|
|
308
|
+
sourceKind: 'query_handle_field_index',
|
|
309
|
+
});
|
|
310
|
+
if (indexed)
|
|
311
|
+
return indexed;
|
|
312
|
+
}
|
|
313
|
+
return aggregatePanelFieldRecords(rows
|
|
314
|
+
.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
|
|
315
|
+
.map((row) => row.fields), {
|
|
316
|
+
op: params.options.op,
|
|
317
|
+
field: params.options.field,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
async function listUnsortedPanelQueryHandleRows(params) {
|
|
321
|
+
if (params.indexContext && params.options.filterField && params.options.filterValue !== undefined) {
|
|
322
|
+
const metadataKey = await loadQueryHandleIndexMetadataKey({
|
|
323
|
+
context: params.indexContext,
|
|
324
|
+
workspaceBroker: params.workspaceBroker,
|
|
325
|
+
nodeId: params.nodeId,
|
|
326
|
+
workspaceRoot: params.workspaceRoot,
|
|
327
|
+
dataSources: params.dataSources,
|
|
328
|
+
});
|
|
329
|
+
if (metadataKey) {
|
|
330
|
+
const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
|
|
331
|
+
sourceKind: 'query_handle_field_index',
|
|
332
|
+
filterField: params.options.filterField,
|
|
333
|
+
filterValue: params.options.filterValue,
|
|
334
|
+
});
|
|
335
|
+
const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
|
|
336
|
+
db: params.db,
|
|
337
|
+
panel: params.panel,
|
|
338
|
+
conversationManager: params.conversationManager,
|
|
339
|
+
workspaceBroker: params.workspaceBroker,
|
|
340
|
+
config: params.config,
|
|
341
|
+
datasetSchema: params.datasetSchema,
|
|
342
|
+
sourceManifestKey: params.indexContext.sourceManifestKey,
|
|
343
|
+
metadataKey,
|
|
344
|
+
options: params.options,
|
|
345
|
+
position: cursor.position,
|
|
346
|
+
limit: params.limit,
|
|
347
|
+
});
|
|
348
|
+
if (indexed)
|
|
349
|
+
return indexed;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
const cursor = parseQueryHandleCursor(params.options.cursor);
|
|
353
|
+
let sourceIndex = Math.min(Math.max(0, cursor.sourceIndex ?? 0), params.dataSources.length - 1);
|
|
354
|
+
if (!Number.isFinite(sourceIndex))
|
|
355
|
+
sourceIndex = 0;
|
|
356
|
+
const rows = [];
|
|
357
|
+
let offset = cursor.offset;
|
|
358
|
+
let rowIndex = cursor.rowIndex;
|
|
359
|
+
let chunks = 0;
|
|
360
|
+
const sourceSawRows = new Set();
|
|
361
|
+
if (offset > 0)
|
|
362
|
+
sourceSawRows.add(sourceIndex);
|
|
363
|
+
while (rows.length < params.limit && sourceIndex < params.dataSources.length) {
|
|
364
|
+
chunks += 1;
|
|
365
|
+
if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_PAGE) {
|
|
366
|
+
throw new PanelQueryHandleError('query_handle page scan exceeded the per-page work limit.', 413);
|
|
367
|
+
}
|
|
368
|
+
const chunk = await readQueryHandleChunk({
|
|
369
|
+
workspaceBroker: params.workspaceBroker,
|
|
370
|
+
nodeId: params.nodeId,
|
|
371
|
+
workspaceRoot: params.workspaceRoot,
|
|
372
|
+
dataSource: params.dataSources[sourceIndex],
|
|
373
|
+
offset,
|
|
374
|
+
limit: QUERY_HANDLE_READ_CHUNK_BYTES,
|
|
375
|
+
});
|
|
376
|
+
const content = chunk.content ?? '';
|
|
377
|
+
const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
|
|
378
|
+
if (parsed.completeText.length === 0 && chunk.hasMore) {
|
|
379
|
+
throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
|
|
380
|
+
}
|
|
381
|
+
const baseOffset = chunk.offset ?? offset;
|
|
382
|
+
let consumedBytes = 0;
|
|
383
|
+
for (const line of parsed.lines) {
|
|
384
|
+
const lineStartOffset = baseOffset + consumedBytes;
|
|
385
|
+
consumedBytes += line.byteLength;
|
|
386
|
+
const trimmed = line.text.trim();
|
|
387
|
+
if (!trimmed)
|
|
388
|
+
continue;
|
|
389
|
+
const rawRow = parseJsonlRow(trimmed, rowIndex);
|
|
390
|
+
const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
|
|
391
|
+
const isSourceStart = !sourceSawRows.has(sourceIndex);
|
|
392
|
+
sourceSawRows.add(sourceIndex);
|
|
393
|
+
recordQueryHandleCheckpoint(params.indexContext, {
|
|
394
|
+
sourceIndex,
|
|
395
|
+
chunk,
|
|
396
|
+
rowIndex,
|
|
397
|
+
byteOffset: lineStartOffset,
|
|
398
|
+
isSourceStart,
|
|
399
|
+
});
|
|
400
|
+
rowIndex += 1;
|
|
401
|
+
if (!matchesFilter(row.fields, params.options.filterField, params.options.filterValue)) {
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
rows.push(row);
|
|
405
|
+
if (rows.length >= params.limit)
|
|
406
|
+
break;
|
|
407
|
+
}
|
|
408
|
+
offset = baseOffset + consumedBytes;
|
|
409
|
+
if (offset >= chunk.size) {
|
|
410
|
+
sourceIndex += 1;
|
|
411
|
+
offset = 0;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return {
|
|
415
|
+
rows,
|
|
416
|
+
nextCursor: sourceIndex < params.dataSources.length
|
|
417
|
+
? encodeQueryHandleCursor({ sourceIndex, offset, rowIndex })
|
|
418
|
+
: null,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
async function listSortedPanelQueryHandleRows(params) {
|
|
422
|
+
const sortField = params.options.sortField;
|
|
423
|
+
const sortDirection = params.options.sortDirection;
|
|
424
|
+
if (!sortField || !sortDirection) {
|
|
425
|
+
throw new PanelQueryHandleError('query_handle sort requires both sort field and direction.', 400);
|
|
426
|
+
}
|
|
427
|
+
const queryKey = queryHandleViewIndexQueryKey(params.options);
|
|
428
|
+
const cursor = parseQueryHandleSortCursor(params.options.cursor, {
|
|
429
|
+
sortField,
|
|
430
|
+
sortDirection,
|
|
431
|
+
filterField: params.options.filterField,
|
|
432
|
+
filterValue: params.options.filterValue,
|
|
433
|
+
});
|
|
434
|
+
const metadataKey = params.indexContext
|
|
435
|
+
? await loadQueryHandleIndexMetadataKey({
|
|
436
|
+
context: params.indexContext,
|
|
437
|
+
workspaceBroker: params.workspaceBroker,
|
|
438
|
+
nodeId: params.nodeId,
|
|
439
|
+
workspaceRoot: params.workspaceRoot,
|
|
440
|
+
dataSources: params.dataSources,
|
|
441
|
+
})
|
|
442
|
+
: null;
|
|
443
|
+
if (metadataKey) {
|
|
444
|
+
const cached = await readCachedQueryHandleViewIndexRows({
|
|
445
|
+
db: params.db,
|
|
446
|
+
panel: params.panel,
|
|
447
|
+
conversationManager: params.conversationManager,
|
|
448
|
+
workspaceBroker: params.workspaceBroker,
|
|
449
|
+
config: params.config,
|
|
450
|
+
datasetSchema: params.datasetSchema,
|
|
451
|
+
sourceManifestKey: params.indexContext.sourceManifestKey,
|
|
452
|
+
queryKey,
|
|
453
|
+
metadataKey,
|
|
454
|
+
position: cursor.position,
|
|
455
|
+
limit: params.limit,
|
|
456
|
+
});
|
|
457
|
+
if (cached)
|
|
458
|
+
return cached;
|
|
459
|
+
const indexed = await readQueryHandleRowsFromFieldIndex({
|
|
460
|
+
db: params.db,
|
|
461
|
+
panel: params.panel,
|
|
462
|
+
conversationManager: params.conversationManager,
|
|
463
|
+
workspaceBroker: params.workspaceBroker,
|
|
464
|
+
config: params.config,
|
|
465
|
+
datasetSchema: params.datasetSchema,
|
|
466
|
+
sourceManifestKey: params.indexContext.sourceManifestKey,
|
|
467
|
+
metadataKey,
|
|
468
|
+
options: params.options,
|
|
469
|
+
position: cursor.position,
|
|
470
|
+
limit: params.limit,
|
|
471
|
+
});
|
|
472
|
+
if (indexed)
|
|
473
|
+
return indexed;
|
|
474
|
+
}
|
|
475
|
+
const rows = [];
|
|
476
|
+
const fieldIndexRows = [];
|
|
477
|
+
let rowIndex = 0;
|
|
478
|
+
let chunks = 0;
|
|
479
|
+
const sourceSawRows = new Set();
|
|
480
|
+
for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
|
|
481
|
+
const dataSource = params.dataSources[sourceIndex];
|
|
482
|
+
let offset = 0;
|
|
483
|
+
let hasMore = true;
|
|
484
|
+
while (hasMore) {
|
|
485
|
+
chunks += 1;
|
|
486
|
+
if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_SORT) {
|
|
487
|
+
throw new PanelQueryHandleError('query_handle sorted scan exceeded the per-request work limit.', 413);
|
|
488
|
+
}
|
|
489
|
+
const chunk = await readQueryHandleChunk({
|
|
490
|
+
workspaceBroker: params.workspaceBroker,
|
|
491
|
+
nodeId: params.nodeId,
|
|
492
|
+
workspaceRoot: params.workspaceRoot,
|
|
493
|
+
dataSource,
|
|
494
|
+
offset,
|
|
495
|
+
limit: QUERY_HANDLE_READ_CHUNK_BYTES,
|
|
496
|
+
});
|
|
497
|
+
const content = chunk.content ?? '';
|
|
498
|
+
const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
|
|
499
|
+
if (parsed.completeText.length === 0 && chunk.hasMore) {
|
|
500
|
+
throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
|
|
501
|
+
}
|
|
502
|
+
const baseOffset = chunk.offset ?? offset;
|
|
503
|
+
let consumedBytes = 0;
|
|
504
|
+
for (const line of parsed.lines) {
|
|
505
|
+
const lineStartOffset = baseOffset + consumedBytes;
|
|
506
|
+
consumedBytes += line.byteLength;
|
|
507
|
+
const trimmed = line.text.trim();
|
|
508
|
+
if (!trimmed)
|
|
509
|
+
continue;
|
|
510
|
+
const rawRow = parseJsonlRow(trimmed, rowIndex);
|
|
511
|
+
const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
|
|
512
|
+
const isSourceStart = !sourceSawRows.has(sourceIndex);
|
|
513
|
+
sourceSawRows.add(sourceIndex);
|
|
514
|
+
recordQueryHandleCheckpoint(params.indexContext, {
|
|
515
|
+
sourceIndex,
|
|
516
|
+
chunk,
|
|
517
|
+
rowIndex,
|
|
518
|
+
byteOffset: lineStartOffset,
|
|
519
|
+
isSourceStart,
|
|
520
|
+
});
|
|
521
|
+
rowIndex += 1;
|
|
522
|
+
fieldIndexRows.push({ rowIndex: row.rowIndex, fields: row.fields });
|
|
523
|
+
if (!matchesFilter(row.fields, params.options.filterField, params.options.filterValue)) {
|
|
524
|
+
continue;
|
|
525
|
+
}
|
|
526
|
+
rows.push(row);
|
|
527
|
+
}
|
|
528
|
+
offset = baseOffset + consumedBytes;
|
|
529
|
+
hasMore = offset < chunk.size;
|
|
530
|
+
if (parsed.lines.length === 0 && !hasMore)
|
|
531
|
+
break;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
rows.sort((left, right) => comparePanelRowsBySortField(left, right, sortField, sortDirection, params.options.sortFieldType));
|
|
535
|
+
if (metadataKey && params.indexContext) {
|
|
536
|
+
writeQueryHandleViewIndexRows({
|
|
537
|
+
context: params.indexContext,
|
|
538
|
+
queryKey,
|
|
539
|
+
metadataKey,
|
|
540
|
+
rowIndices: rows.map((row) => row.rowIndex),
|
|
541
|
+
});
|
|
542
|
+
writeQueryHandleFieldIndexRows({
|
|
543
|
+
context: params.indexContext,
|
|
544
|
+
metadataKey,
|
|
545
|
+
datasetSchema: params.datasetSchema,
|
|
546
|
+
rows: fieldIndexRows,
|
|
547
|
+
});
|
|
548
|
+
}
|
|
549
|
+
const start = Math.min(cursor.position, rows.length);
|
|
550
|
+
const resultRows = rows.slice(start, start + params.limit);
|
|
551
|
+
const nextPosition = start + resultRows.length;
|
|
552
|
+
return {
|
|
553
|
+
rows: resultRows,
|
|
554
|
+
nextCursor: nextPosition < rows.length
|
|
555
|
+
? encodeQueryHandleSortCursor({
|
|
556
|
+
position: nextPosition,
|
|
557
|
+
sortField,
|
|
558
|
+
sortDirection,
|
|
559
|
+
filterField: params.options.filterField,
|
|
560
|
+
filterValue: params.options.filterValue,
|
|
561
|
+
})
|
|
562
|
+
: null,
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
async function listPanelApiJsonlRows(params) {
|
|
566
|
+
const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, params.sourceUrls, params.authProfile);
|
|
567
|
+
const selectRowsByIndicesFromDynamicSource = async (rowIndices) => (await listPanelQueryHandleRowsByIndices({
|
|
568
|
+
db: params.db,
|
|
569
|
+
panel: params.panel,
|
|
570
|
+
conversationManager: params.conversationManager,
|
|
571
|
+
workspaceBroker: params.workspaceBroker,
|
|
572
|
+
config: params.config,
|
|
573
|
+
apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
|
|
574
|
+
apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
575
|
+
apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
|
|
576
|
+
datasetSchema: params.datasetSchema,
|
|
577
|
+
rowIndices,
|
|
578
|
+
})).rows;
|
|
579
|
+
if (!params.options.sortField && !params.options.filterField && !isPositionCursor(params.options.cursor)) {
|
|
580
|
+
const ranged = await listPanelApiJsonlRowsByRange({
|
|
581
|
+
...params,
|
|
582
|
+
indexContext,
|
|
583
|
+
});
|
|
584
|
+
if (ranged)
|
|
585
|
+
return ranged;
|
|
586
|
+
}
|
|
587
|
+
const sourceManifestKey = indexContext?.sourceManifestKey ?? apiJsonlSourceManifestKey(params.sourceUrls, params.authProfile);
|
|
588
|
+
const hasBoundedResponseCache = hasApiJsonlResponseCacheForAllSources(params.db, {
|
|
589
|
+
panelId: params.panel.id,
|
|
590
|
+
sourceManifestKey,
|
|
591
|
+
sourceUrls: params.sourceUrls,
|
|
592
|
+
});
|
|
593
|
+
if (!hasBoundedResponseCache && ((params.options.sortField && params.options.sortDirection) || (params.options.filterField && params.options.filterValue !== undefined))) {
|
|
594
|
+
const rangeMetadataKey = await ensureApiJsonlRangeFieldIndex({
|
|
595
|
+
db: params.db,
|
|
596
|
+
panel: params.panel,
|
|
597
|
+
sourceUrls: params.sourceUrls,
|
|
598
|
+
authProfile: params.authProfile,
|
|
599
|
+
workspaceBroker: params.workspaceBroker,
|
|
600
|
+
nodeId: params.nodeId,
|
|
601
|
+
datasetSchema: params.datasetSchema,
|
|
602
|
+
indexContext,
|
|
603
|
+
extraFieldNames: [],
|
|
604
|
+
});
|
|
605
|
+
if (indexContext && rangeMetadataKey && params.options.sortField && params.options.sortDirection) {
|
|
606
|
+
const cursor = parseQueryHandleSortCursor(params.options.cursor, {
|
|
607
|
+
sortField: params.options.sortField,
|
|
608
|
+
sortDirection: params.options.sortDirection,
|
|
609
|
+
filterField: params.options.filterField,
|
|
610
|
+
filterValue: params.options.filterValue,
|
|
611
|
+
});
|
|
612
|
+
const indexed = await readQueryHandleRowsFromFieldIndex({
|
|
613
|
+
db: params.db,
|
|
614
|
+
panel: params.panel,
|
|
615
|
+
conversationManager: params.conversationManager,
|
|
616
|
+
workspaceBroker: params.workspaceBroker,
|
|
617
|
+
config: params.config,
|
|
618
|
+
datasetSchema: params.datasetSchema,
|
|
619
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
620
|
+
metadataKey: rangeMetadataKey,
|
|
621
|
+
options: params.options,
|
|
622
|
+
position: cursor.position,
|
|
623
|
+
limit: params.limit,
|
|
624
|
+
selectRowsByIndices: selectRowsByIndicesFromDynamicSource,
|
|
625
|
+
});
|
|
626
|
+
if (indexed)
|
|
627
|
+
return indexed;
|
|
628
|
+
}
|
|
629
|
+
if (indexContext && rangeMetadataKey && params.options.filterField && params.options.filterValue !== undefined) {
|
|
630
|
+
const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
|
|
631
|
+
sourceKind: 'api_jsonl_field_index',
|
|
632
|
+
filterField: params.options.filterField,
|
|
633
|
+
filterValue: params.options.filterValue,
|
|
634
|
+
});
|
|
635
|
+
const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
|
|
636
|
+
db: params.db,
|
|
637
|
+
panel: params.panel,
|
|
638
|
+
conversationManager: params.conversationManager,
|
|
639
|
+
workspaceBroker: params.workspaceBroker,
|
|
640
|
+
config: params.config,
|
|
641
|
+
datasetSchema: params.datasetSchema,
|
|
642
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
643
|
+
metadataKey: rangeMetadataKey,
|
|
644
|
+
options: params.options,
|
|
645
|
+
position: cursor.position,
|
|
646
|
+
limit: params.limit,
|
|
647
|
+
sourceKind: 'api_jsonl_field_index',
|
|
648
|
+
selectRowsByIndices: selectRowsByIndicesFromDynamicSource,
|
|
649
|
+
});
|
|
650
|
+
if (indexed)
|
|
651
|
+
return indexed;
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const sources = await fetchApiJsonlSources({
|
|
655
|
+
db: params.db,
|
|
656
|
+
panelId: params.panel.id,
|
|
657
|
+
sourceUrls: params.sourceUrls,
|
|
658
|
+
authProfile: params.authProfile,
|
|
659
|
+
sourceManifestKey,
|
|
660
|
+
workspaceBroker: params.workspaceBroker,
|
|
661
|
+
nodeId: params.nodeId,
|
|
662
|
+
});
|
|
663
|
+
const metadataKey = apiJsonlMetadataKey(sources);
|
|
664
|
+
const selectRowsByIndices = async (rowIndices) => selectApiJsonlRowsByIndices(sources, params.datasetSchema, rowIndices);
|
|
665
|
+
if (params.options.sortField && params.options.sortDirection) {
|
|
666
|
+
const queryKey = queryHandleViewIndexQueryKey(params.options);
|
|
667
|
+
const cursor = parseQueryHandleSortCursor(params.options.cursor, {
|
|
668
|
+
sortField: params.options.sortField,
|
|
669
|
+
sortDirection: params.options.sortDirection,
|
|
670
|
+
filterField: params.options.filterField,
|
|
671
|
+
filterValue: params.options.filterValue,
|
|
672
|
+
});
|
|
673
|
+
if (indexContext) {
|
|
674
|
+
const cached = await readCachedQueryHandleViewIndexRows({
|
|
675
|
+
db: params.db,
|
|
676
|
+
panel: params.panel,
|
|
677
|
+
conversationManager: params.conversationManager,
|
|
678
|
+
workspaceBroker: params.workspaceBroker,
|
|
679
|
+
config: params.config,
|
|
680
|
+
datasetSchema: params.datasetSchema,
|
|
681
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
682
|
+
queryKey,
|
|
683
|
+
metadataKey,
|
|
684
|
+
position: cursor.position,
|
|
685
|
+
limit: params.limit,
|
|
686
|
+
selectRowsByIndices,
|
|
687
|
+
});
|
|
688
|
+
if (cached)
|
|
689
|
+
return cached;
|
|
690
|
+
const indexed = await readQueryHandleRowsFromFieldIndex({
|
|
691
|
+
db: params.db,
|
|
692
|
+
panel: params.panel,
|
|
693
|
+
conversationManager: params.conversationManager,
|
|
694
|
+
workspaceBroker: params.workspaceBroker,
|
|
695
|
+
config: params.config,
|
|
696
|
+
datasetSchema: params.datasetSchema,
|
|
697
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
698
|
+
metadataKey,
|
|
699
|
+
options: params.options,
|
|
700
|
+
position: cursor.position,
|
|
701
|
+
limit: params.limit,
|
|
702
|
+
selectRowsByIndices,
|
|
703
|
+
});
|
|
704
|
+
if (indexed)
|
|
705
|
+
return indexed;
|
|
706
|
+
}
|
|
707
|
+
const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
|
|
708
|
+
const filteredRows = params.options.filterField && params.options.filterValue !== undefined
|
|
709
|
+
? rows.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue))
|
|
710
|
+
: rows;
|
|
711
|
+
filteredRows.sort((left, right) => comparePanelRowsBySortField(left, right, params.options.sortField, params.options.sortDirection, params.options.sortFieldType));
|
|
712
|
+
if (indexContext) {
|
|
713
|
+
writeQueryHandleViewIndexRows({
|
|
714
|
+
context: indexContext,
|
|
715
|
+
queryKey,
|
|
716
|
+
metadataKey,
|
|
717
|
+
rowIndices: filteredRows.map((row) => row.rowIndex),
|
|
718
|
+
});
|
|
719
|
+
writeQueryHandleFieldIndexRows({
|
|
720
|
+
context: indexContext,
|
|
721
|
+
metadataKey,
|
|
722
|
+
datasetSchema: params.datasetSchema,
|
|
723
|
+
rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
|
|
724
|
+
});
|
|
725
|
+
}
|
|
726
|
+
return sliceRowsByPositionCursor(filteredRows, cursor.position, params.limit, (position) => encodeQueryHandleSortCursor({
|
|
727
|
+
position,
|
|
728
|
+
sortField: params.options.sortField,
|
|
729
|
+
sortDirection: params.options.sortDirection,
|
|
730
|
+
filterField: params.options.filterField,
|
|
731
|
+
filterValue: params.options.filterValue,
|
|
732
|
+
}));
|
|
733
|
+
}
|
|
734
|
+
if (params.options.filterField && params.options.filterValue !== undefined) {
|
|
735
|
+
const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
|
|
736
|
+
sourceKind: 'api_jsonl_field_index',
|
|
737
|
+
filterField: params.options.filterField,
|
|
738
|
+
filterValue: params.options.filterValue,
|
|
739
|
+
});
|
|
740
|
+
if (indexContext) {
|
|
741
|
+
const indexed = await readFilteredQueryHandleRowsFromFieldIndex({
|
|
742
|
+
db: params.db,
|
|
743
|
+
panel: params.panel,
|
|
744
|
+
conversationManager: params.conversationManager,
|
|
745
|
+
workspaceBroker: params.workspaceBroker,
|
|
746
|
+
config: params.config,
|
|
747
|
+
datasetSchema: params.datasetSchema,
|
|
748
|
+
sourceManifestKey: indexContext.sourceManifestKey,
|
|
749
|
+
metadataKey,
|
|
750
|
+
options: params.options,
|
|
751
|
+
position: cursor.position,
|
|
752
|
+
limit: params.limit,
|
|
753
|
+
sourceKind: 'api_jsonl_field_index',
|
|
754
|
+
selectRowsByIndices,
|
|
755
|
+
});
|
|
756
|
+
if (indexed)
|
|
757
|
+
return indexed;
|
|
758
|
+
}
|
|
759
|
+
const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
|
|
760
|
+
if (indexContext) {
|
|
761
|
+
writeQueryHandleFieldIndexRows({
|
|
762
|
+
context: indexContext,
|
|
763
|
+
metadataKey,
|
|
764
|
+
datasetSchema: params.datasetSchema,
|
|
765
|
+
rows: rows.map((row) => ({ rowIndex: row.rowIndex, fields: row.fields })),
|
|
766
|
+
});
|
|
767
|
+
}
|
|
768
|
+
const filteredRows = rows.filter((row) => matchesFilter(row.fields, params.options.filterField, params.options.filterValue));
|
|
769
|
+
return sliceRowsByPositionCursor(filteredRows, cursor.position, params.limit, (position) => encodeQueryHandlePositionCursor({
|
|
770
|
+
position,
|
|
771
|
+
sourceKind: 'api_jsonl_field_index',
|
|
772
|
+
filterField: params.options.filterField,
|
|
773
|
+
filterValue: params.options.filterValue,
|
|
774
|
+
}));
|
|
775
|
+
}
|
|
776
|
+
const rows = parseApiJsonlRowsFromSources(sources, params.datasetSchema);
|
|
777
|
+
const cursor = parseQueryHandlePositionCursor(params.options.cursor, {
|
|
778
|
+
sourceKind: 'api_jsonl',
|
|
779
|
+
filterField: params.options.filterField,
|
|
780
|
+
filterValue: params.options.filterValue,
|
|
781
|
+
});
|
|
782
|
+
return sliceRowsByPositionCursor(rows, cursor.position, params.limit, (position) => encodeQueryHandlePositionCursor({
|
|
783
|
+
position,
|
|
784
|
+
sourceKind: 'api_jsonl',
|
|
785
|
+
filterField: params.options.filterField,
|
|
786
|
+
filterValue: params.options.filterValue,
|
|
787
|
+
}));
|
|
788
|
+
}
|
|
789
|
+
async function listPanelApiJsonlRowsByRange(params) {
|
|
790
|
+
const cursor = parseApiJsonlRangeCursor(params.options.cursor);
|
|
791
|
+
let sourceIndex = Math.min(Math.max(0, cursor.sourceIndex), params.sourceUrls.length - 1);
|
|
792
|
+
if (!Number.isFinite(sourceIndex))
|
|
793
|
+
sourceIndex = 0;
|
|
794
|
+
let offset = cursor.offset;
|
|
795
|
+
let rowIndex = cursor.rowIndex;
|
|
796
|
+
const rows = [];
|
|
797
|
+
let chunks = 0;
|
|
798
|
+
const sourceSawRows = new Set();
|
|
799
|
+
if (offset > 0)
|
|
800
|
+
sourceSawRows.add(sourceIndex);
|
|
801
|
+
let currentValidator = cursor.url === params.sourceUrls[sourceIndex]
|
|
802
|
+
? {
|
|
803
|
+
url: cursor.url,
|
|
804
|
+
etag: cursor.etag ?? null,
|
|
805
|
+
lastModified: cursor.lastModified ?? null,
|
|
806
|
+
totalSize: cursor.totalSize ?? null,
|
|
807
|
+
}
|
|
808
|
+
: null;
|
|
809
|
+
while (rows.length < params.limit && sourceIndex < params.sourceUrls.length) {
|
|
810
|
+
chunks += 1;
|
|
811
|
+
if (chunks > API_JSONL_MAX_RANGE_CHUNKS_PER_PAGE) {
|
|
812
|
+
throw new PanelQueryHandleError('api_jsonl ranged page scan exceeded the per-page work limit.', 413);
|
|
813
|
+
}
|
|
814
|
+
const url = params.sourceUrls[sourceIndex];
|
|
815
|
+
const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
|
|
816
|
+
maxBytes: API_JSONL_MAX_BYTES,
|
|
817
|
+
...(params.authProfile ? { authProfile: params.authProfile } : {}),
|
|
818
|
+
rangeStart: offset,
|
|
819
|
+
rangeLength: API_JSONL_RANGE_CHUNK_BYTES,
|
|
820
|
+
...(currentValidator?.url === url ? withApiJsonlIfRange(currentValidator) : {}),
|
|
821
|
+
});
|
|
822
|
+
if (result.status !== 206
|
|
823
|
+
|| result.partialContent !== true
|
|
824
|
+
|| typeof result.rangeStart !== 'number'
|
|
825
|
+
|| typeof result.rangeEnd !== 'number'
|
|
826
|
+
|| result.rangeStart !== offset) {
|
|
827
|
+
return null;
|
|
828
|
+
}
|
|
829
|
+
if (Buffer.byteLength(result.content, 'utf8') > API_JSONL_MAX_BYTES) {
|
|
830
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds byte limit (${API_JSONL_MAX_BYTES}).`, 413);
|
|
831
|
+
}
|
|
832
|
+
currentValidator = {
|
|
833
|
+
url,
|
|
834
|
+
etag: result.etag ?? null,
|
|
835
|
+
lastModified: result.lastModified ?? null,
|
|
836
|
+
totalSize: result.totalSize ?? null,
|
|
837
|
+
};
|
|
838
|
+
const checkpointChunk = apiJsonlRangeCheckpointChunk(result);
|
|
839
|
+
const parsed = parseCompleteJsonlLines(result.content, Boolean(result.hasMore));
|
|
840
|
+
if (parsed.completeText.length === 0 && result.hasMore) {
|
|
841
|
+
throw new PanelQueryHandleError('api_jsonl JSONL row exceeds the per-row size limit.', 413);
|
|
842
|
+
}
|
|
843
|
+
const baseOffset = result.rangeStart;
|
|
844
|
+
let consumedBytes = 0;
|
|
845
|
+
let stoppedForLimit = false;
|
|
846
|
+
let lastConsumedLineIndex = -1;
|
|
847
|
+
for (const [lineIndex, line] of parsed.lines.entries()) {
|
|
848
|
+
const lineStartOffset = baseOffset + consumedBytes;
|
|
849
|
+
consumedBytes += line.byteLength;
|
|
850
|
+
lastConsumedLineIndex = lineIndex;
|
|
851
|
+
const trimmed = line.text.trim();
|
|
852
|
+
if (!trimmed)
|
|
853
|
+
continue;
|
|
854
|
+
if (rowIndex >= API_JSONL_MAX_ROWS) {
|
|
855
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
|
|
856
|
+
}
|
|
857
|
+
const rawRow = parseJsonlRow(trimmed, rowIndex);
|
|
858
|
+
const isSourceStart = !sourceSawRows.has(sourceIndex);
|
|
859
|
+
sourceSawRows.add(sourceIndex);
|
|
860
|
+
recordApiJsonlRangeCheckpoint(params.indexContext, {
|
|
861
|
+
sourceIndex,
|
|
862
|
+
chunk: checkpointChunk,
|
|
863
|
+
rowIndex,
|
|
864
|
+
byteOffset: lineStartOffset,
|
|
865
|
+
isSourceStart,
|
|
866
|
+
});
|
|
867
|
+
rows.push(buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema));
|
|
868
|
+
rowIndex += 1;
|
|
869
|
+
if (rows.length >= params.limit) {
|
|
870
|
+
stoppedForLimit = true;
|
|
871
|
+
break;
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
offset = baseOffset + consumedBytes;
|
|
875
|
+
const consumedAllParsedLines = lastConsumedLineIndex >= parsed.lines.length - 1;
|
|
876
|
+
if (result.hasMore !== true && (!stoppedForLimit || consumedAllParsedLines)) {
|
|
877
|
+
sourceIndex += 1;
|
|
878
|
+
offset = 0;
|
|
879
|
+
currentValidator = null;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
if (sourceIndex >= params.sourceUrls.length) {
|
|
883
|
+
return { rows, nextCursor: null };
|
|
884
|
+
}
|
|
885
|
+
const nextUrl = params.sourceUrls[sourceIndex];
|
|
886
|
+
return {
|
|
887
|
+
rows,
|
|
888
|
+
nextCursor: encodeApiJsonlRangeCursor({
|
|
889
|
+
sourceKind: 'api_jsonl_range',
|
|
890
|
+
sourceIndex,
|
|
891
|
+
offset,
|
|
892
|
+
rowIndex,
|
|
893
|
+
url: nextUrl,
|
|
894
|
+
...(currentValidator?.url === nextUrl
|
|
895
|
+
? {
|
|
896
|
+
etag: currentValidator.etag ?? null,
|
|
897
|
+
lastModified: currentValidator.lastModified ?? null,
|
|
898
|
+
totalSize: currentValidator.totalSize ?? null,
|
|
899
|
+
}
|
|
900
|
+
: {}),
|
|
901
|
+
}),
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
async function fetchApiJsonlSources(params) {
|
|
905
|
+
const sources = [];
|
|
906
|
+
let totalRows = 0;
|
|
907
|
+
if (params.db) {
|
|
908
|
+
pruneApiJsonlResponseCache(params.db, params.panelId, params.sourceManifestKey);
|
|
909
|
+
}
|
|
910
|
+
for (const [sourceIndex, url] of params.sourceUrls.entries()) {
|
|
911
|
+
const cached = params.db
|
|
912
|
+
? loadApiJsonlResponseCache(params.db, {
|
|
913
|
+
panelId: params.panelId,
|
|
914
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
915
|
+
sourceIndex,
|
|
916
|
+
url,
|
|
917
|
+
})
|
|
918
|
+
: null;
|
|
919
|
+
const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
|
|
920
|
+
maxBytes: API_JSONL_MAX_BYTES,
|
|
921
|
+
...(params.authProfile ? { authProfile: params.authProfile } : {}),
|
|
922
|
+
...(cached?.etag ? { ifNoneMatch: cached.etag } : {}),
|
|
923
|
+
...(cached?.lastModified ? { ifModifiedSince: cached.lastModified } : {}),
|
|
924
|
+
});
|
|
925
|
+
if (result.notModified || result.status === 304) {
|
|
926
|
+
if (!cached) {
|
|
927
|
+
throw new PanelQueryHandleError('api_jsonl endpoint returned 304 but no cached response is available.', 502);
|
|
928
|
+
}
|
|
929
|
+
totalRows += cached.rowCount;
|
|
930
|
+
if (totalRows > API_JSONL_MAX_ROWS) {
|
|
931
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
|
|
932
|
+
}
|
|
933
|
+
const refreshed = {
|
|
934
|
+
...cached,
|
|
935
|
+
etag: result.etag ?? cached.etag,
|
|
936
|
+
lastModified: result.lastModified ?? cached.lastModified,
|
|
937
|
+
fetchedFromCache: true,
|
|
938
|
+
};
|
|
939
|
+
if (params.db && (refreshed.etag !== cached.etag || refreshed.lastModified !== cached.lastModified)) {
|
|
940
|
+
upsertApiJsonlResponseCache(params.db, {
|
|
941
|
+
panelId: params.panelId,
|
|
942
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
943
|
+
sourceIndex,
|
|
944
|
+
url,
|
|
945
|
+
source: refreshed,
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
sources.push(refreshed);
|
|
949
|
+
continue;
|
|
950
|
+
}
|
|
951
|
+
const byteLength = Buffer.byteLength(result.content, 'utf8');
|
|
952
|
+
if (byteLength > API_JSONL_MAX_BYTES) {
|
|
953
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds byte limit (${API_JSONL_MAX_BYTES}).`, 413);
|
|
954
|
+
}
|
|
955
|
+
const rowCount = countApiJsonlRows(result.content);
|
|
956
|
+
totalRows += rowCount;
|
|
957
|
+
if (totalRows > API_JSONL_MAX_ROWS) {
|
|
958
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
|
|
959
|
+
}
|
|
960
|
+
const source = {
|
|
961
|
+
url,
|
|
962
|
+
content: result.content,
|
|
963
|
+
byteLength,
|
|
964
|
+
contentHash: createHash('sha256').update(result.content).digest('hex'),
|
|
965
|
+
rowCount,
|
|
966
|
+
contentType: result.contentType,
|
|
967
|
+
etag: result.etag ?? null,
|
|
968
|
+
lastModified: result.lastModified ?? null,
|
|
969
|
+
fetchedFromCache: false,
|
|
970
|
+
};
|
|
971
|
+
sources.push(source);
|
|
972
|
+
if (params.db) {
|
|
973
|
+
upsertApiJsonlResponseCache(params.db, {
|
|
974
|
+
panelId: params.panelId,
|
|
975
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
976
|
+
sourceIndex,
|
|
977
|
+
url,
|
|
978
|
+
source,
|
|
979
|
+
});
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
return sources;
|
|
983
|
+
}
|
|
984
|
+
function countApiJsonlRows(content) {
|
|
985
|
+
return content.split('\n').filter((line) => line.trim() !== '').length;
|
|
986
|
+
}
|
|
987
|
+
function createApiJsonlIndexContext(db, panelId, sourceUrls, authProfile) {
|
|
988
|
+
if (!db || sourceUrls.length < 1)
|
|
989
|
+
return null;
|
|
990
|
+
const sourceKeys = sourceUrls.map((url) => `api_jsonl:${url}`);
|
|
991
|
+
return {
|
|
992
|
+
db,
|
|
993
|
+
panelId,
|
|
994
|
+
sourceManifestKey: apiJsonlSourceManifestKey(sourceUrls, authProfile),
|
|
995
|
+
sourceKeys,
|
|
996
|
+
metadataBySourceIndex: new Map(),
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
export function apiJsonlSourceManifestKey(sourceUrls, authProfile) {
|
|
1000
|
+
return JSON.stringify({ kind: 'api_jsonl', authProfile, urls: sourceUrls });
|
|
1001
|
+
}
|
|
1002
|
+
function apiJsonlMetadataKey(sources) {
|
|
1003
|
+
return JSON.stringify(sources.map((source) => ({
|
|
1004
|
+
url: source.url,
|
|
1005
|
+
byteLength: source.byteLength,
|
|
1006
|
+
rowCount: source.rowCount,
|
|
1007
|
+
sha256: source.contentHash,
|
|
1008
|
+
})));
|
|
1009
|
+
}
|
|
1010
|
+
function loadApiJsonlResponseCache(db, params) {
|
|
1011
|
+
const row = db.prepare(`SELECT url,
|
|
1012
|
+
content,
|
|
1013
|
+
byte_length as byteLength,
|
|
1014
|
+
content_hash as contentHash,
|
|
1015
|
+
row_count as rowCount,
|
|
1016
|
+
content_type as contentType,
|
|
1017
|
+
etag,
|
|
1018
|
+
last_modified as lastModified
|
|
1019
|
+
FROM panel_api_jsonl_response_cache
|
|
1020
|
+
WHERE panel_id = ?
|
|
1021
|
+
AND source_manifest_key = ?
|
|
1022
|
+
AND source_index = ?
|
|
1023
|
+
AND url = ?
|
|
1024
|
+
LIMIT 1`).get(params.panelId, params.sourceManifestKey, params.sourceIndex, params.url);
|
|
1025
|
+
return row ? { ...row, fetchedFromCache: true } : null;
|
|
1026
|
+
}
|
|
1027
|
+
function hasApiJsonlResponseCacheForAllSources(db, params) {
|
|
1028
|
+
if (!db || params.sourceUrls.length === 0)
|
|
1029
|
+
return false;
|
|
1030
|
+
const lookup = db.prepare(`SELECT 1
|
|
1031
|
+
FROM panel_api_jsonl_response_cache
|
|
1032
|
+
WHERE panel_id = ?
|
|
1033
|
+
AND source_manifest_key = ?
|
|
1034
|
+
AND source_index = ?
|
|
1035
|
+
AND url = ?
|
|
1036
|
+
LIMIT 1`);
|
|
1037
|
+
return params.sourceUrls.every((url, sourceIndex) => Boolean(lookup.get(params.panelId, params.sourceManifestKey, sourceIndex, url)));
|
|
1038
|
+
}
|
|
1039
|
+
function upsertApiJsonlResponseCache(db, params) {
|
|
1040
|
+
const now = Date.now();
|
|
1041
|
+
db.prepare(`INSERT INTO panel_api_jsonl_response_cache(
|
|
1042
|
+
panel_id, source_manifest_key, source_index, url,
|
|
1043
|
+
etag, last_modified, content_type, content, content_hash,
|
|
1044
|
+
byte_length, row_count, fetched_at, updated_at
|
|
1045
|
+
)
|
|
1046
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1047
|
+
ON CONFLICT(panel_id, source_manifest_key, source_index, url)
|
|
1048
|
+
DO UPDATE SET
|
|
1049
|
+
etag = excluded.etag,
|
|
1050
|
+
last_modified = excluded.last_modified,
|
|
1051
|
+
content_type = excluded.content_type,
|
|
1052
|
+
content = excluded.content,
|
|
1053
|
+
content_hash = excluded.content_hash,
|
|
1054
|
+
byte_length = excluded.byte_length,
|
|
1055
|
+
row_count = excluded.row_count,
|
|
1056
|
+
updated_at = excluded.updated_at`).run(params.panelId, params.sourceManifestKey, params.sourceIndex, params.url, params.source.etag, params.source.lastModified, params.source.contentType, params.source.content, params.source.contentHash, params.source.byteLength, params.source.rowCount, now, now);
|
|
1057
|
+
}
|
|
1058
|
+
export function pruneApiJsonlResponseCache(db, panelId, sourceManifestKey) {
|
|
1059
|
+
const now = Date.now();
|
|
1060
|
+
const ttlCutoff = now - PANEL_API_JSONL_RESPONSE_CACHE_TTL_MS;
|
|
1061
|
+
const evictedKeys = new Set();
|
|
1062
|
+
// Source-churn eviction: any manifest key other than the active one is stale.
|
|
1063
|
+
const nonActiveKeys = db.prepare(`SELECT DISTINCT source_manifest_key
|
|
1064
|
+
FROM panel_api_jsonl_response_cache
|
|
1065
|
+
WHERE panel_id = ?
|
|
1066
|
+
AND source_manifest_key != ?`).all(panelId, sourceManifestKey);
|
|
1067
|
+
for (const row of nonActiveKeys) {
|
|
1068
|
+
evictedKeys.add(row.source_manifest_key);
|
|
1069
|
+
}
|
|
1070
|
+
if (nonActiveKeys.length > 0) {
|
|
1071
|
+
db.prepare(`DELETE FROM panel_api_jsonl_response_cache
|
|
1072
|
+
WHERE panel_id = ?
|
|
1073
|
+
AND source_manifest_key != ?`).run(panelId, sourceManifestKey);
|
|
1074
|
+
}
|
|
1075
|
+
// TTL eviction for the active manifest key (whole-source granularity).
|
|
1076
|
+
const staleActiveKey = db.prepare(`SELECT source_manifest_key
|
|
1077
|
+
FROM panel_api_jsonl_response_cache
|
|
1078
|
+
WHERE panel_id = ?
|
|
1079
|
+
AND source_manifest_key = ?
|
|
1080
|
+
GROUP BY source_manifest_key
|
|
1081
|
+
HAVING MAX(updated_at) < ?`).get(panelId, sourceManifestKey, ttlCutoff);
|
|
1082
|
+
if (staleActiveKey) {
|
|
1083
|
+
evictedKeys.add(staleActiveKey.source_manifest_key);
|
|
1084
|
+
db.prepare(`DELETE FROM panel_api_jsonl_response_cache
|
|
1085
|
+
WHERE panel_id = ?
|
|
1086
|
+
AND source_manifest_key = ?`).run(panelId, staleActiveKey.source_manifest_key);
|
|
1087
|
+
}
|
|
1088
|
+
// Size-budget eviction, preserving the active manifest key until last.
|
|
1089
|
+
let totalBytes = db.prepare(`SELECT COALESCE(SUM(byte_length), 0) as total
|
|
1090
|
+
FROM panel_api_jsonl_response_cache
|
|
1091
|
+
WHERE panel_id = ?`).get(panelId).total;
|
|
1092
|
+
if (totalBytes > PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL) {
|
|
1093
|
+
const candidates = db.prepare(`SELECT source_manifest_key, SUM(byte_length) as byte_length
|
|
1094
|
+
FROM panel_api_jsonl_response_cache
|
|
1095
|
+
WHERE panel_id = ?
|
|
1096
|
+
GROUP BY source_manifest_key
|
|
1097
|
+
ORDER BY CASE WHEN source_manifest_key = ? THEN 1 ELSE 0 END ASC,
|
|
1098
|
+
MAX(updated_at) ASC,
|
|
1099
|
+
source_manifest_key ASC`).all(panelId, sourceManifestKey);
|
|
1100
|
+
for (const candidate of candidates) {
|
|
1101
|
+
if (totalBytes <= PANEL_API_JSONL_RESPONSE_CACHE_MAX_TOTAL_BYTES_PER_PANEL)
|
|
1102
|
+
break;
|
|
1103
|
+
if (evictedKeys.has(candidate.source_manifest_key))
|
|
1104
|
+
continue;
|
|
1105
|
+
db.prepare(`DELETE FROM panel_api_jsonl_response_cache
|
|
1106
|
+
WHERE panel_id = ?
|
|
1107
|
+
AND source_manifest_key = ?`).run(panelId, candidate.source_manifest_key);
|
|
1108
|
+
totalBytes -= candidate.byte_length;
|
|
1109
|
+
evictedKeys.add(candidate.source_manifest_key);
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
if (evictedKeys.size === 0)
|
|
1113
|
+
return;
|
|
1114
|
+
const keys = Array.from(evictedKeys);
|
|
1115
|
+
const placeholders = keys.map(() => '?').join(',');
|
|
1116
|
+
db.transaction(() => {
|
|
1117
|
+
db.prepare(`DELETE FROM panel_query_handle_view_index_meta
|
|
1118
|
+
WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
|
|
1119
|
+
db.prepare(`DELETE FROM panel_query_handle_view_index_rows
|
|
1120
|
+
WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
|
|
1121
|
+
db.prepare(`DELETE FROM panel_query_handle_field_index_meta
|
|
1122
|
+
WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
|
|
1123
|
+
db.prepare(`DELETE FROM panel_query_handle_field_index_rows
|
|
1124
|
+
WHERE panel_id = ? AND source_manifest_key IN (${placeholders})`).run(panelId, ...keys);
|
|
1125
|
+
})();
|
|
1126
|
+
}
|
|
1127
|
+
function parseApiJsonlRowsFromSources(sources, datasetSchema) {
|
|
1128
|
+
let rows = [];
|
|
1129
|
+
for (const source of sources) {
|
|
1130
|
+
rows = rows.concat(parseApiJsonlRows(source.content, datasetSchema, rows.length));
|
|
1131
|
+
}
|
|
1132
|
+
return rows;
|
|
1133
|
+
}
|
|
1134
|
+
async function selectApiJsonlRowsByIndices(sources, datasetSchema, rowIndices) {
|
|
1135
|
+
const targetIndices = new Set(rowIndices);
|
|
1136
|
+
if (targetIndices.size === 0)
|
|
1137
|
+
return [];
|
|
1138
|
+
const found = new Map();
|
|
1139
|
+
let globalRowIndex = 0;
|
|
1140
|
+
for (const source of sources) {
|
|
1141
|
+
for (const line of source.content.split('\n')) {
|
|
1142
|
+
const trimmed = line.trim();
|
|
1143
|
+
if (!trimmed)
|
|
1144
|
+
continue;
|
|
1145
|
+
if (targetIndices.has(globalRowIndex)) {
|
|
1146
|
+
const rawRow = parseJsonlRow(trimmed, globalRowIndex);
|
|
1147
|
+
found.set(globalRowIndex, buildPanelRowFromQueryRow(rawRow, globalRowIndex, datasetSchema));
|
|
1148
|
+
if (found.size === targetIndices.size) {
|
|
1149
|
+
return rowIndices
|
|
1150
|
+
.map((rowIndex) => found.get(rowIndex))
|
|
1151
|
+
.filter((row) => Boolean(row));
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
globalRowIndex += 1;
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
return rowIndices
|
|
1158
|
+
.map((rowIndex) => found.get(rowIndex))
|
|
1159
|
+
.filter((row) => Boolean(row));
|
|
1160
|
+
}
|
|
1161
|
+
function parseApiJsonlRows(content, datasetSchema, startRowIndex = 0) {
|
|
1162
|
+
const lines = content.split('\n').filter((line) => line.trim() !== '');
|
|
1163
|
+
if (startRowIndex + lines.length > API_JSONL_MAX_ROWS) {
|
|
1164
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
|
|
1165
|
+
}
|
|
1166
|
+
return lines.map((line, rowIndex) => {
|
|
1167
|
+
const globalRowIndex = startRowIndex + rowIndex;
|
|
1168
|
+
const rawRow = parseJsonlRow(line.trim(), globalRowIndex);
|
|
1169
|
+
return buildPanelRowFromQueryRow(rawRow, globalRowIndex, datasetSchema);
|
|
1170
|
+
});
|
|
1171
|
+
}
|
|
1172
|
+
function sliceRowsByPositionCursor(rows, startPosition, limit, encodeCursor) {
|
|
1173
|
+
const start = Math.min(Math.max(0, startPosition), rows.length);
|
|
1174
|
+
const resultRows = rows.slice(start, start + limit);
|
|
1175
|
+
const nextPosition = start + resultRows.length;
|
|
1176
|
+
return {
|
|
1177
|
+
rows: resultRows,
|
|
1178
|
+
nextCursor: nextPosition < rows.length ? encodeCursor(nextPosition) : null,
|
|
1179
|
+
};
|
|
1180
|
+
}
|
|
1181
|
+
export async function listPanelQueryHandleRowsByIndices(params) {
|
|
1182
|
+
const uniqueIndices = Array.from(new Set(params.rowIndices.filter((rowIndex) => Number.isInteger(rowIndex) && rowIndex >= 0)));
|
|
1183
|
+
if (uniqueIndices.length === 0) {
|
|
1184
|
+
return { rows: [], nextCursor: null };
|
|
1185
|
+
}
|
|
1186
|
+
const targetIndices = new Set(uniqueIndices);
|
|
1187
|
+
const maxTargetIndex = Math.max(...uniqueIndices);
|
|
1188
|
+
const found = new Map();
|
|
1189
|
+
let cursor = await getIndexedCursorForDynamicRowIndices(params, Math.min(...uniqueIndices));
|
|
1190
|
+
while (found.size < targetIndices.size) {
|
|
1191
|
+
const result = await listPanelQueryHandleRows({
|
|
1192
|
+
db: params.db,
|
|
1193
|
+
panel: params.panel,
|
|
1194
|
+
conversationManager: params.conversationManager,
|
|
1195
|
+
workspaceBroker: params.workspaceBroker,
|
|
1196
|
+
config: params.config,
|
|
1197
|
+
apiJsonlAllowedOrigins: params.apiJsonlAllowedOrigins,
|
|
1198
|
+
apiJsonlAllowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
1199
|
+
apiJsonlAuthProfiles: params.apiJsonlAuthProfiles,
|
|
1200
|
+
datasetSchema: params.datasetSchema,
|
|
1201
|
+
options: {
|
|
1202
|
+
cursor,
|
|
1203
|
+
limit: 200,
|
|
1204
|
+
},
|
|
1205
|
+
});
|
|
1206
|
+
for (const row of result.rows) {
|
|
1207
|
+
if (targetIndices.has(row.rowIndex)) {
|
|
1208
|
+
found.set(row.rowIndex, row);
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
const lastRowIndex = result.rows[result.rows.length - 1]?.rowIndex;
|
|
1212
|
+
if (!result.nextCursor || lastRowIndex === undefined || lastRowIndex >= maxTargetIndex) {
|
|
1213
|
+
break;
|
|
1214
|
+
}
|
|
1215
|
+
cursor = result.nextCursor;
|
|
1216
|
+
}
|
|
1217
|
+
return {
|
|
1218
|
+
rows: params.rowIndices
|
|
1219
|
+
.map((rowIndex) => found.get(rowIndex))
|
|
1220
|
+
.filter((row) => Boolean(row)),
|
|
1221
|
+
nextCursor: null,
|
|
1222
|
+
};
|
|
1223
|
+
}
|
|
1224
|
+
function resolveQueryHandleDataSources(params) {
|
|
1225
|
+
const parsed = parsePanelQueryHandle(params.handle);
|
|
1226
|
+
return parsed.paths.map((sourcePath) => resolveQueryHandleDataSourcePath({
|
|
1227
|
+
sourcePath,
|
|
1228
|
+
workspaceRoot: params.workspaceRoot,
|
|
1229
|
+
config: params.config,
|
|
1230
|
+
}));
|
|
1231
|
+
}
|
|
1232
|
+
function resolveQueryHandleDataSourcePath(params) {
|
|
1233
|
+
if (!path.isAbsolute(params.sourcePath)) {
|
|
1234
|
+
return {
|
|
1235
|
+
kind: 'workspace_relative',
|
|
1236
|
+
relativePath: params.sourcePath,
|
|
1237
|
+
};
|
|
1238
|
+
}
|
|
1239
|
+
const resolved = resolvePanelDatasetPath(params.workspaceRoot, params.sourcePath, params.config);
|
|
1240
|
+
if (!resolved.ok) {
|
|
1241
|
+
throw new PanelQueryHandleError(resolved.reason, 403);
|
|
1242
|
+
}
|
|
1243
|
+
return {
|
|
1244
|
+
kind: 'absolute',
|
|
1245
|
+
absolutePath: resolved.absolutePath,
|
|
1246
|
+
allowedRoot: resolved.allowedRoot,
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
1249
|
+
function createQueryHandleIndexContext(db, panelId, dataSources) {
|
|
1250
|
+
if (!db || dataSources.length < 1)
|
|
1251
|
+
return null;
|
|
1252
|
+
const sourceKeys = dataSources.map((dataSource) => queryHandleSourceKey(dataSource));
|
|
1253
|
+
return {
|
|
1254
|
+
db,
|
|
1255
|
+
panelId,
|
|
1256
|
+
sourceManifestKey: JSON.stringify(sourceKeys),
|
|
1257
|
+
sourceKeys,
|
|
1258
|
+
metadataBySourceIndex: new Map(),
|
|
1259
|
+
};
|
|
1260
|
+
}
|
|
1261
|
+
async function getIndexedCursorForDynamicRowIndices(params, minTargetIndex) {
|
|
1262
|
+
if (params.panel.datasetSource?.kind === 'api_jsonl') {
|
|
1263
|
+
return getIndexedCursorForApiJsonlRowIndices(params, minTargetIndex);
|
|
1264
|
+
}
|
|
1265
|
+
return getIndexedCursorForQueryHandleRowIndices(params, minTargetIndex);
|
|
1266
|
+
}
|
|
1267
|
+
async function getIndexedCursorForQueryHandleRowIndices(params, minTargetIndex) {
|
|
1268
|
+
if (!params.db || minTargetIndex <= 0 || params.panel.datasetSource?.kind !== 'query_handle') {
|
|
1269
|
+
return null;
|
|
1270
|
+
}
|
|
1271
|
+
const agent = params.conversationManager.getAgent(params.panel.agentId);
|
|
1272
|
+
if (!agent?.nodeId || !agent.workspacePath || !params.workspaceBroker) {
|
|
1273
|
+
return null;
|
|
1274
|
+
}
|
|
1275
|
+
const dataSources = resolveQueryHandleDataSources({
|
|
1276
|
+
handle: params.panel.datasetSource.handle,
|
|
1277
|
+
workspaceRoot: agent.workspacePath,
|
|
1278
|
+
config: params.config,
|
|
1279
|
+
});
|
|
1280
|
+
const indexContext = createQueryHandleIndexContext(params.db, params.panel.id, dataSources);
|
|
1281
|
+
if (!indexContext)
|
|
1282
|
+
return null;
|
|
1283
|
+
const existing = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
|
|
1284
|
+
if (!existing)
|
|
1285
|
+
return null;
|
|
1286
|
+
const sourceRows = listIndexedQueryHandleSources(indexContext);
|
|
1287
|
+
if (!hasIndexedRowsForEveryQueryHandleSource(indexContext, sourceRows))
|
|
1288
|
+
return null;
|
|
1289
|
+
for (let sourceIndex = 0; sourceIndex < dataSources.length; sourceIndex += 1) {
|
|
1290
|
+
const metadataChunk = await readQueryHandleChunk({
|
|
1291
|
+
workspaceBroker: params.workspaceBroker,
|
|
1292
|
+
nodeId: agent.nodeId,
|
|
1293
|
+
workspaceRoot: agent.workspacePath,
|
|
1294
|
+
dataSource: dataSources[sourceIndex],
|
|
1295
|
+
offset: 0,
|
|
1296
|
+
limit: 1,
|
|
1297
|
+
});
|
|
1298
|
+
if (!ensureQueryHandleIndexMetadataFresh(indexContext, sourceIndex, metadataChunk)) {
|
|
1299
|
+
return null;
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
const fresh = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
|
|
1303
|
+
if (!fresh)
|
|
1304
|
+
return null;
|
|
1305
|
+
return encodeQueryHandleCursor({
|
|
1306
|
+
sourceIndex: fresh.sourceIndex,
|
|
1307
|
+
offset: fresh.byteOffset,
|
|
1308
|
+
rowIndex: fresh.rowIndex,
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
async function getIndexedCursorForApiJsonlRowIndices(params, minTargetIndex) {
|
|
1312
|
+
if (!params.db || minTargetIndex <= 0 || params.panel.datasetSource?.kind !== 'api_jsonl') {
|
|
1313
|
+
return null;
|
|
1314
|
+
}
|
|
1315
|
+
const agent = params.conversationManager.getAgent(params.panel.agentId);
|
|
1316
|
+
if (!agent?.nodeId || !params.workspaceBroker)
|
|
1317
|
+
return null;
|
|
1318
|
+
const parsed = parsePanelApiJsonlUrls(params.panel.datasetSource, {
|
|
1319
|
+
allowedOrigins: params.apiJsonlAllowedOrigins,
|
|
1320
|
+
allowedUrlPrefixes: params.apiJsonlAllowedUrlPrefixes,
|
|
1321
|
+
allowedAuthProfiles: params.apiJsonlAuthProfiles,
|
|
1322
|
+
});
|
|
1323
|
+
const indexContext = createApiJsonlIndexContext(params.db, params.panel.id, parsed.urls, parsed.authProfile);
|
|
1324
|
+
if (!indexContext)
|
|
1325
|
+
return null;
|
|
1326
|
+
const existing = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
|
|
1327
|
+
if (!existing)
|
|
1328
|
+
return null;
|
|
1329
|
+
const sourceRows = listIndexedQueryHandleSources(indexContext);
|
|
1330
|
+
if (!hasIndexedRowsThroughSource(indexContext, sourceRows, existing.sourceIndex))
|
|
1331
|
+
return null;
|
|
1332
|
+
const sourceMetadata = new Map();
|
|
1333
|
+
for (let sourceIndex = 0; sourceIndex <= existing.sourceIndex; sourceIndex += 1) {
|
|
1334
|
+
const metadata = await readApiJsonlRangeMetadata({
|
|
1335
|
+
workspaceBroker: params.workspaceBroker,
|
|
1336
|
+
nodeId: agent.nodeId,
|
|
1337
|
+
url: parsed.urls[sourceIndex],
|
|
1338
|
+
authProfile: parsed.authProfile,
|
|
1339
|
+
});
|
|
1340
|
+
if (!metadata)
|
|
1341
|
+
return null;
|
|
1342
|
+
sourceMetadata.set(sourceIndex, metadata);
|
|
1343
|
+
if (!ensureQueryHandleIndexMetadataFresh(indexContext, sourceIndex, metadata.chunk)) {
|
|
1344
|
+
return null;
|
|
1345
|
+
}
|
|
1346
|
+
}
|
|
1347
|
+
const fresh = selectNearestQueryHandleCheckpoint(indexContext, minTargetIndex);
|
|
1348
|
+
if (!fresh)
|
|
1349
|
+
return null;
|
|
1350
|
+
const metadata = sourceMetadata.get(fresh.sourceIndex);
|
|
1351
|
+
if (!metadata)
|
|
1352
|
+
return null;
|
|
1353
|
+
return encodeApiJsonlRangeCursor({
|
|
1354
|
+
sourceKind: 'api_jsonl_range',
|
|
1355
|
+
sourceIndex: fresh.sourceIndex,
|
|
1356
|
+
offset: fresh.byteOffset,
|
|
1357
|
+
rowIndex: fresh.rowIndex,
|
|
1358
|
+
url: parsed.urls[fresh.sourceIndex],
|
|
1359
|
+
etag: metadata.etag,
|
|
1360
|
+
lastModified: metadata.lastModified,
|
|
1361
|
+
totalSize: metadata.totalSize,
|
|
1362
|
+
});
|
|
1363
|
+
}
|
|
1364
|
+
function queryHandleSourceKey(dataSource) {
|
|
1365
|
+
return dataSource.kind === 'workspace_relative'
|
|
1366
|
+
? `workspace:${dataSource.relativePath}`
|
|
1367
|
+
: `absolute:${dataSource.absolutePath}`;
|
|
1368
|
+
}
|
|
1369
|
+
function selectNearestQueryHandleCheckpoint(context, targetRowIndex) {
|
|
1370
|
+
const row = context.db.prepare(`SELECT source_index as sourceIndex, row_index as rowIndex, byte_offset as byteOffset
|
|
1371
|
+
FROM panel_query_handle_index
|
|
1372
|
+
WHERE panel_id = ?
|
|
1373
|
+
AND source_manifest_key = ?
|
|
1374
|
+
AND row_index <= ?
|
|
1375
|
+
ORDER BY row_index DESC
|
|
1376
|
+
LIMIT 1`).get(context.panelId, context.sourceManifestKey, targetRowIndex);
|
|
1377
|
+
if (!row
|
|
1378
|
+
|| !Number.isInteger(row.sourceIndex)
|
|
1379
|
+
|| !Number.isInteger(row.rowIndex)
|
|
1380
|
+
|| !Number.isInteger(row.byteOffset)
|
|
1381
|
+
|| context.sourceKeys[row.sourceIndex] == null) {
|
|
1382
|
+
return null;
|
|
1383
|
+
}
|
|
1384
|
+
return row;
|
|
1385
|
+
}
|
|
1386
|
+
function listIndexedQueryHandleSources(context) {
|
|
1387
|
+
return context.db.prepare(`SELECT source_index as sourceIndex, source_key as sourceKey
|
|
1388
|
+
FROM panel_query_handle_index
|
|
1389
|
+
WHERE panel_id = ?
|
|
1390
|
+
AND source_manifest_key = ?
|
|
1391
|
+
GROUP BY source_index, source_key
|
|
1392
|
+
ORDER BY source_index ASC`).all(context.panelId, context.sourceManifestKey);
|
|
1393
|
+
}
|
|
1394
|
+
function hasIndexedRowsForEveryQueryHandleSource(context, rows) {
|
|
1395
|
+
const bySourceIndex = new Map(rows.map((row) => [row.sourceIndex, row.sourceKey]));
|
|
1396
|
+
return context.sourceKeys.every((sourceKey, sourceIndex) => bySourceIndex.get(sourceIndex) === sourceKey);
|
|
1397
|
+
}
|
|
1398
|
+
function hasIndexedRowsThroughSource(context, rows, maxSourceIndex) {
|
|
1399
|
+
const bySourceIndex = new Map(rows.map((row) => [row.sourceIndex, row.sourceKey]));
|
|
1400
|
+
for (let sourceIndex = 0; sourceIndex <= maxSourceIndex; sourceIndex += 1) {
|
|
1401
|
+
if (bySourceIndex.get(sourceIndex) !== context.sourceKeys[sourceIndex])
|
|
1402
|
+
return false;
|
|
1403
|
+
}
|
|
1404
|
+
return true;
|
|
1405
|
+
}
|
|
1406
|
+
async function readApiJsonlRangeMetadata(params) {
|
|
1407
|
+
const result = await params.workspaceBroker.fetchHttpText(params.nodeId, params.url, {
|
|
1408
|
+
maxBytes: API_JSONL_MAX_BYTES,
|
|
1409
|
+
...(params.authProfile ? { authProfile: params.authProfile } : {}),
|
|
1410
|
+
rangeStart: 0,
|
|
1411
|
+
rangeLength: 1,
|
|
1412
|
+
});
|
|
1413
|
+
if (result.status !== 206
|
|
1414
|
+
|| result.partialContent !== true
|
|
1415
|
+
|| result.rangeStart !== 0) {
|
|
1416
|
+
return null;
|
|
1417
|
+
}
|
|
1418
|
+
const chunk = apiJsonlRangeCheckpointChunk(result);
|
|
1419
|
+
if (!chunk || chunk.modifiedAt == null)
|
|
1420
|
+
return null;
|
|
1421
|
+
return {
|
|
1422
|
+
chunk,
|
|
1423
|
+
etag: result.etag ?? null,
|
|
1424
|
+
lastModified: result.lastModified ?? null,
|
|
1425
|
+
totalSize: chunk.size,
|
|
1426
|
+
};
|
|
1427
|
+
}
|
|
1428
|
+
async function ensureApiJsonlRangeFieldIndex(params) {
|
|
1429
|
+
if (!params.db || !params.indexContext)
|
|
1430
|
+
return null;
|
|
1431
|
+
const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
|
|
1432
|
+
if (!fieldSchemaKey)
|
|
1433
|
+
return null;
|
|
1434
|
+
const sourceMetadata = [];
|
|
1435
|
+
for (const [sourceIndex, url] of params.sourceUrls.entries()) {
|
|
1436
|
+
const metadata = await readApiJsonlRangeMetadata({
|
|
1437
|
+
workspaceBroker: params.workspaceBroker,
|
|
1438
|
+
nodeId: params.nodeId,
|
|
1439
|
+
url,
|
|
1440
|
+
authProfile: params.authProfile,
|
|
1441
|
+
});
|
|
1442
|
+
if (!metadata)
|
|
1443
|
+
return null;
|
|
1444
|
+
sourceMetadata.push(metadata);
|
|
1445
|
+
if (!ensureQueryHandleIndexMetadataFresh(params.indexContext, sourceIndex, metadata.chunk)) {
|
|
1446
|
+
return null;
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
const metadataKey = apiJsonlRangeMetadataKey(params.sourceUrls, sourceMetadata);
|
|
1450
|
+
const existing = params.db.prepare(`SELECT 1
|
|
1451
|
+
FROM panel_query_handle_field_index_meta
|
|
1452
|
+
WHERE panel_id = ?
|
|
1453
|
+
AND source_manifest_key = ?
|
|
1454
|
+
AND metadata_key = ?
|
|
1455
|
+
AND field_schema_key = ?
|
|
1456
|
+
LIMIT 1`).get(params.panel.id, params.indexContext.sourceManifestKey, metadataKey, fieldSchemaKey);
|
|
1457
|
+
if (existing)
|
|
1458
|
+
return metadataKey;
|
|
1459
|
+
const rows = await scanApiJsonlRowsForFieldIndexByRange({
|
|
1460
|
+
panel: params.panel,
|
|
1461
|
+
sourceUrls: params.sourceUrls,
|
|
1462
|
+
authProfile: params.authProfile,
|
|
1463
|
+
workspaceBroker: params.workspaceBroker,
|
|
1464
|
+
nodeId: params.nodeId,
|
|
1465
|
+
datasetSchema: params.datasetSchema,
|
|
1466
|
+
indexContext: params.indexContext,
|
|
1467
|
+
sourceMetadata,
|
|
1468
|
+
});
|
|
1469
|
+
if (!rows)
|
|
1470
|
+
return null;
|
|
1471
|
+
writeQueryHandleFieldIndexRows({
|
|
1472
|
+
context: params.indexContext,
|
|
1473
|
+
metadataKey,
|
|
1474
|
+
datasetSchema: params.datasetSchema,
|
|
1475
|
+
rows,
|
|
1476
|
+
extraFieldNames: params.extraFieldNames,
|
|
1477
|
+
});
|
|
1478
|
+
return metadataKey;
|
|
1479
|
+
}
|
|
1480
|
+
function apiJsonlRangeMetadataKey(sourceUrls, metadata) {
|
|
1481
|
+
return JSON.stringify(metadata.map((source, sourceIndex) => ({
|
|
1482
|
+
url: sourceUrls[sourceIndex],
|
|
1483
|
+
byteLength: source.totalSize,
|
|
1484
|
+
etag: source.etag,
|
|
1485
|
+
lastModified: source.lastModified,
|
|
1486
|
+
})));
|
|
1487
|
+
}
|
|
1488
|
+
async function scanApiJsonlRowsForFieldIndexByRange(params) {
|
|
1489
|
+
void params.panel;
|
|
1490
|
+
const rows = [];
|
|
1491
|
+
let rowIndex = 0;
|
|
1492
|
+
let chunks = 0;
|
|
1493
|
+
const sourceSawRows = new Set();
|
|
1494
|
+
for (const [sourceIndex, url] of params.sourceUrls.entries()) {
|
|
1495
|
+
const metadata = params.sourceMetadata[sourceIndex];
|
|
1496
|
+
if (!metadata)
|
|
1497
|
+
return null;
|
|
1498
|
+
let offset = 0;
|
|
1499
|
+
let hasMore = true;
|
|
1500
|
+
const currentValidator = {
|
|
1501
|
+
url,
|
|
1502
|
+
etag: metadata.etag,
|
|
1503
|
+
lastModified: metadata.lastModified,
|
|
1504
|
+
totalSize: metadata.totalSize,
|
|
1505
|
+
};
|
|
1506
|
+
while (hasMore) {
|
|
1507
|
+
chunks += 1;
|
|
1508
|
+
if (chunks > API_JSONL_MAX_RANGE_CHUNKS_PER_INDEX) {
|
|
1509
|
+
throw new PanelQueryHandleError('api_jsonl range index scan exceeded the per-request work limit.', 413);
|
|
1510
|
+
}
|
|
1511
|
+
const result = await params.workspaceBroker.fetchHttpText(params.nodeId, url, {
|
|
1512
|
+
maxBytes: API_JSONL_MAX_BYTES,
|
|
1513
|
+
...(params.authProfile ? { authProfile: params.authProfile } : {}),
|
|
1514
|
+
rangeStart: offset,
|
|
1515
|
+
rangeLength: API_JSONL_RANGE_CHUNK_BYTES,
|
|
1516
|
+
...withApiJsonlIfRange(currentValidator),
|
|
1517
|
+
});
|
|
1518
|
+
if (result.status !== 206
|
|
1519
|
+
|| result.partialContent !== true
|
|
1520
|
+
|| typeof result.rangeStart !== 'number'
|
|
1521
|
+
|| typeof result.rangeEnd !== 'number'
|
|
1522
|
+
|| result.rangeStart !== offset) {
|
|
1523
|
+
return null;
|
|
1524
|
+
}
|
|
1525
|
+
const checkpointChunk = apiJsonlRangeCheckpointChunk(result);
|
|
1526
|
+
if (!checkpointChunk || !ensureQueryHandleIndexMetadataFresh(params.indexContext, sourceIndex, checkpointChunk)) {
|
|
1527
|
+
return null;
|
|
1528
|
+
}
|
|
1529
|
+
const parsed = parseCompleteJsonlLines(result.content, Boolean(result.hasMore));
|
|
1530
|
+
if (parsed.completeText.length === 0 && result.hasMore) {
|
|
1531
|
+
throw new PanelQueryHandleError('api_jsonl JSONL row exceeds the per-row size limit.', 413);
|
|
1532
|
+
}
|
|
1533
|
+
const baseOffset = result.rangeStart;
|
|
1534
|
+
let consumedBytes = 0;
|
|
1535
|
+
for (const line of parsed.lines) {
|
|
1536
|
+
const lineStartOffset = baseOffset + consumedBytes;
|
|
1537
|
+
consumedBytes += line.byteLength;
|
|
1538
|
+
const trimmed = line.text.trim();
|
|
1539
|
+
if (!trimmed)
|
|
1540
|
+
continue;
|
|
1541
|
+
if (rowIndex >= API_JSONL_MAX_ROWS) {
|
|
1542
|
+
throw new PanelQueryHandleError(`api_jsonl response exceeds row limit (${API_JSONL_MAX_ROWS}).`, 413);
|
|
1543
|
+
}
|
|
1544
|
+
const rawRow = parseJsonlRow(trimmed, rowIndex);
|
|
1545
|
+
const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
|
|
1546
|
+
const isSourceStart = !sourceSawRows.has(sourceIndex);
|
|
1547
|
+
sourceSawRows.add(sourceIndex);
|
|
1548
|
+
recordApiJsonlRangeCheckpoint(params.indexContext, {
|
|
1549
|
+
sourceIndex,
|
|
1550
|
+
chunk: checkpointChunk,
|
|
1551
|
+
rowIndex,
|
|
1552
|
+
byteOffset: lineStartOffset,
|
|
1553
|
+
isSourceStart,
|
|
1554
|
+
});
|
|
1555
|
+
rows.push({ rowIndex: row.rowIndex, fields: row.fields });
|
|
1556
|
+
rowIndex += 1;
|
|
1557
|
+
}
|
|
1558
|
+
offset = baseOffset + consumedBytes;
|
|
1559
|
+
hasMore = result.hasMore === true;
|
|
1560
|
+
if (parsed.lines.length === 0 && !hasMore)
|
|
1561
|
+
break;
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
return rows;
|
|
1565
|
+
}
|
|
1566
|
+
function queryHandleViewIndexQueryKey(options) {
|
|
1567
|
+
return JSON.stringify({
|
|
1568
|
+
sortField: options.sortField ?? null,
|
|
1569
|
+
sortDirection: options.sortDirection ?? null,
|
|
1570
|
+
sortFieldType: options.sortFieldType ?? null,
|
|
1571
|
+
filterField: options.filterField ?? null,
|
|
1572
|
+
filterValue: options.filterValue ?? null,
|
|
1573
|
+
});
|
|
1574
|
+
}
|
|
1575
|
+
function parseQueryHandleViewIndexCursorRequest(queryKey) {
|
|
1576
|
+
try {
|
|
1577
|
+
const parsed = JSON.parse(queryKey);
|
|
1578
|
+
if (typeof parsed.sortField !== 'string')
|
|
1579
|
+
return null;
|
|
1580
|
+
if (parsed.sortDirection !== 'asc' && parsed.sortDirection !== 'desc')
|
|
1581
|
+
return null;
|
|
1582
|
+
return {
|
|
1583
|
+
sortField: parsed.sortField,
|
|
1584
|
+
sortDirection: parsed.sortDirection,
|
|
1585
|
+
...(typeof parsed.filterField === 'string' ? { filterField: parsed.filterField } : {}),
|
|
1586
|
+
...(parsed.filterValue !== null && parsed.filterValue !== undefined ? { filterValue: parsed.filterValue } : {}),
|
|
1587
|
+
};
|
|
1588
|
+
}
|
|
1589
|
+
catch {
|
|
1590
|
+
return null;
|
|
1591
|
+
}
|
|
1592
|
+
}
|
|
1593
|
+
async function loadQueryHandleIndexMetadataKey(params) {
|
|
1594
|
+
for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
|
|
1595
|
+
const existing = params.context.metadataBySourceIndex.get(sourceIndex);
|
|
1596
|
+
if (existing)
|
|
1597
|
+
continue;
|
|
1598
|
+
const metadataChunk = await readQueryHandleChunk({
|
|
1599
|
+
workspaceBroker: params.workspaceBroker,
|
|
1600
|
+
nodeId: params.nodeId,
|
|
1601
|
+
workspaceRoot: params.workspaceRoot,
|
|
1602
|
+
dataSource: params.dataSources[sourceIndex],
|
|
1603
|
+
offset: 0,
|
|
1604
|
+
limit: 1,
|
|
1605
|
+
});
|
|
1606
|
+
if (!ensureQueryHandleIndexMetadataFresh(params.context, sourceIndex, metadataChunk)) {
|
|
1607
|
+
return null;
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
const metadata = params.context.sourceKeys.map((sourceKey, sourceIndex) => {
|
|
1611
|
+
const sourceMetadata = params.context.metadataBySourceIndex.get(sourceIndex);
|
|
1612
|
+
return sourceMetadata ? {
|
|
1613
|
+
sourceKey,
|
|
1614
|
+
fileSize: sourceMetadata.fileSize,
|
|
1615
|
+
modifiedAt: sourceMetadata.modifiedAt,
|
|
1616
|
+
} : null;
|
|
1617
|
+
});
|
|
1618
|
+
return metadata.every(Boolean) ? JSON.stringify(metadata) : null;
|
|
1619
|
+
}
|
|
1620
|
+
async function readCachedQueryHandleViewIndexRows(params) {
|
|
1621
|
+
if (!params.db)
|
|
1622
|
+
return null;
|
|
1623
|
+
const meta = params.db.prepare(`SELECT total_count as totalCount
|
|
1624
|
+
FROM panel_query_handle_view_index_meta
|
|
1625
|
+
WHERE panel_id = ?
|
|
1626
|
+
AND source_manifest_key = ?
|
|
1627
|
+
AND query_key = ?
|
|
1628
|
+
AND metadata_key = ?`).get(params.panel.id, params.sourceManifestKey, params.queryKey, params.metadataKey);
|
|
1629
|
+
if (!meta || !Number.isInteger(meta.totalCount) || meta.totalCount < 0)
|
|
1630
|
+
return null;
|
|
1631
|
+
const start = Math.min(Math.max(0, params.position), meta.totalCount);
|
|
1632
|
+
const expectedRows = Math.min(params.limit, Math.max(0, meta.totalCount - start));
|
|
1633
|
+
if (expectedRows === 0) {
|
|
1634
|
+
return { rows: [], nextCursor: null };
|
|
1635
|
+
}
|
|
1636
|
+
const indexRows = params.db.prepare(`SELECT row_index as rowIndex
|
|
1637
|
+
FROM panel_query_handle_view_index_rows
|
|
1638
|
+
WHERE panel_id = ?
|
|
1639
|
+
AND source_manifest_key = ?
|
|
1640
|
+
AND query_key = ?
|
|
1641
|
+
AND metadata_key = ?
|
|
1642
|
+
AND position >= ?
|
|
1643
|
+
ORDER BY position ASC
|
|
1644
|
+
LIMIT ?`).all(params.panel.id, params.sourceManifestKey, params.queryKey, params.metadataKey, start, expectedRows);
|
|
1645
|
+
if (indexRows.length !== expectedRows || indexRows.some((row) => !Number.isInteger(row.rowIndex))) {
|
|
1646
|
+
return null;
|
|
1647
|
+
}
|
|
1648
|
+
const rowIndices = indexRows.map((row) => row.rowIndex);
|
|
1649
|
+
const selectedRows = params.selectRowsByIndices
|
|
1650
|
+
? await params.selectRowsByIndices(rowIndices)
|
|
1651
|
+
: (await listPanelQueryHandleRowsByIndices({
|
|
1652
|
+
db: params.db,
|
|
1653
|
+
panel: params.panel,
|
|
1654
|
+
conversationManager: params.conversationManager,
|
|
1655
|
+
workspaceBroker: params.workspaceBroker,
|
|
1656
|
+
config: params.config,
|
|
1657
|
+
datasetSchema: params.datasetSchema,
|
|
1658
|
+
rowIndices,
|
|
1659
|
+
})).rows;
|
|
1660
|
+
if (selectedRows.length !== indexRows.length)
|
|
1661
|
+
return null;
|
|
1662
|
+
const nextPosition = start + selectedRows.length;
|
|
1663
|
+
const cursorRequest = parseQueryHandleViewIndexCursorRequest(params.queryKey);
|
|
1664
|
+
if (!cursorRequest)
|
|
1665
|
+
return null;
|
|
1666
|
+
return {
|
|
1667
|
+
rows: selectedRows,
|
|
1668
|
+
nextCursor: nextPosition < meta.totalCount
|
|
1669
|
+
? encodeQueryHandleSortCursor({
|
|
1670
|
+
position: nextPosition,
|
|
1671
|
+
...cursorRequest,
|
|
1672
|
+
})
|
|
1673
|
+
: null,
|
|
1674
|
+
};
|
|
1675
|
+
}
|
|
1676
|
+
function writeQueryHandleViewIndexRows(params) {
|
|
1677
|
+
const now = Date.now();
|
|
1678
|
+
const deleteMeta = params.context.db.prepare(`DELETE FROM panel_query_handle_view_index_meta
|
|
1679
|
+
WHERE panel_id = ?
|
|
1680
|
+
AND source_manifest_key = ?
|
|
1681
|
+
AND query_key = ?`);
|
|
1682
|
+
const deleteRows = params.context.db.prepare(`DELETE FROM panel_query_handle_view_index_rows
|
|
1683
|
+
WHERE panel_id = ?
|
|
1684
|
+
AND source_manifest_key = ?
|
|
1685
|
+
AND query_key = ?`);
|
|
1686
|
+
const insertMeta = params.context.db.prepare(`INSERT INTO panel_query_handle_view_index_meta(
|
|
1687
|
+
panel_id, source_manifest_key, query_key, metadata_key,
|
|
1688
|
+
total_count, created_at, updated_at
|
|
1689
|
+
)
|
|
1690
|
+
VALUES(?, ?, ?, ?, ?, ?, ?)`);
|
|
1691
|
+
const insertRow = params.context.db.prepare(`INSERT INTO panel_query_handle_view_index_rows(
|
|
1692
|
+
panel_id, source_manifest_key, query_key, metadata_key,
|
|
1693
|
+
position, row_index, created_at
|
|
1694
|
+
)
|
|
1695
|
+
VALUES(?, ?, ?, ?, ?, ?, ?)`);
|
|
1696
|
+
params.context.db.transaction(() => {
|
|
1697
|
+
deleteRows.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey);
|
|
1698
|
+
deleteMeta.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey);
|
|
1699
|
+
insertMeta.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey, params.metadataKey, params.rowIndices.length, now, now);
|
|
1700
|
+
params.rowIndices.forEach((rowIndex, position) => {
|
|
1701
|
+
insertRow.run(params.context.panelId, params.context.sourceManifestKey, params.queryKey, params.metadataKey, position, rowIndex, now);
|
|
1702
|
+
});
|
|
1703
|
+
})();
|
|
1704
|
+
}
|
|
1705
|
+
function aggregateFieldIndexNames(options) {
|
|
1706
|
+
return Array.from(new Set([
|
|
1707
|
+
...(options.field ? [options.field] : []),
|
|
1708
|
+
...(options.filterField ? [options.filterField] : []),
|
|
1709
|
+
]));
|
|
1710
|
+
}
|
|
1711
|
+
async function scanQueryHandleRowsForFieldIndex(params) {
|
|
1712
|
+
const rows = [];
|
|
1713
|
+
let rowIndex = 0;
|
|
1714
|
+
let chunks = 0;
|
|
1715
|
+
const sourceSawRows = new Set();
|
|
1716
|
+
for (let sourceIndex = 0; sourceIndex < params.dataSources.length; sourceIndex += 1) {
|
|
1717
|
+
const dataSource = params.dataSources[sourceIndex];
|
|
1718
|
+
let offset = 0;
|
|
1719
|
+
let hasMore = true;
|
|
1720
|
+
while (hasMore) {
|
|
1721
|
+
chunks += 1;
|
|
1722
|
+
if (chunks > QUERY_HANDLE_MAX_CHUNKS_PER_SORT) {
|
|
1723
|
+
throw new PanelQueryHandleError('query_handle aggregate scan exceeded the per-request work limit.', 413);
|
|
1724
|
+
}
|
|
1725
|
+
const chunk = await readQueryHandleChunk({
|
|
1726
|
+
workspaceBroker: params.workspaceBroker,
|
|
1727
|
+
nodeId: params.nodeId,
|
|
1728
|
+
workspaceRoot: params.workspaceRoot,
|
|
1729
|
+
dataSource,
|
|
1730
|
+
offset,
|
|
1731
|
+
limit: QUERY_HANDLE_READ_CHUNK_BYTES,
|
|
1732
|
+
});
|
|
1733
|
+
const content = chunk.content ?? '';
|
|
1734
|
+
const parsed = parseCompleteJsonlLines(content, Boolean(chunk.hasMore));
|
|
1735
|
+
if (parsed.completeText.length === 0 && chunk.hasMore) {
|
|
1736
|
+
throw new PanelQueryHandleError('query_handle JSONL row exceeds the per-row size limit.', 413);
|
|
1737
|
+
}
|
|
1738
|
+
const baseOffset = chunk.offset ?? offset;
|
|
1739
|
+
let consumedBytes = 0;
|
|
1740
|
+
for (const line of parsed.lines) {
|
|
1741
|
+
const lineStartOffset = baseOffset + consumedBytes;
|
|
1742
|
+
consumedBytes += line.byteLength;
|
|
1743
|
+
const trimmed = line.text.trim();
|
|
1744
|
+
if (!trimmed)
|
|
1745
|
+
continue;
|
|
1746
|
+
const rawRow = parseJsonlRow(trimmed, rowIndex);
|
|
1747
|
+
const row = buildPanelRowFromQueryRow(rawRow, rowIndex, params.datasetSchema);
|
|
1748
|
+
const isSourceStart = !sourceSawRows.has(sourceIndex);
|
|
1749
|
+
sourceSawRows.add(sourceIndex);
|
|
1750
|
+
recordQueryHandleCheckpoint(params.indexContext, {
|
|
1751
|
+
sourceIndex,
|
|
1752
|
+
chunk,
|
|
1753
|
+
rowIndex,
|
|
1754
|
+
byteOffset: lineStartOffset,
|
|
1755
|
+
isSourceStart,
|
|
1756
|
+
});
|
|
1757
|
+
rows.push({ rowIndex: row.rowIndex, fields: row.fields });
|
|
1758
|
+
rowIndex += 1;
|
|
1759
|
+
}
|
|
1760
|
+
offset = baseOffset + consumedBytes;
|
|
1761
|
+
hasMore = offset < chunk.size;
|
|
1762
|
+
if (parsed.lines.length === 0 && !hasMore)
|
|
1763
|
+
break;
|
|
1764
|
+
}
|
|
1765
|
+
}
|
|
1766
|
+
if (params.indexContext && params.metadataKey) {
|
|
1767
|
+
writeQueryHandleFieldIndexRows({
|
|
1768
|
+
context: params.indexContext,
|
|
1769
|
+
metadataKey: params.metadataKey,
|
|
1770
|
+
datasetSchema: params.datasetSchema,
|
|
1771
|
+
rows,
|
|
1772
|
+
extraFieldNames: params.extraFieldNames,
|
|
1773
|
+
});
|
|
1774
|
+
}
|
|
1775
|
+
return rows;
|
|
1776
|
+
}
|
|
1777
|
+
function queryHandleFieldIndexFields(datasetSchema, extraFieldNames = []) {
|
|
1778
|
+
const extraFields = new Set(extraFieldNames);
|
|
1779
|
+
return (datasetSchema?.fields ?? []).filter((field) => field.filterable || field.sortable || extraFields.has(field.name));
|
|
1780
|
+
}
|
|
1781
|
+
function queryHandleFieldIndexSchemaKey(datasetSchema, extraFieldNames = []) {
|
|
1782
|
+
const fields = queryHandleFieldIndexFields(datasetSchema, extraFieldNames);
|
|
1783
|
+
if (fields.length === 0)
|
|
1784
|
+
return null;
|
|
1785
|
+
return JSON.stringify(fields.map((field) => ({
|
|
1786
|
+
name: field.name,
|
|
1787
|
+
type: field.type,
|
|
1788
|
+
filterable: Boolean(field.filterable),
|
|
1789
|
+
sortable: Boolean(field.sortable),
|
|
1790
|
+
})));
|
|
1791
|
+
}
|
|
1792
|
+
function readAggregateFromQueryHandleFieldIndex(params) {
|
|
1793
|
+
if (!params.db)
|
|
1794
|
+
return null;
|
|
1795
|
+
void params.sourceKind;
|
|
1796
|
+
const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
|
|
1797
|
+
if (!fieldSchemaKey)
|
|
1798
|
+
return null;
|
|
1799
|
+
const meta = params.db.prepare(`SELECT row_count as rowCount
|
|
1800
|
+
FROM panel_query_handle_field_index_meta
|
|
1801
|
+
WHERE panel_id = ?
|
|
1802
|
+
AND source_manifest_key = ?
|
|
1803
|
+
AND metadata_key = ?
|
|
1804
|
+
AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
|
|
1805
|
+
if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
|
|
1806
|
+
return null;
|
|
1807
|
+
const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
|
|
1808
|
+
if (params.options.filterField && !filterField)
|
|
1809
|
+
return null;
|
|
1810
|
+
const filter = filterField && params.options.filterValue !== undefined
|
|
1811
|
+
? normalizeQueryHandleIndexedValue(filterField, params.options.filterValue)
|
|
1812
|
+
: null;
|
|
1813
|
+
if (filterField && !filter?.hasValue) {
|
|
1814
|
+
return emptyAggregateResult(params.options);
|
|
1815
|
+
}
|
|
1816
|
+
const totalCount = filterField && filter
|
|
1817
|
+
? countRowsMatchingQueryHandleFieldFilter({
|
|
1818
|
+
db: params.db,
|
|
1819
|
+
panelId: params.panel.id,
|
|
1820
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
1821
|
+
metadataKey: params.metadataKey,
|
|
1822
|
+
fieldSchemaKey,
|
|
1823
|
+
filterField,
|
|
1824
|
+
filter,
|
|
1825
|
+
})
|
|
1826
|
+
: meta.rowCount;
|
|
1827
|
+
if (params.options.op === 'count' && !params.options.field) {
|
|
1828
|
+
return {
|
|
1829
|
+
op: params.options.op,
|
|
1830
|
+
field: null,
|
|
1831
|
+
value: totalCount,
|
|
1832
|
+
count: totalCount,
|
|
1833
|
+
totalCount,
|
|
1834
|
+
hasValue: true,
|
|
1835
|
+
};
|
|
1836
|
+
}
|
|
1837
|
+
if (!params.options.field)
|
|
1838
|
+
return emptyAggregateResult(params.options, totalCount);
|
|
1839
|
+
const aggregateField = (params.datasetSchema?.fields ?? []).find((field) => field.name === params.options.field);
|
|
1840
|
+
if (!aggregateField)
|
|
1841
|
+
return null;
|
|
1842
|
+
if (!filterField && params.options.filterValue === undefined) {
|
|
1843
|
+
const summary = readAggregateFromQueryHandleAggregateIndex({
|
|
1844
|
+
db: params.db,
|
|
1845
|
+
panelId: params.panel.id,
|
|
1846
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
1847
|
+
metadataKey: params.metadataKey,
|
|
1848
|
+
fieldSchemaKey,
|
|
1849
|
+
aggregateField,
|
|
1850
|
+
op: params.options.op,
|
|
1851
|
+
});
|
|
1852
|
+
if (summary) {
|
|
1853
|
+
return {
|
|
1854
|
+
op: params.options.op,
|
|
1855
|
+
field: aggregateField.name,
|
|
1856
|
+
value: summary.value,
|
|
1857
|
+
count: summary.count,
|
|
1858
|
+
totalCount,
|
|
1859
|
+
hasValue: params.options.op === 'count' || summary.value !== null,
|
|
1860
|
+
};
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
const aggregate = queryAggregateFieldIndexRows({
|
|
1864
|
+
db: params.db,
|
|
1865
|
+
panelId: params.panel.id,
|
|
1866
|
+
sourceManifestKey: params.sourceManifestKey,
|
|
1867
|
+
metadataKey: params.metadataKey,
|
|
1868
|
+
fieldSchemaKey,
|
|
1869
|
+
aggregateField,
|
|
1870
|
+
filterField,
|
|
1871
|
+
filter,
|
|
1872
|
+
op: params.options.op,
|
|
1873
|
+
});
|
|
1874
|
+
return {
|
|
1875
|
+
op: params.options.op,
|
|
1876
|
+
field: aggregateField.name,
|
|
1877
|
+
value: aggregate.value,
|
|
1878
|
+
count: aggregate.count,
|
|
1879
|
+
totalCount,
|
|
1880
|
+
hasValue: params.options.op === 'count' || aggregate.value !== null,
|
|
1881
|
+
};
|
|
1882
|
+
}
|
|
1883
|
+
function readAggregateFromQueryHandleAggregateIndex(params) {
|
|
1884
|
+
const row = params.db.prepare(`SELECT value_count as valueCount,
|
|
1885
|
+
non_empty_count as nonEmptyCount,
|
|
1886
|
+
numeric_count as numericCount,
|
|
1887
|
+
numeric_sum as numericSum,
|
|
1888
|
+
numeric_min as numericMin,
|
|
1889
|
+
numeric_max as numericMax
|
|
1890
|
+
FROM panel_query_handle_field_aggregate_index
|
|
1891
|
+
WHERE panel_id = ?
|
|
1892
|
+
AND source_manifest_key = ?
|
|
1893
|
+
AND metadata_key = ?
|
|
1894
|
+
AND field_schema_key = ?
|
|
1895
|
+
AND field_name = ?
|
|
1896
|
+
LIMIT 1`).get(params.panelId, params.sourceManifestKey, params.metadataKey, params.fieldSchemaKey, params.aggregateField.name);
|
|
1897
|
+
if (!row)
|
|
1898
|
+
return null;
|
|
1899
|
+
if (params.op === 'count') {
|
|
1900
|
+
const count = params.aggregateField.type === 'string' ? row.nonEmptyCount : row.valueCount;
|
|
1901
|
+
return { value: count, count };
|
|
1902
|
+
}
|
|
1903
|
+
if (params.aggregateField.type !== 'number' || row.numericCount <= 0) {
|
|
1904
|
+
return { value: null, count: 0 };
|
|
1905
|
+
}
|
|
1906
|
+
if (params.op === 'sum')
|
|
1907
|
+
return { value: row.numericSum, count: row.numericCount };
|
|
1908
|
+
if (params.op === 'avg')
|
|
1909
|
+
return { value: row.numericSum / row.numericCount, count: row.numericCount };
|
|
1910
|
+
if (params.op === 'min')
|
|
1911
|
+
return { value: row.numericMin, count: row.numericCount };
|
|
1912
|
+
return { value: row.numericMax, count: row.numericCount };
|
|
1913
|
+
}
|
|
1914
|
+
function emptyAggregateResult(options, totalCount = 0) {
|
|
1915
|
+
if (options.op === 'count') {
|
|
1916
|
+
return {
|
|
1917
|
+
op: options.op,
|
|
1918
|
+
field: options.field ?? null,
|
|
1919
|
+
value: 0,
|
|
1920
|
+
count: 0,
|
|
1921
|
+
totalCount,
|
|
1922
|
+
hasValue: true,
|
|
1923
|
+
};
|
|
1924
|
+
}
|
|
1925
|
+
return {
|
|
1926
|
+
op: options.op,
|
|
1927
|
+
field: options.field ?? null,
|
|
1928
|
+
value: null,
|
|
1929
|
+
count: 0,
|
|
1930
|
+
totalCount,
|
|
1931
|
+
hasValue: false,
|
|
1932
|
+
};
|
|
1933
|
+
}
|
|
1934
|
+
function countRowsMatchingQueryHandleFieldFilter(params) {
|
|
1935
|
+
const filterColumn = queryHandleFieldValueColumn(params.filterField);
|
|
1936
|
+
const filterValue = filterColumn === 'string_value'
|
|
1937
|
+
? params.filter.stringValue
|
|
1938
|
+
: filterColumn === 'number_value'
|
|
1939
|
+
? params.filter.numberValue
|
|
1940
|
+
: params.filter.booleanValue;
|
|
1941
|
+
const row = params.db.prepare(`SELECT COUNT(*) as totalCount
|
|
1942
|
+
FROM panel_query_handle_field_index_rows
|
|
1943
|
+
WHERE panel_id = ?
|
|
1944
|
+
AND source_manifest_key = ?
|
|
1945
|
+
AND metadata_key = ?
|
|
1946
|
+
AND field_schema_key = ?
|
|
1947
|
+
AND field_name = ?
|
|
1948
|
+
AND has_value = 1
|
|
1949
|
+
AND ${filterColumn} = ?`).get(params.panelId, params.sourceManifestKey, params.metadataKey, params.fieldSchemaKey, params.filterField.name, filterValue);
|
|
1950
|
+
return row?.totalCount ?? 0;
|
|
1951
|
+
}
|
|
1952
|
+
function queryAggregateFieldIndexRows(params) {
|
|
1953
|
+
const filterColumn = params.filterField ? queryHandleFieldValueColumn(params.filterField) : null;
|
|
1954
|
+
const filterValue = params.filterField && params.filter && filterColumn
|
|
1955
|
+
? filterColumn === 'string_value'
|
|
1956
|
+
? params.filter.stringValue
|
|
1957
|
+
: filterColumn === 'number_value'
|
|
1958
|
+
? params.filter.numberValue
|
|
1959
|
+
: params.filter.booleanValue
|
|
1960
|
+
: null;
|
|
1961
|
+
const filterJoin = params.filterField
|
|
1962
|
+
? `JOIN panel_query_handle_field_index_rows f
|
|
1963
|
+
ON f.panel_id = a.panel_id
|
|
1964
|
+
AND f.source_manifest_key = a.source_manifest_key
|
|
1965
|
+
AND f.metadata_key = a.metadata_key
|
|
1966
|
+
AND f.field_schema_key = a.field_schema_key
|
|
1967
|
+
AND f.row_index = a.row_index
|
|
1968
|
+
AND f.field_name = ?`
|
|
1969
|
+
: '';
|
|
1970
|
+
const filterWhere = params.filterField && filterColumn
|
|
1971
|
+
? `AND f.has_value = 1 AND f.${filterColumn} = ?`
|
|
1972
|
+
: '';
|
|
1973
|
+
const filterParams = params.filterField && filterValue !== null
|
|
1974
|
+
? [params.filterField.name, filterValue]
|
|
1975
|
+
: [];
|
|
1976
|
+
const nonEmptyPredicate = params.aggregateField.type === 'string'
|
|
1977
|
+
? "a.has_value = 1 AND trim(a.string_value) != ''"
|
|
1978
|
+
: 'a.has_value = 1';
|
|
1979
|
+
const aggregatePredicate = params.op === 'count'
|
|
1980
|
+
? nonEmptyPredicate
|
|
1981
|
+
: "a.has_value = 1 AND a.field_type = 'number'";
|
|
1982
|
+
const aggregateExpression = params.op === 'count'
|
|
1983
|
+
? 'COUNT(*)'
|
|
1984
|
+
: params.op === 'avg'
|
|
1985
|
+
? 'AVG(a.number_value)'
|
|
1986
|
+
: `${params.op.toUpperCase()}(a.number_value)`;
|
|
1987
|
+
const sql = `
|
|
1988
|
+
SELECT ${aggregateExpression} as value, COUNT(*) as count
|
|
1989
|
+
FROM panel_query_handle_field_index_rows a
|
|
1990
|
+
${filterJoin}
|
|
1991
|
+
WHERE a.panel_id = ?
|
|
1992
|
+
AND a.source_manifest_key = ?
|
|
1993
|
+
AND a.metadata_key = ?
|
|
1994
|
+
AND a.field_schema_key = ?
|
|
1995
|
+
AND a.field_name = ?
|
|
1996
|
+
AND ${aggregatePredicate}
|
|
1997
|
+
${filterWhere}`;
|
|
1998
|
+
const paramsForQuery = params.filterField
|
|
1999
|
+
? [
|
|
2000
|
+
...filterParams.slice(0, 1),
|
|
2001
|
+
params.panelId,
|
|
2002
|
+
params.sourceManifestKey,
|
|
2003
|
+
params.metadataKey,
|
|
2004
|
+
params.fieldSchemaKey,
|
|
2005
|
+
params.aggregateField.name,
|
|
2006
|
+
...filterParams.slice(1),
|
|
2007
|
+
]
|
|
2008
|
+
: [
|
|
2009
|
+
params.panelId,
|
|
2010
|
+
params.sourceManifestKey,
|
|
2011
|
+
params.metadataKey,
|
|
2012
|
+
params.fieldSchemaKey,
|
|
2013
|
+
params.aggregateField.name,
|
|
2014
|
+
];
|
|
2015
|
+
const row = params.db.prepare(sql).get(...paramsForQuery);
|
|
2016
|
+
const value = typeof row?.value === 'number' && Number.isFinite(row.value) ? row.value : null;
|
|
2017
|
+
return {
|
|
2018
|
+
value: params.op === 'count' ? (row?.count ?? 0) : value,
|
|
2019
|
+
count: row?.count ?? 0,
|
|
2020
|
+
};
|
|
2021
|
+
}
|
|
2022
|
+
function getQueryHandleIndexedField(datasetSchema, fieldName, purpose) {
|
|
2023
|
+
if (!fieldName)
|
|
2024
|
+
return null;
|
|
2025
|
+
const field = (datasetSchema?.fields ?? []).find((candidate) => candidate.name === fieldName);
|
|
2026
|
+
if (!field)
|
|
2027
|
+
return null;
|
|
2028
|
+
if (purpose === 'filter' && !field.filterable)
|
|
2029
|
+
return null;
|
|
2030
|
+
if (purpose === 'sort' && !field.sortable)
|
|
2031
|
+
return null;
|
|
2032
|
+
return field;
|
|
2033
|
+
}
|
|
2034
|
+
function queryHandleFieldValueColumn(field) {
|
|
2035
|
+
if (field.type === 'number')
|
|
2036
|
+
return 'number_value';
|
|
2037
|
+
if (field.type === 'boolean')
|
|
2038
|
+
return 'boolean_value';
|
|
2039
|
+
return 'string_value';
|
|
2040
|
+
}
|
|
2041
|
+
function normalizeQueryHandleIndexedValue(field, value) {
|
|
2042
|
+
if (field.type === 'number') {
|
|
2043
|
+
return typeof value === 'number' && Number.isFinite(value)
|
|
2044
|
+
? { hasValue: 1, stringValue: '', numberValue: value, booleanValue: 0 }
|
|
2045
|
+
: { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
|
|
2046
|
+
}
|
|
2047
|
+
if (field.type === 'boolean') {
|
|
2048
|
+
return typeof value === 'boolean'
|
|
2049
|
+
? { hasValue: 1, stringValue: '', numberValue: 0, booleanValue: value ? 1 : 0 }
|
|
2050
|
+
: { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
|
|
2051
|
+
}
|
|
2052
|
+
return typeof value === 'string'
|
|
2053
|
+
? { hasValue: 1, stringValue: value, numberValue: 0, booleanValue: 0 }
|
|
2054
|
+
: { hasValue: 0, stringValue: '', numberValue: 0, booleanValue: 0 };
|
|
2055
|
+
}
|
|
2056
|
+
async function readQueryHandleRowsFromFieldIndex(params) {
|
|
2057
|
+
if (!params.db || !params.options.sortField || !params.options.sortDirection)
|
|
2058
|
+
return null;
|
|
2059
|
+
const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema);
|
|
2060
|
+
if (!fieldSchemaKey)
|
|
2061
|
+
return null;
|
|
2062
|
+
const sortField = getQueryHandleIndexedField(params.datasetSchema, params.options.sortField, 'sort');
|
|
2063
|
+
if (!sortField)
|
|
2064
|
+
return null;
|
|
2065
|
+
const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
|
|
2066
|
+
if (params.options.filterField && !filterField)
|
|
2067
|
+
return null;
|
|
2068
|
+
const meta = params.db.prepare(`SELECT row_count as rowCount
|
|
2069
|
+
FROM panel_query_handle_field_index_meta
|
|
2070
|
+
WHERE panel_id = ?
|
|
2071
|
+
AND source_manifest_key = ?
|
|
2072
|
+
AND metadata_key = ?
|
|
2073
|
+
AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
|
|
2074
|
+
if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
|
|
2075
|
+
return null;
|
|
2076
|
+
const sortColumn = queryHandleFieldValueColumn(sortField);
|
|
2077
|
+
const sortDirection = params.options.sortDirection === 'desc' ? 'DESC' : 'ASC';
|
|
2078
|
+
const tieDirection = params.options.sortDirection === 'desc' ? 'DESC' : 'ASC';
|
|
2079
|
+
const start = Math.max(0, params.position);
|
|
2080
|
+
const filter = filterField && params.options.filterValue !== undefined
|
|
2081
|
+
? normalizeQueryHandleIndexedValue(filterField, params.options.filterValue)
|
|
2082
|
+
: null;
|
|
2083
|
+
if (filterField && !filter?.hasValue)
|
|
2084
|
+
return { rows: [], nextCursor: null };
|
|
2085
|
+
const filterColumn = filterField ? queryHandleFieldValueColumn(filterField) : null;
|
|
2086
|
+
const baseParams = [
|
|
2087
|
+
params.panel.id,
|
|
2088
|
+
params.sourceManifestKey,
|
|
2089
|
+
params.metadataKey,
|
|
2090
|
+
fieldSchemaKey,
|
|
2091
|
+
sortField.name,
|
|
2092
|
+
];
|
|
2093
|
+
const filterJoin = filterField
|
|
2094
|
+
? `JOIN panel_query_handle_field_index_rows f
|
|
2095
|
+
ON f.panel_id = s.panel_id
|
|
2096
|
+
AND f.source_manifest_key = s.source_manifest_key
|
|
2097
|
+
AND f.metadata_key = s.metadata_key
|
|
2098
|
+
AND f.field_schema_key = s.field_schema_key
|
|
2099
|
+
AND f.row_index = s.row_index
|
|
2100
|
+
AND f.field_name = ?`
|
|
2101
|
+
: '';
|
|
2102
|
+
const filterWhere = filterField && filterColumn && filter
|
|
2103
|
+
? `AND f.has_value = 1 AND f.${filterColumn} = ?`
|
|
2104
|
+
: '';
|
|
2105
|
+
const filterParams = filterField && filterColumn && filter
|
|
2106
|
+
? [filterField.name, filterColumn === 'string_value' ? filter.stringValue : filterColumn === 'number_value' ? filter.numberValue : filter.booleanValue]
|
|
2107
|
+
: [];
|
|
2108
|
+
const countSql = `
|
|
2109
|
+
SELECT COUNT(*) as totalCount
|
|
2110
|
+
FROM panel_query_handle_field_index_rows s
|
|
2111
|
+
${filterJoin}
|
|
2112
|
+
WHERE s.panel_id = ?
|
|
2113
|
+
AND s.source_manifest_key = ?
|
|
2114
|
+
AND s.metadata_key = ?
|
|
2115
|
+
AND s.field_schema_key = ?
|
|
2116
|
+
AND s.field_name = ?
|
|
2117
|
+
${filterWhere}`;
|
|
2118
|
+
const countParams = filterField
|
|
2119
|
+
? [...filterParams.slice(0, 1), ...baseParams, ...filterParams.slice(1)]
|
|
2120
|
+
: baseParams;
|
|
2121
|
+
const count = params.db.prepare(countSql).get(...countParams);
|
|
2122
|
+
const totalCount = count?.totalCount ?? 0;
|
|
2123
|
+
if (totalCount <= start)
|
|
2124
|
+
return { rows: [], nextCursor: null };
|
|
2125
|
+
const rowSql = `
|
|
2126
|
+
SELECT s.row_index as rowIndex
|
|
2127
|
+
FROM panel_query_handle_field_index_rows s
|
|
2128
|
+
${filterJoin}
|
|
2129
|
+
WHERE s.panel_id = ?
|
|
2130
|
+
AND s.source_manifest_key = ?
|
|
2131
|
+
AND s.metadata_key = ?
|
|
2132
|
+
AND s.field_schema_key = ?
|
|
2133
|
+
AND s.field_name = ?
|
|
2134
|
+
${filterWhere}
|
|
2135
|
+
ORDER BY s.${sortColumn} ${sortDirection}, s.row_index ${tieDirection}
|
|
2136
|
+
LIMIT ? OFFSET ?`;
|
|
2137
|
+
const rowParams = filterField
|
|
2138
|
+
? [...filterParams.slice(0, 1), ...baseParams, ...filterParams.slice(1), params.limit, start]
|
|
2139
|
+
: [...baseParams, params.limit, start];
|
|
2140
|
+
const indexRows = params.db.prepare(rowSql).all(...rowParams);
|
|
2141
|
+
if (indexRows.some((row) => !Number.isInteger(row.rowIndex)))
|
|
2142
|
+
return null;
|
|
2143
|
+
const rowIndices = indexRows.map((row) => row.rowIndex);
|
|
2144
|
+
const selectedRows = params.selectRowsByIndices
|
|
2145
|
+
? await params.selectRowsByIndices(rowIndices)
|
|
2146
|
+
: (await listPanelQueryHandleRowsByIndices({
|
|
2147
|
+
db: params.db,
|
|
2148
|
+
panel: params.panel,
|
|
2149
|
+
conversationManager: params.conversationManager,
|
|
2150
|
+
workspaceBroker: params.workspaceBroker,
|
|
2151
|
+
config: params.config,
|
|
2152
|
+
datasetSchema: params.datasetSchema,
|
|
2153
|
+
rowIndices,
|
|
2154
|
+
})).rows;
|
|
2155
|
+
if (selectedRows.length !== indexRows.length)
|
|
2156
|
+
return null;
|
|
2157
|
+
const nextPosition = start + selectedRows.length;
|
|
2158
|
+
return {
|
|
2159
|
+
rows: selectedRows,
|
|
2160
|
+
nextCursor: nextPosition < totalCount
|
|
2161
|
+
? encodeQueryHandleSortCursor({
|
|
2162
|
+
position: nextPosition,
|
|
2163
|
+
sortField: sortField.name,
|
|
2164
|
+
sortDirection: params.options.sortDirection,
|
|
2165
|
+
filterField: filterField?.name,
|
|
2166
|
+
filterValue: params.options.filterValue,
|
|
2167
|
+
})
|
|
2168
|
+
: null,
|
|
2169
|
+
};
|
|
2170
|
+
}
|
|
2171
|
+
async function readFilteredQueryHandleRowsFromFieldIndex(params) {
|
|
2172
|
+
if (!params.db || !params.options.filterField || params.options.filterValue === undefined)
|
|
2173
|
+
return null;
|
|
2174
|
+
const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema);
|
|
2175
|
+
if (!fieldSchemaKey)
|
|
2176
|
+
return null;
|
|
2177
|
+
const filterField = getQueryHandleIndexedField(params.datasetSchema, params.options.filterField, 'filter');
|
|
2178
|
+
if (!filterField)
|
|
2179
|
+
return null;
|
|
2180
|
+
const meta = params.db.prepare(`SELECT row_count as rowCount
|
|
2181
|
+
FROM panel_query_handle_field_index_meta
|
|
2182
|
+
WHERE panel_id = ?
|
|
2183
|
+
AND source_manifest_key = ?
|
|
2184
|
+
AND metadata_key = ?
|
|
2185
|
+
AND field_schema_key = ?`).get(params.panel.id, params.sourceManifestKey, params.metadataKey, fieldSchemaKey);
|
|
2186
|
+
if (!meta || !Number.isInteger(meta.rowCount) || meta.rowCount < 0)
|
|
2187
|
+
return null;
|
|
2188
|
+
const filter = normalizeQueryHandleIndexedValue(filterField, params.options.filterValue);
|
|
2189
|
+
if (!filter.hasValue)
|
|
2190
|
+
return { rows: [], nextCursor: null };
|
|
2191
|
+
const filterColumn = queryHandleFieldValueColumn(filterField);
|
|
2192
|
+
const filterValue = filterColumn === 'string_value'
|
|
2193
|
+
? filter.stringValue
|
|
2194
|
+
: filterColumn === 'number_value'
|
|
2195
|
+
? filter.numberValue
|
|
2196
|
+
: filter.booleanValue;
|
|
2197
|
+
const start = Math.max(0, params.position);
|
|
2198
|
+
const baseParams = [
|
|
2199
|
+
params.panel.id,
|
|
2200
|
+
params.sourceManifestKey,
|
|
2201
|
+
params.metadataKey,
|
|
2202
|
+
fieldSchemaKey,
|
|
2203
|
+
filterField.name,
|
|
2204
|
+
filterValue,
|
|
2205
|
+
];
|
|
2206
|
+
const count = params.db.prepare(`SELECT COUNT(*) as totalCount
|
|
2207
|
+
FROM panel_query_handle_field_index_rows
|
|
2208
|
+
WHERE panel_id = ?
|
|
2209
|
+
AND source_manifest_key = ?
|
|
2210
|
+
AND metadata_key = ?
|
|
2211
|
+
AND field_schema_key = ?
|
|
2212
|
+
AND field_name = ?
|
|
2213
|
+
AND has_value = 1
|
|
2214
|
+
AND ${filterColumn} = ?`).get(...baseParams);
|
|
2215
|
+
const totalCount = count?.totalCount ?? 0;
|
|
2216
|
+
if (totalCount <= start)
|
|
2217
|
+
return { rows: [], nextCursor: null };
|
|
2218
|
+
const indexRows = params.db.prepare(`SELECT row_index as rowIndex
|
|
2219
|
+
FROM panel_query_handle_field_index_rows
|
|
2220
|
+
WHERE panel_id = ?
|
|
2221
|
+
AND source_manifest_key = ?
|
|
2222
|
+
AND metadata_key = ?
|
|
2223
|
+
AND field_schema_key = ?
|
|
2224
|
+
AND field_name = ?
|
|
2225
|
+
AND has_value = 1
|
|
2226
|
+
AND ${filterColumn} = ?
|
|
2227
|
+
ORDER BY row_index ASC
|
|
2228
|
+
LIMIT ? OFFSET ?`).all(...baseParams, params.limit, start);
|
|
2229
|
+
if (indexRows.some((row) => !Number.isInteger(row.rowIndex)))
|
|
2230
|
+
return null;
|
|
2231
|
+
const rowIndices = indexRows.map((row) => row.rowIndex);
|
|
2232
|
+
const selectedRows = params.selectRowsByIndices
|
|
2233
|
+
? await params.selectRowsByIndices(rowIndices)
|
|
2234
|
+
: (await listPanelQueryHandleRowsByIndices({
|
|
2235
|
+
db: params.db,
|
|
2236
|
+
panel: params.panel,
|
|
2237
|
+
conversationManager: params.conversationManager,
|
|
2238
|
+
workspaceBroker: params.workspaceBroker,
|
|
2239
|
+
config: params.config,
|
|
2240
|
+
datasetSchema: params.datasetSchema,
|
|
2241
|
+
rowIndices,
|
|
2242
|
+
})).rows;
|
|
2243
|
+
if (selectedRows.length !== indexRows.length)
|
|
2244
|
+
return null;
|
|
2245
|
+
const nextPosition = start + selectedRows.length;
|
|
2246
|
+
return {
|
|
2247
|
+
rows: selectedRows,
|
|
2248
|
+
nextCursor: nextPosition < totalCount
|
|
2249
|
+
? encodeQueryHandlePositionCursor({
|
|
2250
|
+
position: nextPosition,
|
|
2251
|
+
sourceKind: params.sourceKind ?? 'query_handle_field_index',
|
|
2252
|
+
filterField: filterField.name,
|
|
2253
|
+
filterValue: params.options.filterValue,
|
|
2254
|
+
})
|
|
2255
|
+
: null,
|
|
2256
|
+
};
|
|
2257
|
+
}
|
|
2258
|
+
function writeQueryHandleFieldIndexRows(params) {
|
|
2259
|
+
const fields = queryHandleFieldIndexFields(params.datasetSchema, params.extraFieldNames);
|
|
2260
|
+
const fieldSchemaKey = queryHandleFieldIndexSchemaKey(params.datasetSchema, params.extraFieldNames);
|
|
2261
|
+
if (!fieldSchemaKey || fields.length === 0)
|
|
2262
|
+
return;
|
|
2263
|
+
const now = Date.now();
|
|
2264
|
+
const deleteMeta = params.context.db.prepare(`DELETE FROM panel_query_handle_field_index_meta
|
|
2265
|
+
WHERE panel_id = ?
|
|
2266
|
+
AND source_manifest_key = ?
|
|
2267
|
+
AND field_schema_key = ?`);
|
|
2268
|
+
const deleteRows = params.context.db.prepare(`DELETE FROM panel_query_handle_field_index_rows
|
|
2269
|
+
WHERE panel_id = ?
|
|
2270
|
+
AND source_manifest_key = ?
|
|
2271
|
+
AND field_schema_key = ?`);
|
|
2272
|
+
const deleteAggregates = params.context.db.prepare(`DELETE FROM panel_query_handle_field_aggregate_index
|
|
2273
|
+
WHERE panel_id = ?
|
|
2274
|
+
AND source_manifest_key = ?
|
|
2275
|
+
AND field_schema_key = ?`);
|
|
2276
|
+
const insertMeta = params.context.db.prepare(`INSERT INTO panel_query_handle_field_index_meta(
|
|
2277
|
+
panel_id, source_manifest_key, metadata_key, field_schema_key,
|
|
2278
|
+
row_count, created_at, updated_at
|
|
2279
|
+
)
|
|
2280
|
+
VALUES(?, ?, ?, ?, ?, ?, ?)`);
|
|
2281
|
+
const insertRow = params.context.db.prepare(`INSERT INTO panel_query_handle_field_index_rows(
|
|
2282
|
+
panel_id, source_manifest_key, metadata_key, field_schema_key,
|
|
2283
|
+
field_name, row_index, field_type, has_value,
|
|
2284
|
+
string_value, number_value, boolean_value, created_at
|
|
2285
|
+
)
|
|
2286
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2287
|
+
const insertAggregate = params.context.db.prepare(`INSERT INTO panel_query_handle_field_aggregate_index(
|
|
2288
|
+
panel_id, source_manifest_key, metadata_key, field_schema_key,
|
|
2289
|
+
field_name, field_type, value_count, non_empty_count,
|
|
2290
|
+
numeric_count, numeric_sum, numeric_min, numeric_max, created_at
|
|
2291
|
+
)
|
|
2292
|
+
VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`);
|
|
2293
|
+
const aggregateSummaries = buildQueryHandleFieldAggregateSummaries({
|
|
2294
|
+
datasetSchema: params.datasetSchema,
|
|
2295
|
+
rows: params.rows,
|
|
2296
|
+
extraFieldNames: params.extraFieldNames,
|
|
2297
|
+
});
|
|
2298
|
+
params.context.db.transaction(() => {
|
|
2299
|
+
deleteRows.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
|
|
2300
|
+
deleteAggregates.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
|
|
2301
|
+
deleteMeta.run(params.context.panelId, params.context.sourceManifestKey, fieldSchemaKey);
|
|
2302
|
+
insertMeta.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, params.rows.length, now, now);
|
|
2303
|
+
for (const row of params.rows) {
|
|
2304
|
+
for (const field of fields) {
|
|
2305
|
+
const indexedValue = normalizeQueryHandleIndexedValue(field, getPanelFieldValue(row.fields, field.name));
|
|
2306
|
+
insertRow.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, field.name, row.rowIndex, field.type, indexedValue.hasValue, indexedValue.stringValue, indexedValue.numberValue, indexedValue.booleanValue, now);
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
for (const summary of aggregateSummaries) {
|
|
2310
|
+
insertAggregate.run(params.context.panelId, params.context.sourceManifestKey, params.metadataKey, fieldSchemaKey, summary.fieldName, summary.fieldType, summary.valueCount, summary.nonEmptyCount, summary.numericCount, summary.numericSum, summary.numericMin, summary.numericMax, now);
|
|
2311
|
+
}
|
|
2312
|
+
})();
|
|
2313
|
+
}
|
|
2314
|
+
function buildQueryHandleFieldAggregateSummaries(params) {
|
|
2315
|
+
return queryHandleFieldIndexFields(params.datasetSchema, params.extraFieldNames).map((field) => {
|
|
2316
|
+
const summary = {
|
|
2317
|
+
fieldName: field.name,
|
|
2318
|
+
fieldType: field.type,
|
|
2319
|
+
valueCount: 0,
|
|
2320
|
+
nonEmptyCount: 0,
|
|
2321
|
+
numericCount: 0,
|
|
2322
|
+
numericSum: 0,
|
|
2323
|
+
numericMin: null,
|
|
2324
|
+
numericMax: null,
|
|
2325
|
+
};
|
|
2326
|
+
for (const row of params.rows) {
|
|
2327
|
+
const indexedValue = normalizeQueryHandleIndexedValue(field, getPanelFieldValue(row.fields, field.name));
|
|
2328
|
+
if (!indexedValue.hasValue)
|
|
2329
|
+
continue;
|
|
2330
|
+
summary.valueCount += 1;
|
|
2331
|
+
if (field.type === 'string') {
|
|
2332
|
+
if (indexedValue.stringValue.trim() !== '')
|
|
2333
|
+
summary.nonEmptyCount += 1;
|
|
2334
|
+
}
|
|
2335
|
+
else {
|
|
2336
|
+
summary.nonEmptyCount += 1;
|
|
2337
|
+
}
|
|
2338
|
+
if (field.type === 'number') {
|
|
2339
|
+
summary.numericCount += 1;
|
|
2340
|
+
summary.numericSum += indexedValue.numberValue;
|
|
2341
|
+
summary.numericMin = summary.numericMin === null
|
|
2342
|
+
? indexedValue.numberValue
|
|
2343
|
+
: Math.min(summary.numericMin, indexedValue.numberValue);
|
|
2344
|
+
summary.numericMax = summary.numericMax === null
|
|
2345
|
+
? indexedValue.numberValue
|
|
2346
|
+
: Math.max(summary.numericMax, indexedValue.numberValue);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
return summary;
|
|
2350
|
+
});
|
|
2351
|
+
}
|
|
2352
|
+
function recordApiJsonlRangeCheckpoint(context, params) {
|
|
2353
|
+
if (!params.chunk || params.chunk.modifiedAt == null)
|
|
2354
|
+
return;
|
|
2355
|
+
recordQueryHandleCheckpoint(context, {
|
|
2356
|
+
sourceIndex: params.sourceIndex,
|
|
2357
|
+
chunk: params.chunk,
|
|
2358
|
+
rowIndex: params.rowIndex,
|
|
2359
|
+
byteOffset: params.byteOffset,
|
|
2360
|
+
isSourceStart: params.isSourceStart,
|
|
2361
|
+
});
|
|
2362
|
+
}
|
|
2363
|
+
function recordQueryHandleCheckpoint(context, params) {
|
|
2364
|
+
if (!context || context.sourceKeys[params.sourceIndex] == null)
|
|
2365
|
+
return;
|
|
2366
|
+
if (!params.isSourceStart
|
|
2367
|
+
&& params.rowIndex !== 0
|
|
2368
|
+
&& params.rowIndex % QUERY_HANDLE_INDEX_INTERVAL_ROWS !== 0) {
|
|
2369
|
+
return;
|
|
2370
|
+
}
|
|
2371
|
+
if (!ensureQueryHandleIndexMetadataFresh(context, params.sourceIndex, params.chunk))
|
|
2372
|
+
return;
|
|
2373
|
+
const metadata = context.metadataBySourceIndex.get(params.sourceIndex);
|
|
2374
|
+
if (!metadata)
|
|
2375
|
+
return;
|
|
2376
|
+
const now = Date.now();
|
|
2377
|
+
context.db.prepare(`INSERT OR REPLACE INTO panel_query_handle_index(
|
|
2378
|
+
panel_id, source_manifest_key, source_index, source_key, file_size, modified_at,
|
|
2379
|
+
row_index, byte_offset, created_at, updated_at
|
|
2380
|
+
)
|
|
2381
|
+
VALUES(
|
|
2382
|
+
?, ?, ?, ?, ?, ?,
|
|
2383
|
+
?, ?,
|
|
2384
|
+
COALESCE((
|
|
2385
|
+
SELECT created_at FROM panel_query_handle_index
|
|
2386
|
+
WHERE panel_id = ? AND source_manifest_key = ? AND source_index = ? AND source_key = ? AND row_index = ?
|
|
2387
|
+
), ?),
|
|
2388
|
+
?
|
|
2389
|
+
)`).run(context.panelId, context.sourceManifestKey, params.sourceIndex, context.sourceKeys[params.sourceIndex], metadata.fileSize, metadata.modifiedAt, params.rowIndex, params.byteOffset, context.panelId, context.sourceManifestKey, params.sourceIndex, context.sourceKeys[params.sourceIndex], params.rowIndex, now, now);
|
|
2390
|
+
}
|
|
2391
|
+
function apiJsonlRangeCheckpointChunk(result) {
|
|
2392
|
+
const totalSize = normalizeApiJsonlTotalSize(result.totalSize);
|
|
2393
|
+
if (totalSize == null)
|
|
2394
|
+
return null;
|
|
2395
|
+
return {
|
|
2396
|
+
content: result.content,
|
|
2397
|
+
size: totalSize,
|
|
2398
|
+
modifiedAt: apiJsonlValidatorFingerprint(result),
|
|
2399
|
+
offset: result.rangeStart,
|
|
2400
|
+
hasMore: result.hasMore,
|
|
2401
|
+
};
|
|
2402
|
+
}
|
|
2403
|
+
function normalizeApiJsonlTotalSize(totalSize) {
|
|
2404
|
+
return typeof totalSize === 'number' && Number.isSafeInteger(totalSize) && totalSize >= 0
|
|
2405
|
+
? totalSize
|
|
2406
|
+
: null;
|
|
2407
|
+
}
|
|
2408
|
+
function apiJsonlValidatorFingerprint(source) {
|
|
2409
|
+
const etag = typeof source.etag === 'string' ? source.etag.trim() : '';
|
|
2410
|
+
if (etag)
|
|
2411
|
+
return hashStringToSafeInteger(`etag:${etag}`);
|
|
2412
|
+
const lastModified = typeof source.lastModified === 'string' ? source.lastModified.trim() : '';
|
|
2413
|
+
if (lastModified)
|
|
2414
|
+
return hashStringToSafeInteger(`last-modified:${lastModified}`);
|
|
2415
|
+
return null;
|
|
2416
|
+
}
|
|
2417
|
+
function hashStringToSafeInteger(value) {
|
|
2418
|
+
return createHash('sha256').update(value).digest().readUIntBE(0, 6);
|
|
2419
|
+
}
|
|
2420
|
+
function ensureQueryHandleIndexMetadataFresh(context, sourceIndex, chunk) {
|
|
2421
|
+
const existingMetadata = context.metadataBySourceIndex.get(sourceIndex);
|
|
2422
|
+
if (existingMetadata)
|
|
2423
|
+
return true;
|
|
2424
|
+
const sourceKey = context.sourceKeys[sourceIndex];
|
|
2425
|
+
if (sourceKey == null)
|
|
2426
|
+
return false;
|
|
2427
|
+
const fileSize = chunk.size;
|
|
2428
|
+
const modifiedAt = normalizeModifiedAt(chunk.modifiedAt);
|
|
2429
|
+
context.db.prepare(`DELETE FROM panel_query_handle_index
|
|
2430
|
+
WHERE panel_id = ?
|
|
2431
|
+
AND source_manifest_key = ?
|
|
2432
|
+
AND source_index = ?
|
|
2433
|
+
AND source_key != ?`).run(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey);
|
|
2434
|
+
const mismatch = modifiedAt == null
|
|
2435
|
+
? context.db.prepare(`SELECT 1
|
|
2436
|
+
FROM panel_query_handle_index
|
|
2437
|
+
WHERE panel_id = ?
|
|
2438
|
+
AND source_manifest_key = ?
|
|
2439
|
+
AND source_index = ?
|
|
2440
|
+
AND source_key = ?
|
|
2441
|
+
AND file_size != ?
|
|
2442
|
+
LIMIT 1`).get(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey, fileSize)
|
|
2443
|
+
: context.db.prepare(`SELECT 1
|
|
2444
|
+
FROM panel_query_handle_index
|
|
2445
|
+
WHERE panel_id = ?
|
|
2446
|
+
AND source_manifest_key = ?
|
|
2447
|
+
AND source_index = ?
|
|
2448
|
+
AND source_key = ?
|
|
2449
|
+
AND (file_size != ? OR modified_at IS NULL OR modified_at != ?)
|
|
2450
|
+
LIMIT 1`).get(context.panelId, context.sourceManifestKey, sourceIndex, sourceKey, fileSize, modifiedAt);
|
|
2451
|
+
if (mismatch) {
|
|
2452
|
+
context.db.prepare(`DELETE FROM panel_query_handle_index
|
|
2453
|
+
WHERE panel_id = ?
|
|
2454
|
+
AND source_manifest_key = ?
|
|
2455
|
+
AND source_index = ?`).run(context.panelId, context.sourceManifestKey, sourceIndex);
|
|
2456
|
+
}
|
|
2457
|
+
context.metadataBySourceIndex.set(sourceIndex, { fileSize, modifiedAt });
|
|
2458
|
+
return true;
|
|
2459
|
+
}
|
|
2460
|
+
function normalizeModifiedAt(modifiedAt) {
|
|
2461
|
+
return typeof modifiedAt === 'number' && Number.isFinite(modifiedAt)
|
|
2462
|
+
? Math.trunc(modifiedAt)
|
|
2463
|
+
: null;
|
|
2464
|
+
}
|
|
2465
|
+
async function readQueryHandleChunk(params) {
|
|
2466
|
+
try {
|
|
2467
|
+
if (params.dataSource.kind === 'workspace_relative') {
|
|
2468
|
+
return await params.workspaceBroker.readFile(params.nodeId, params.workspaceRoot, params.dataSource.relativePath, {
|
|
2469
|
+
scaffold: false,
|
|
2470
|
+
offset: params.offset,
|
|
2471
|
+
limit: params.limit,
|
|
2472
|
+
});
|
|
2473
|
+
}
|
|
2474
|
+
const exact = await params.workspaceBroker.readExactPath(params.nodeId, params.dataSource.absolutePath, {
|
|
2475
|
+
allowedRoot: params.dataSource.allowedRoot,
|
|
2476
|
+
offset: params.offset,
|
|
2477
|
+
limit: params.limit,
|
|
2478
|
+
});
|
|
2479
|
+
if (exact.kind !== 'file') {
|
|
2480
|
+
throw new PanelQueryHandleError('query_handle workspace_jsonl path must point to a file.', 400);
|
|
2481
|
+
}
|
|
2482
|
+
return exact;
|
|
2483
|
+
}
|
|
2484
|
+
catch (error) {
|
|
2485
|
+
if (error instanceof PanelQueryHandleError)
|
|
2486
|
+
throw error;
|
|
2487
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2488
|
+
if (message.includes('not_found') || message.includes('ENOENT') || message.includes('No such file')) {
|
|
2489
|
+
const path = params.dataSource.kind === 'workspace_relative'
|
|
2490
|
+
? params.dataSource.relativePath
|
|
2491
|
+
: params.dataSource.absolutePath;
|
|
2492
|
+
throw new PanelQueryHandleError(`query_handle source not found: ${path}`, 404);
|
|
2493
|
+
}
|
|
2494
|
+
throw error;
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
function parseCompleteJsonlLines(content, hasMore) {
|
|
2498
|
+
if (!hasMore) {
|
|
2499
|
+
return { completeText: content, lines: splitJsonlLineSegments(content) };
|
|
2500
|
+
}
|
|
2501
|
+
const lastNewline = content.lastIndexOf('\n');
|
|
2502
|
+
if (lastNewline < 0)
|
|
2503
|
+
return { completeText: '', lines: [] };
|
|
2504
|
+
const completeText = content.slice(0, lastNewline + 1);
|
|
2505
|
+
return {
|
|
2506
|
+
completeText,
|
|
2507
|
+
lines: splitJsonlLineSegments(completeText),
|
|
2508
|
+
};
|
|
2509
|
+
}
|
|
2510
|
+
function splitJsonlLineSegments(text) {
|
|
2511
|
+
const segments = [];
|
|
2512
|
+
let start = 0;
|
|
2513
|
+
while (start < text.length) {
|
|
2514
|
+
const newlineIndex = text.indexOf('\n', start);
|
|
2515
|
+
const end = newlineIndex >= 0 ? newlineIndex + 1 : text.length;
|
|
2516
|
+
const segment = text.slice(start, end);
|
|
2517
|
+
const line = segment.endsWith('\n') ? segment.slice(0, -1) : segment;
|
|
2518
|
+
segments.push({
|
|
2519
|
+
text: line.endsWith('\r') ? line.slice(0, -1) : line,
|
|
2520
|
+
byteLength: Buffer.byteLength(segment, 'utf8'),
|
|
2521
|
+
});
|
|
2522
|
+
start = end;
|
|
2523
|
+
}
|
|
2524
|
+
return segments;
|
|
2525
|
+
}
|
|
2526
|
+
function parseJsonlRow(line, rowIndex) {
|
|
2527
|
+
try {
|
|
2528
|
+
const parsed = JSON.parse(line);
|
|
2529
|
+
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
|
|
2530
|
+
throw new Error('row must be an object');
|
|
2531
|
+
}
|
|
2532
|
+
return parsed;
|
|
2533
|
+
}
|
|
2534
|
+
catch (error) {
|
|
2535
|
+
throw new PanelQueryHandleError(`Invalid dynamic panel JSONL row at index ${rowIndex}: ${String(error?.message ?? error)}`, 400);
|
|
2536
|
+
}
|
|
2537
|
+
}
|
|
2538
|
+
function buildPanelRowFromQueryRow(rawRow, rowIndex, datasetSchema) {
|
|
2539
|
+
const fields = {};
|
|
2540
|
+
for (const field of datasetSchema?.fields ?? []) {
|
|
2541
|
+
const value = getPanelFieldValue(rawRow, field.name);
|
|
2542
|
+
if (!isFieldValueValid(field, value)) {
|
|
2543
|
+
throw new PanelQueryHandleError(`Invalid dynamic panel row ${rowIndex}: field "${field.name}" must be a ${field.type}.`, 400);
|
|
2544
|
+
}
|
|
2545
|
+
fields[field.name] = value;
|
|
2546
|
+
}
|
|
2547
|
+
const media = {};
|
|
2548
|
+
for (const slot of datasetSchema?.mediaSlots ?? []) {
|
|
2549
|
+
const slotValue = rawRow[slot.name];
|
|
2550
|
+
if (typeof slotValue === 'string') {
|
|
2551
|
+
media[slot.name] = { kind: slot.kind, value: slotValue };
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
const rowId = typeof rawRow.row_id === 'string' && rawRow.row_id.trim()
|
|
2555
|
+
? rawRow.row_id.trim()
|
|
2556
|
+
: typeof rawRow.rowId === 'string' && rawRow.rowId.trim()
|
|
2557
|
+
? rawRow.rowId.trim()
|
|
2558
|
+
: null;
|
|
2559
|
+
return {
|
|
2560
|
+
rowIndex,
|
|
2561
|
+
rowId,
|
|
2562
|
+
fields,
|
|
2563
|
+
media,
|
|
2564
|
+
};
|
|
2565
|
+
}
|
|
2566
|
+
function isFieldValueValid(field, value) {
|
|
2567
|
+
if (value === undefined || value === null)
|
|
2568
|
+
return true;
|
|
2569
|
+
if (field.type === 'number')
|
|
2570
|
+
return typeof value === 'number' && Number.isFinite(value);
|
|
2571
|
+
return typeof value === field.type;
|
|
2572
|
+
}
|
|
2573
|
+
function matchesFilter(fields, filterField, filterValue) {
|
|
2574
|
+
if (!filterField || filterValue === undefined)
|
|
2575
|
+
return true;
|
|
2576
|
+
return getPanelFieldValue(fields, filterField) === filterValue;
|
|
2577
|
+
}
|
|
2578
|
+
function parseQueryHandleCursor(cursor) {
|
|
2579
|
+
if (!cursor)
|
|
2580
|
+
return { sourceIndex: 0, offset: 0, rowIndex: 0 };
|
|
2581
|
+
try {
|
|
2582
|
+
const parsed = decodeCursorObject(cursor);
|
|
2583
|
+
const sourceIndex = typeof parsed.sourceIndex === 'number' && Number.isInteger(parsed.sourceIndex) && parsed.sourceIndex >= 0
|
|
2584
|
+
? parsed.sourceIndex
|
|
2585
|
+
: 0;
|
|
2586
|
+
const offset = typeof parsed.offset === 'number' && Number.isInteger(parsed.offset) && parsed.offset >= 0
|
|
2587
|
+
? parsed.offset
|
|
2588
|
+
: 0;
|
|
2589
|
+
const rowIndex = typeof parsed.rowIndex === 'number' && Number.isInteger(parsed.rowIndex) && parsed.rowIndex >= 0
|
|
2590
|
+
? parsed.rowIndex
|
|
2591
|
+
: 0;
|
|
2592
|
+
return { sourceIndex, offset, rowIndex };
|
|
2593
|
+
}
|
|
2594
|
+
catch {
|
|
2595
|
+
return { sourceIndex: 0, offset: 0, rowIndex: 0 };
|
|
2596
|
+
}
|
|
2597
|
+
}
|
|
2598
|
+
function encodeQueryHandleCursor(cursor) {
|
|
2599
|
+
return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
|
|
2600
|
+
}
|
|
2601
|
+
function parseQueryHandleSortCursor(cursor, request) {
|
|
2602
|
+
if (!cursor)
|
|
2603
|
+
return { position: 0 };
|
|
2604
|
+
try {
|
|
2605
|
+
const parsed = decodeCursorObject(cursor);
|
|
2606
|
+
const sameSort = parsed.sortField === request.sortField
|
|
2607
|
+
&& parsed.sortDirection === request.sortDirection
|
|
2608
|
+
&& parsed.filterField === request.filterField
|
|
2609
|
+
&& parsed.filterValue === request.filterValue;
|
|
2610
|
+
if (!sameSort)
|
|
2611
|
+
return { position: 0 };
|
|
2612
|
+
const position = typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0
|
|
2613
|
+
? parsed.position
|
|
2614
|
+
: 0;
|
|
2615
|
+
return { position };
|
|
2616
|
+
}
|
|
2617
|
+
catch {
|
|
2618
|
+
return { position: 0 };
|
|
2619
|
+
}
|
|
2620
|
+
}
|
|
2621
|
+
function encodeQueryHandleSortCursor(cursor) {
|
|
2622
|
+
return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
|
|
2623
|
+
}
|
|
2624
|
+
function parseQueryHandlePositionCursor(cursor, request) {
|
|
2625
|
+
if (!cursor)
|
|
2626
|
+
return { position: 0 };
|
|
2627
|
+
try {
|
|
2628
|
+
const parsed = decodeCursorObject(cursor);
|
|
2629
|
+
if (request.sourceKind === 'api_jsonl'
|
|
2630
|
+
&& parsed.sourceKind === 'api_jsonl_range'
|
|
2631
|
+
&& typeof parsed.rowIndex === 'number'
|
|
2632
|
+
&& Number.isInteger(parsed.rowIndex)
|
|
2633
|
+
&& parsed.rowIndex >= 0) {
|
|
2634
|
+
return { position: parsed.rowIndex };
|
|
2635
|
+
}
|
|
2636
|
+
const sameRequest = parsed.sourceKind === request.sourceKind
|
|
2637
|
+
&& parsed.filterField === request.filterField
|
|
2638
|
+
&& parsed.filterValue === request.filterValue;
|
|
2639
|
+
if (!sameRequest)
|
|
2640
|
+
return { position: 0 };
|
|
2641
|
+
const position = typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0
|
|
2642
|
+
? parsed.position
|
|
2643
|
+
: 0;
|
|
2644
|
+
return { position };
|
|
2645
|
+
}
|
|
2646
|
+
catch {
|
|
2647
|
+
return { position: 0 };
|
|
2648
|
+
}
|
|
2649
|
+
}
|
|
2650
|
+
function encodeQueryHandlePositionCursor(cursor) {
|
|
2651
|
+
return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
|
|
2652
|
+
}
|
|
2653
|
+
function parseApiJsonlRangeCursor(cursor) {
|
|
2654
|
+
if (!cursor)
|
|
2655
|
+
return defaultApiJsonlRangeCursor();
|
|
2656
|
+
try {
|
|
2657
|
+
const parsed = decodeCursorObject(cursor);
|
|
2658
|
+
if (parsed.sourceKind !== 'api_jsonl_range')
|
|
2659
|
+
return defaultApiJsonlRangeCursor();
|
|
2660
|
+
return {
|
|
2661
|
+
sourceKind: 'api_jsonl_range',
|
|
2662
|
+
sourceIndex: typeof parsed.sourceIndex === 'number' && Number.isInteger(parsed.sourceIndex) && parsed.sourceIndex >= 0
|
|
2663
|
+
? parsed.sourceIndex
|
|
2664
|
+
: 0,
|
|
2665
|
+
offset: typeof parsed.offset === 'number' && Number.isInteger(parsed.offset) && parsed.offset >= 0
|
|
2666
|
+
? parsed.offset
|
|
2667
|
+
: 0,
|
|
2668
|
+
rowIndex: typeof parsed.rowIndex === 'number' && Number.isInteger(parsed.rowIndex) && parsed.rowIndex >= 0
|
|
2669
|
+
? parsed.rowIndex
|
|
2670
|
+
: 0,
|
|
2671
|
+
...(typeof parsed.url === 'string' && parsed.url ? { url: parsed.url } : {}),
|
|
2672
|
+
etag: typeof parsed.etag === 'string' && parsed.etag ? parsed.etag : null,
|
|
2673
|
+
lastModified: typeof parsed.lastModified === 'string' && parsed.lastModified ? parsed.lastModified : null,
|
|
2674
|
+
totalSize: typeof parsed.totalSize === 'number' && Number.isSafeInteger(parsed.totalSize) && parsed.totalSize >= 0
|
|
2675
|
+
? parsed.totalSize
|
|
2676
|
+
: null,
|
|
2677
|
+
};
|
|
2678
|
+
}
|
|
2679
|
+
catch {
|
|
2680
|
+
return defaultApiJsonlRangeCursor();
|
|
2681
|
+
}
|
|
2682
|
+
}
|
|
2683
|
+
function defaultApiJsonlRangeCursor() {
|
|
2684
|
+
return {
|
|
2685
|
+
sourceKind: 'api_jsonl_range',
|
|
2686
|
+
sourceIndex: 0,
|
|
2687
|
+
offset: 0,
|
|
2688
|
+
rowIndex: 0,
|
|
2689
|
+
etag: null,
|
|
2690
|
+
lastModified: null,
|
|
2691
|
+
totalSize: null,
|
|
2692
|
+
};
|
|
2693
|
+
}
|
|
2694
|
+
function encodeApiJsonlRangeCursor(cursor) {
|
|
2695
|
+
return Buffer.from(JSON.stringify(cursor), 'utf8').toString('base64');
|
|
2696
|
+
}
|
|
2697
|
+
function isPositionCursor(cursor) {
|
|
2698
|
+
if (!cursor)
|
|
2699
|
+
return false;
|
|
2700
|
+
try {
|
|
2701
|
+
const parsed = decodeCursorObject(cursor);
|
|
2702
|
+
return typeof parsed.position === 'number' && Number.isInteger(parsed.position) && parsed.position >= 0;
|
|
2703
|
+
}
|
|
2704
|
+
catch {
|
|
2705
|
+
return false;
|
|
2706
|
+
}
|
|
2707
|
+
}
|
|
2708
|
+
function withApiJsonlIfRange(cursor) {
|
|
2709
|
+
if (typeof cursor.etag === 'string' && cursor.etag.trim())
|
|
2710
|
+
return { ifRange: cursor.etag.trim() };
|
|
2711
|
+
if (typeof cursor.lastModified === 'string' && cursor.lastModified.trim())
|
|
2712
|
+
return { ifRange: cursor.lastModified.trim() };
|
|
2713
|
+
return {};
|
|
2714
|
+
}
|
|
2715
|
+
function decodeCursorObject(cursor) {
|
|
2716
|
+
const parsed = JSON.parse(Buffer.from(cursor, 'base64').toString('utf8'));
|
|
2717
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
2718
|
+
? parsed
|
|
2719
|
+
: {};
|
|
2720
|
+
}
|
|
2721
|
+
function comparePanelRowsBySortField(left, right, sortField, sortDirection, sortFieldType) {
|
|
2722
|
+
const leftValue = getPanelFieldValue(left.fields, sortField);
|
|
2723
|
+
const rightValue = getPanelFieldValue(right.fields, sortField);
|
|
2724
|
+
const primary = compareSortValues(leftValue, rightValue, sortFieldType);
|
|
2725
|
+
if (primary !== 0) {
|
|
2726
|
+
return sortDirection === 'asc' ? primary : -primary;
|
|
2727
|
+
}
|
|
2728
|
+
const tieBreaker = left.rowIndex - right.rowIndex;
|
|
2729
|
+
return sortDirection === 'asc' ? tieBreaker : -tieBreaker;
|
|
2730
|
+
}
|
|
2731
|
+
function compareSortValues(left, right, sortFieldType) {
|
|
2732
|
+
if (sortFieldType === 'number') {
|
|
2733
|
+
return normalizedSortNumber(left) - normalizedSortNumber(right);
|
|
2734
|
+
}
|
|
2735
|
+
if (sortFieldType === 'boolean') {
|
|
2736
|
+
return Number(left === true) - Number(right === true);
|
|
2737
|
+
}
|
|
2738
|
+
return normalizedSortString(left).localeCompare(normalizedSortString(right));
|
|
2739
|
+
}
|
|
2740
|
+
function normalizedSortNumber(value) {
|
|
2741
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : 0;
|
|
2742
|
+
}
|
|
2743
|
+
function normalizedSortString(value) {
|
|
2744
|
+
if (typeof value === 'string')
|
|
2745
|
+
return value;
|
|
2746
|
+
if (typeof value === 'number' || typeof value === 'boolean')
|
|
2747
|
+
return String(value);
|
|
2748
|
+
return '';
|
|
2749
|
+
}
|