@duckcodeailabs/dql-cli 1.6.30 → 1.6.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/apps-api.d.ts.map +1 -1
- package/dist/apps-api.js +3 -1
- package/dist/apps-api.js.map +1 -1
- package/dist/args.d.ts +18 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +37 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-B6x1cRJo.js +5698 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/commands/agent.d.ts +47 -1
- package/dist/commands/agent.d.ts.map +1 -1
- package/dist/commands/agent.js +635 -27
- package/dist/commands/agent.js.map +1 -1
- package/dist/commands/app.d.ts.map +1 -1
- package/dist/commands/app.js +3 -1
- package/dist/commands/app.js.map +1 -1
- package/dist/commands/diff.d.ts.map +1 -1
- package/dist/commands/diff.js +111 -22
- package/dist/commands/diff.js.map +1 -1
- package/dist/commands/eval-judge.d.ts +37 -0
- package/dist/commands/eval-judge.d.ts.map +1 -0
- package/dist/commands/eval-judge.js +99 -0
- package/dist/commands/eval-judge.js.map +1 -0
- package/dist/commands/eval.d.ts +27 -1
- package/dist/commands/eval.d.ts.map +1 -1
- package/dist/commands/eval.js +143 -6
- package/dist/commands/eval.js.map +1 -1
- package/dist/commands/mcp.js +27 -3
- package/dist/commands/mcp.js.map +1 -1
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -1
- package/dist/llm/analytics-tools.d.ts.map +1 -1
- package/dist/llm/analytics-tools.js +12 -7
- package/dist/llm/analytics-tools.js.map +1 -1
- package/dist/llm/answer-loop-tools.d.ts +7 -0
- package/dist/llm/answer-loop-tools.d.ts.map +1 -0
- package/dist/llm/answer-loop-tools.js +67 -0
- package/dist/llm/answer-loop-tools.js.map +1 -0
- package/dist/llm/proposal-metadata.d.ts +5 -0
- package/dist/llm/proposal-metadata.d.ts.map +1 -0
- package/dist/llm/proposal-metadata.js +27 -0
- package/dist/llm/proposal-metadata.js.map +1 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts +6 -0
- package/dist/llm/providers/claude-agent-sdk.d.ts.map +1 -1
- package/dist/llm/providers/claude-agent-sdk.js +17 -5
- package/dist/llm/providers/claude-agent-sdk.js.map +1 -1
- package/dist/llm/providers/claude-code.d.ts +7 -1
- package/dist/llm/providers/claude-code.d.ts.map +1 -1
- package/dist/llm/providers/claude-code.js +11 -2
- package/dist/llm/providers/claude-code.js.map +1 -1
- package/dist/llm/providers/dql-agent-provider.d.ts +29 -0
- package/dist/llm/providers/dql-agent-provider.d.ts.map +1 -1
- package/dist/llm/providers/dql-agent-provider.js +777 -32
- package/dist/llm/providers/dql-agent-provider.js.map +1 -1
- package/dist/llm/providers/native-sdk-provider.d.ts.map +1 -1
- package/dist/llm/providers/native-sdk-provider.js +7 -1
- package/dist/llm/providers/native-sdk-provider.js.map +1 -1
- package/dist/llm/tools.d.ts.map +1 -1
- package/dist/llm/tools.js +34 -137
- package/dist/llm/tools.js.map +1 -1
- package/dist/llm/types.d.ts +73 -2
- package/dist/llm/types.d.ts.map +1 -1
- package/dist/local-runtime.d.ts +160 -5
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +1862 -181
- package/dist/local-runtime.js.map +1 -1
- package/dist/package.json +10 -10
- package/dist/providers/oauth/claude-oauth.d.ts +80 -0
- package/dist/providers/oauth/claude-oauth.d.ts.map +1 -0
- package/dist/providers/oauth/claude-oauth.js +352 -0
- package/dist/providers/oauth/claude-oauth.js.map +1 -0
- package/dist/providers/oauth/codex-oauth.d.ts +70 -0
- package/dist/providers/oauth/codex-oauth.d.ts.map +1 -0
- package/dist/providers/oauth/codex-oauth.js +413 -0
- package/dist/providers/oauth/codex-oauth.js.map +1 -0
- package/dist/providers/oauth/oauth-store.d.ts +31 -0
- package/dist/providers/oauth/oauth-store.d.ts.map +1 -0
- package/dist/providers/oauth/oauth-store.js +61 -0
- package/dist/providers/oauth/oauth-store.js.map +1 -0
- package/dist/settings/provider-settings.d.ts +18 -0
- package/dist/settings/provider-settings.d.ts.map +1 -1
- package/dist/settings/provider-settings.js +44 -10
- package/dist/settings/provider-settings.js.map +1 -1
- package/package.json +11 -11
- package/dist/assets/dql-notebook/assets/index-QJKCrLXe.js +0 -5648
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, OllamaProvider, OpenAIProvider, answer, buildLocalContextPack, loadSkills, reindexProject, } from '@duckcodeailabs/dql-agent';
|
|
1
|
+
import { ClaudeProvider, KGStore, MemoryStore, defaultKgPath, defaultMemoryPath, GeminiProvider, loadAgentSemanticLayer, OllamaProvider, OpenAIProvider, answer, buildAnalysisQuestionPlan, buildLocalContextPack, contextRetrievalBudgetForQuestion, loadSkills, reindexProject, } from '@duckcodeailabs/dql-agent';
|
|
2
|
+
import { normalizeDqlArtifactReference } from '@duckcodeailabs/dql-core';
|
|
2
3
|
import { existsSync } from 'node:fs';
|
|
4
|
+
import { buildAnswerLoopTools, createGroundingContextExpander } from '../answer-loop-tools.js';
|
|
5
|
+
import { blockProposalDqlMetadata } from '../proposal-metadata.js';
|
|
3
6
|
import { getEffectiveProviderConfig } from '../../settings/provider-settings.js';
|
|
4
7
|
import { ClaudeCodeCliProvider, CodexCliProvider } from '../../providers/subscription-cli.js';
|
|
8
|
+
import { ClaudeOAuthProvider, claudeOAuthConnected } from '../../providers/oauth/claude-oauth.js';
|
|
9
|
+
import { CodexOAuthProvider, codexOAuthConnected } from '../../providers/oauth/codex-oauth.js';
|
|
5
10
|
const SPECS = {
|
|
6
11
|
anthropic: {
|
|
7
12
|
label: 'Anthropic Claude',
|
|
@@ -44,16 +49,125 @@ const SPECS = {
|
|
|
44
49
|
},
|
|
45
50
|
},
|
|
46
51
|
'claude-code': {
|
|
47
|
-
label: 'Claude subscription
|
|
48
|
-
setup: '
|
|
49
|
-
|
|
52
|
+
label: 'Claude subscription',
|
|
53
|
+
setup: 'Open Settings → Claude subscription and click "Sign in with Claude" (or install the `claude` CLI and run `claude /login`).',
|
|
54
|
+
// OAuth-first: use the browser-login token when connected; fall back to the CLI-passthrough otherwise.
|
|
55
|
+
create: (projectRoot) => {
|
|
56
|
+
const model = getEffectiveProviderConfig(projectRoot, 'claude-code').model;
|
|
57
|
+
return claudeOAuthConnected(projectRoot)
|
|
58
|
+
? new ClaudeOAuthProvider({ projectRoot, model })
|
|
59
|
+
: new ClaudeCodeCliProvider({ model });
|
|
60
|
+
},
|
|
50
61
|
},
|
|
51
62
|
codex: {
|
|
52
|
-
label: 'ChatGPT subscription
|
|
53
|
-
setup: '
|
|
54
|
-
create: (projectRoot) =>
|
|
63
|
+
label: 'ChatGPT subscription',
|
|
64
|
+
setup: 'Open Settings → ChatGPT subscription and click "Sign in with ChatGPT" (or install the `codex` CLI and run `codex login`).',
|
|
65
|
+
create: (projectRoot) => {
|
|
66
|
+
const model = getEffectiveProviderConfig(projectRoot, 'codex').model;
|
|
67
|
+
return codexOAuthConnected(projectRoot)
|
|
68
|
+
? new CodexOAuthProvider({ projectRoot, model })
|
|
69
|
+
: new CodexCliProvider({ model });
|
|
70
|
+
},
|
|
55
71
|
},
|
|
56
72
|
};
|
|
73
|
+
function createCertifiedFitConfirmation(provider, signal) {
|
|
74
|
+
return async ({ question, questionPlan, block, fit }) => {
|
|
75
|
+
const response = await provider.generate([
|
|
76
|
+
{
|
|
77
|
+
role: 'system',
|
|
78
|
+
content: [
|
|
79
|
+
'You are a strict governed analytics routing judge.',
|
|
80
|
+
'Decide whether the certified block can directly answer the user question.',
|
|
81
|
+
'Allow only when the block covers the requested metric, grain, dimensions, filters, required columns, ranking, and top-N.',
|
|
82
|
+
'If the block is merely useful context, close but wrong grain, missing requested columns, or ambiguous, reject it.',
|
|
83
|
+
'Return JSON only: {"allow":boolean,"confidence":"high|medium|low","reason":"short reason"}.',
|
|
84
|
+
].join(' '),
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
role: 'user',
|
|
88
|
+
content: JSON.stringify({
|
|
89
|
+
question,
|
|
90
|
+
requestedShape: questionPlan.requestedShape,
|
|
91
|
+
candidateBlock: summarizeBlockForFitConfirmation(block),
|
|
92
|
+
deterministicFit: {
|
|
93
|
+
kind: fit.kind,
|
|
94
|
+
confidence: fit.confidence,
|
|
95
|
+
reasons: fit.reasons,
|
|
96
|
+
missingOutputs: fit.missingOutputs,
|
|
97
|
+
missingDimensions: fit.missingDimensions,
|
|
98
|
+
unsupportedFilters: fit.unsupportedFilters,
|
|
99
|
+
grainMismatch: fit.grainMismatch,
|
|
100
|
+
topNAction: fit.topNAction,
|
|
101
|
+
inferredContract: fit.inferredContract,
|
|
102
|
+
},
|
|
103
|
+
}, null, 2),
|
|
104
|
+
},
|
|
105
|
+
], { maxTokens: 220, temperature: 0, signal });
|
|
106
|
+
return parseCertifiedFitConfirmation(response);
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
function summarizeBlockForFitConfirmation(block) {
|
|
110
|
+
const payload = block.payload ?? {};
|
|
111
|
+
return {
|
|
112
|
+
objectKey: block.objectKey,
|
|
113
|
+
objectType: block.objectType,
|
|
114
|
+
name: block.name,
|
|
115
|
+
status: block.status,
|
|
116
|
+
description: block.description ?? stringValue(payload.description),
|
|
117
|
+
grain: stringValue(payload.grain),
|
|
118
|
+
dimensions: stringArray(payload.dimensions),
|
|
119
|
+
entities: stringArray(payload.entities),
|
|
120
|
+
declaredOutputs: stringArray(payload.declaredOutputs),
|
|
121
|
+
outputContract: payload.outputContract,
|
|
122
|
+
allowedFilters: stringArray(payload.allowedFilters),
|
|
123
|
+
sql: truncateForFitPrompt(stringValue(payload.sql), 1200),
|
|
124
|
+
llmContext: truncateForFitPrompt(stringValue(payload.llmContext), 800),
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function parseCertifiedFitConfirmation(text) {
|
|
128
|
+
const jsonText = extractJsonObject(text);
|
|
129
|
+
if (!jsonText) {
|
|
130
|
+
return { allow: false, confidence: 'low', reason: 'fit confirmation did not return JSON' };
|
|
131
|
+
}
|
|
132
|
+
try {
|
|
133
|
+
const parsed = JSON.parse(jsonText);
|
|
134
|
+
const confidence = parsed.confidence === 'high' || parsed.confidence === 'medium' || parsed.confidence === 'low'
|
|
135
|
+
? parsed.confidence
|
|
136
|
+
: undefined;
|
|
137
|
+
return {
|
|
138
|
+
allow: parsed.allow === true,
|
|
139
|
+
...(confidence ? { confidence } : {}),
|
|
140
|
+
reason: typeof parsed.reason === 'string' ? parsed.reason : undefined,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return { allow: false, confidence: 'low', reason: 'fit confirmation returned malformed JSON' };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function extractJsonObject(text) {
|
|
148
|
+
const trimmed = text.trim();
|
|
149
|
+
if (trimmed.startsWith('{') && trimmed.endsWith('}'))
|
|
150
|
+
return trimmed;
|
|
151
|
+
const fenced = trimmed.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/i);
|
|
152
|
+
if (fenced?.[1])
|
|
153
|
+
return fenced[1];
|
|
154
|
+
const start = trimmed.indexOf('{');
|
|
155
|
+
const end = trimmed.lastIndexOf('}');
|
|
156
|
+
return start >= 0 && end > start ? trimmed.slice(start, end + 1) : undefined;
|
|
157
|
+
}
|
|
158
|
+
function stringArray(value) {
|
|
159
|
+
return Array.isArray(value)
|
|
160
|
+
? value.filter((item) => typeof item === 'string' && item.trim().length > 0)
|
|
161
|
+
: [];
|
|
162
|
+
}
|
|
163
|
+
function stringValue(value) {
|
|
164
|
+
return typeof value === 'string' && value.trim().length > 0 ? value : undefined;
|
|
165
|
+
}
|
|
166
|
+
function truncateForFitPrompt(value, max) {
|
|
167
|
+
if (!value)
|
|
168
|
+
return undefined;
|
|
169
|
+
return value.length > max ? `${value.slice(0, max - 1)}…` : value;
|
|
170
|
+
}
|
|
57
171
|
function emitProposalFromText(text, emit) {
|
|
58
172
|
const match = text.match(/DQL_BLOCK_PROPOSAL\s*[:=]?\s*(\{[\s\S]*\})\s*$/);
|
|
59
173
|
if (!match)
|
|
@@ -71,6 +185,7 @@ function emitProposalFromText(text, emit) {
|
|
|
71
185
|
owner: String(raw.owner ?? ''),
|
|
72
186
|
description: String(raw.description ?? ''),
|
|
73
187
|
sql: String(raw.sql),
|
|
188
|
+
...blockProposalDqlMetadata(raw),
|
|
74
189
|
tags: Array.isArray(raw.tags) ? raw.tags.map(String) : undefined,
|
|
75
190
|
chartType: typeof raw.chartType === 'string' ? raw.chartType : undefined,
|
|
76
191
|
},
|
|
@@ -98,14 +213,18 @@ export function createDqlAgentProviderRunner(id) {
|
|
|
98
213
|
emit({ kind: 'thinking', text: 'Building the local agent knowledge graph from terms, business views, blocks, apps, dashboards, dbt, and semantic metadata.' });
|
|
99
214
|
}
|
|
100
215
|
await reindexProject(req.projectRoot, { kgPath });
|
|
101
|
-
const
|
|
102
|
-
if (!
|
|
216
|
+
const rawQuestion = resolveEffectiveQuestion(req);
|
|
217
|
+
if (!rawQuestion) {
|
|
103
218
|
emit({ kind: 'error', message: 'No user question found.' });
|
|
104
219
|
return;
|
|
105
220
|
}
|
|
106
221
|
const memory = new MemoryStore(defaultMemoryPath(req.projectRoot));
|
|
107
222
|
const kg = new KGStore(kgPath);
|
|
108
223
|
try {
|
|
224
|
+
const conversationSnapshot = conversationSnapshotFromContext(req.conversationContext);
|
|
225
|
+
const rawFollowUp = followUpFromConversationContext(req, rawQuestion) ?? inferFollowUpContext(req, rawQuestion);
|
|
226
|
+
const followUp = applyTopicShiftGuard(rawFollowUp, conversationSnapshot);
|
|
227
|
+
const question = rewriteFollowUpQuestion(rawQuestion, followUp);
|
|
109
228
|
// Retrieve durable learnings only — notebook/project/user/artifact scope.
|
|
110
229
|
// `thread` (per-conversation) memory is intentionally excluded: it is
|
|
111
230
|
// raw-chat residue, not a governed learning, and bloats the prompt.
|
|
@@ -119,13 +238,26 @@ export function createDqlAgentProviderRunner(id) {
|
|
|
119
238
|
? await req.getSchemaContext(question).catch(() => [])
|
|
120
239
|
: [];
|
|
121
240
|
const skills = loadSkills(req.projectRoot).skills;
|
|
122
|
-
const
|
|
241
|
+
const semanticLayer = loadAgentSemanticLayer(req.projectRoot);
|
|
242
|
+
const questionPlan = buildAnalysisQuestionPlan(question, followUp);
|
|
243
|
+
const contextBudget = contextRetrievalBudgetForQuestion({
|
|
244
|
+
questionPlan,
|
|
245
|
+
requestedDepth: req.analysisDepth,
|
|
246
|
+
reasoningEffort: req.reasoningEffort,
|
|
247
|
+
});
|
|
123
248
|
const selectedContext = selectedContextForMetadata(req, question);
|
|
124
249
|
const contextPack = await buildLocalContextPack(req.projectRoot, {
|
|
125
250
|
question,
|
|
126
251
|
surface: 'notebook',
|
|
127
252
|
followUp,
|
|
128
253
|
selectedContext,
|
|
254
|
+
strictness: contextBudget.strictness,
|
|
255
|
+
limit: contextBudget.limit,
|
|
256
|
+
confirmCertifiedFit: createCertifiedFitConfirmation(provider, signal),
|
|
257
|
+
// Conversation-aware reuse: same-topic follow-ups seed (or, for
|
|
258
|
+
// filter-only refinements, re-stamp) the prior turn's context pack.
|
|
259
|
+
priorContextPackId: priorContextPackIdFromSnapshot(conversationSnapshot),
|
|
260
|
+
conversationTopicRelation: conversationSnapshot?.topicRelation,
|
|
129
261
|
runtimeSchemaSnapshot: schemaContext.length > 0
|
|
130
262
|
? {
|
|
131
263
|
source: 'agent provider schema context',
|
|
@@ -156,12 +288,20 @@ export function createDqlAgentProviderRunner(id) {
|
|
|
156
288
|
skills,
|
|
157
289
|
blockHints,
|
|
158
290
|
followUp,
|
|
291
|
+
conversationSnapshot,
|
|
159
292
|
memoryContext,
|
|
160
293
|
schemaContext,
|
|
294
|
+
semanticLayer,
|
|
161
295
|
contextPack,
|
|
162
296
|
signal,
|
|
297
|
+
reasoningEffort: req.reasoningEffort,
|
|
298
|
+
analysisDepth: contextBudget.analysisDepth,
|
|
299
|
+
...(req.semanticDriver ? { semanticDriver: req.semanticDriver } : {}),
|
|
300
|
+
...(req.semanticTableMapping ? { semanticTableMapping: req.semanticTableMapping } : {}),
|
|
163
301
|
executeCertifiedBlock: req.executeCertifiedBlock,
|
|
164
302
|
executeGeneratedSql: req.executeGeneratedSql,
|
|
303
|
+
expandGroundingContext: createGroundingContextExpander(req.projectRoot),
|
|
304
|
+
answerLoopTools: buildAnswerLoopTools(req.projectRoot),
|
|
165
305
|
// NOTE: no captureGeneratedDraft here — a plain answer/research question must NOT
|
|
166
306
|
// auto-write a draft into the blocks space. A draft is created only when the user
|
|
167
307
|
// explicitly acts (the "Create DQL draft" action → the dql_block_draft route).
|
|
@@ -244,6 +384,61 @@ export function resolveEffectiveQuestion(req) {
|
|
|
244
384
|
return current;
|
|
245
385
|
return `${original} — clarification: ${current}`;
|
|
246
386
|
}
|
|
387
|
+
export function rewriteFollowUpQuestion(question, followUp) {
|
|
388
|
+
if (!followUp || followUp.kind === 'contextual')
|
|
389
|
+
return question;
|
|
390
|
+
const pieces = [
|
|
391
|
+
`Follow-up request: ${question}`,
|
|
392
|
+
followUp.sourceQuestion ? `Prior question: ${followUp.sourceQuestion}` : '',
|
|
393
|
+
followUp.sourceBlockName ? `Prior certified block: ${followUp.sourceBlockName}` : '',
|
|
394
|
+
followUp.priorResultRef ? formatPriorResultRefForQuestion(followUp.priorResultRef) : '',
|
|
395
|
+
followUp.priorDqlArtifact ? formatPriorDqlArtifactForQuestion(followUp.priorDqlArtifact) : '',
|
|
396
|
+
followUp.priorResultValues ? `Resolved prior result values: ${formatResultDimensionValues(followUp.priorResultValues)}` : '',
|
|
397
|
+
followUp.filters?.length ? `Apply referenced filters: ${formatFollowUpFilters(followUp)}` : '',
|
|
398
|
+
followUp.dimensions?.length ? `Referenced dimensions: ${followUp.dimensions.join(', ')}` : '',
|
|
399
|
+
followUp.priorMeasures?.length ? `Prior measures: ${followUp.priorMeasures.join(', ')}` : '',
|
|
400
|
+
].filter(Boolean);
|
|
401
|
+
return pieces.length > 1 ? pieces.join('\n') : question;
|
|
402
|
+
}
|
|
403
|
+
function formatPriorResultRefForQuestion(ref) {
|
|
404
|
+
const parts = [
|
|
405
|
+
`Prior result ref: result:${ref.id}`,
|
|
406
|
+
ref.columns.length ? `schema=[${ref.columns.slice(0, 24).join(', ')}]` : '',
|
|
407
|
+
typeof ref.rowCount === 'number' ? `row_count=${ref.rowCount}` : '',
|
|
408
|
+
ref.sourceSql ? `source_sql=${compactInline(ref.sourceSql, 500)}` : '',
|
|
409
|
+
].filter(Boolean);
|
|
410
|
+
return parts.join('; ');
|
|
411
|
+
}
|
|
412
|
+
function formatPriorDqlArtifactForQuestion(artifact) {
|
|
413
|
+
const parts = [
|
|
414
|
+
`Prior DQL artifact: kind=${artifact.kind}`,
|
|
415
|
+
artifact.name ? `name=${artifact.name}` : '',
|
|
416
|
+
artifact.sourcePath ? `path=${artifact.sourcePath}` : '',
|
|
417
|
+
artifact.metrics?.length ? `metrics=[${artifact.metrics.slice(0, 12).join(', ')}]` : '',
|
|
418
|
+
artifact.dimensions?.length ? `dimensions=[${artifact.dimensions.slice(0, 12).join(', ')}]` : '',
|
|
419
|
+
artifact.filters?.length ? `filters=[${artifact.filters.slice(0, 8).map(formatDqlArtifactFilterInline).join('; ')}]` : '',
|
|
420
|
+
artifact.timeDimension ? `time=${artifact.timeDimension.name}/${artifact.timeDimension.granularity}` : '',
|
|
421
|
+
artifact.orderBy?.length ? `order_by=[${artifact.orderBy.slice(0, 8).map((order) => `${order.name} ${order.direction}`).join(', ')}]` : '',
|
|
422
|
+
typeof artifact.limit === 'number' ? `limit=${artifact.limit}` : '',
|
|
423
|
+
artifact.source ? `source=${compactInline(artifact.source, 900)}` : '',
|
|
424
|
+
].filter(Boolean);
|
|
425
|
+
return parts.join('; ');
|
|
426
|
+
}
|
|
427
|
+
function formatDqlArtifactFilterInline(filter) {
|
|
428
|
+
return `${filter.dimension} ${filter.operator} ${filter.values.slice(0, 8).join(', ')}`;
|
|
429
|
+
}
|
|
430
|
+
function formatFollowUpFilters(followUp) {
|
|
431
|
+
if (!followUp.filters?.length)
|
|
432
|
+
return '';
|
|
433
|
+
if (followUp.dimensions?.length === 1) {
|
|
434
|
+
return `${followUp.dimensions[0]} in [${followUp.filters.join(', ')}]`;
|
|
435
|
+
}
|
|
436
|
+
return followUp.filters.join(', ');
|
|
437
|
+
}
|
|
438
|
+
function compactInline(value, max) {
|
|
439
|
+
const compact = value.replace(/\s+/g, ' ').trim();
|
|
440
|
+
return compact.length > max ? `${compact.slice(0, max - 3).trimEnd()}...` : compact;
|
|
441
|
+
}
|
|
247
442
|
function renderExtraContext(req, followUp) {
|
|
248
443
|
const parts = [];
|
|
249
444
|
const upstream = req.upstream?.sql?.trim();
|
|
@@ -254,7 +449,10 @@ function renderExtraContext(req, followUp) {
|
|
|
254
449
|
parts.push(`${label}:\n${upstream}`);
|
|
255
450
|
}
|
|
256
451
|
const context = req.conversationContext;
|
|
257
|
-
|
|
452
|
+
// Thread-scoped runs render the structured conversation snapshot in its own
|
|
453
|
+
// prompt section (answer-loop) — skip this text recap to avoid double-carrying.
|
|
454
|
+
const hasServerSnapshot = Boolean(context?.serverSnapshot);
|
|
455
|
+
if (context && !hasServerSnapshot) {
|
|
258
456
|
// Bound the carried-forward "conversation memory" defensively: only the most
|
|
259
457
|
// recent turn's signals, with hard caps on the summary length and list sizes
|
|
260
458
|
// so prompts don't grow across a long multi-turn chat.
|
|
@@ -268,9 +466,12 @@ function renderExtraContext(req, followUp) {
|
|
|
268
466
|
context.trustLabel ? `trust label: ${context.trustLabel}` : '',
|
|
269
467
|
context.reviewStatus ? `review status: ${context.reviewStatus}` : '',
|
|
270
468
|
context.draftBlockPath ? `draft block: ${context.draftBlockPath}` : '',
|
|
469
|
+
context.dqlArtifact ? `prior DQL artifact:\n${formatPriorDqlArtifactForQuestion(context.dqlArtifact)}` : '',
|
|
271
470
|
context.requestedFilters?.length ? `remembered filters: ${clampList(context.requestedFilters)}` : '',
|
|
272
471
|
context.requestedDimensions?.length ? `remembered dimensions: ${clampList(context.requestedDimensions)}` : '',
|
|
273
472
|
context.outputColumns?.length ? `prior output columns: ${clampList(context.outputColumns)}` : '',
|
|
473
|
+
context.resultDimensionValues ? `prior result values: ${formatResultDimensionValues(context.resultDimensionValues)}` : '',
|
|
474
|
+
context.turns?.length ? `recent analytical turns:\n${formatConversationTurnsForPrompt(context.turns)}` : '',
|
|
274
475
|
].filter(Boolean);
|
|
275
476
|
if (contextLines.length > 0) {
|
|
276
477
|
parts.push(`Conversation memory:\n${contextLines.join('\n')}`);
|
|
@@ -279,8 +480,13 @@ function renderExtraContext(req, followUp) {
|
|
|
279
480
|
if (followUp?.sourceBlockName) {
|
|
280
481
|
const suffix = followUp.kind === 'drilldown'
|
|
281
482
|
? 'Use it as source context, but prefer a distinct certified drilldown block or a review-required draft.'
|
|
282
|
-
:
|
|
283
|
-
|
|
483
|
+
: followUp.kind === 'contextual'
|
|
484
|
+
? 'This is advisory prior-turn context — use it only if the question refers to it; on a new topic, ignore it.'
|
|
485
|
+
: 'Reuse it for this generic follow-up.';
|
|
486
|
+
const lead = followUp.kind === 'contextual'
|
|
487
|
+
? `Prior turn used certified block "${followUp.sourceBlockName}".`
|
|
488
|
+
: `Follow-up context: the user is referring to certified block "${followUp.sourceBlockName}".`;
|
|
489
|
+
parts.push(`${lead} ${suffix}`);
|
|
284
490
|
}
|
|
285
491
|
if (followUp?.filters?.length) {
|
|
286
492
|
parts.push(`Requested follow-up filters: ${followUp.filters.join(', ')}`);
|
|
@@ -288,6 +494,12 @@ function renderExtraContext(req, followUp) {
|
|
|
288
494
|
if (followUp?.dimensions?.length) {
|
|
289
495
|
parts.push(`Requested follow-up dimensions: ${followUp.dimensions.join(', ')}`);
|
|
290
496
|
}
|
|
497
|
+
if (followUp?.priorResultRef) {
|
|
498
|
+
parts.push(`Prior result reference:\n${formatPriorResultRefForQuestion(followUp.priorResultRef)}`);
|
|
499
|
+
}
|
|
500
|
+
if (followUp?.priorDqlArtifact) {
|
|
501
|
+
parts.push(`Prior DQL artifact reference:\n${formatPriorDqlArtifactForQuestion(followUp.priorDqlArtifact)}`);
|
|
502
|
+
}
|
|
291
503
|
return parts.length > 0 ? parts.join('\n\n') : undefined;
|
|
292
504
|
}
|
|
293
505
|
function selectedContextForMetadata(req, question) {
|
|
@@ -351,9 +563,9 @@ function extractSelectedBlockHints(req) {
|
|
|
351
563
|
}
|
|
352
564
|
}
|
|
353
565
|
function inferFollowUpContext(req, question) {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
566
|
+
// Messages-only fallback (no structured conversationContext). Regexes classify the
|
|
567
|
+
// kind; a non-matching question still carries the prior turn as advisory 'contextual'.
|
|
568
|
+
const kind = isGenericFollowUp(question) ? 'generic' : isDrilldownFollowUp(question) ? 'drilldown' : 'contextual';
|
|
357
569
|
for (let i = req.messages.length - 1; i >= 0; i--) {
|
|
358
570
|
const msg = req.messages[i];
|
|
359
571
|
if (msg.role !== 'assistant')
|
|
@@ -371,38 +583,335 @@ function inferFollowUpContext(req, question) {
|
|
|
371
583
|
}
|
|
372
584
|
return undefined;
|
|
373
585
|
}
|
|
586
|
+
function priorContextPackIdFromSnapshot(snapshot) {
|
|
587
|
+
const fromState = snapshot?.workingState?.lastContextPackId;
|
|
588
|
+
if (typeof fromState === 'string' && fromState.trim())
|
|
589
|
+
return fromState;
|
|
590
|
+
const fromTurns = snapshot?.recentTurns?.length
|
|
591
|
+
? snapshot.recentTurns[snapshot.recentTurns.length - 1]?.contextPackId
|
|
592
|
+
: undefined;
|
|
593
|
+
return typeof fromTurns === 'string' && fromTurns.trim() ? fromTurns : undefined;
|
|
594
|
+
}
|
|
595
|
+
/** Parse the server-attached conversation snapshot (thread-scoped runs only). */
|
|
596
|
+
function conversationSnapshotFromContext(context) {
|
|
597
|
+
const raw = context?.serverSnapshot;
|
|
598
|
+
if (!raw || typeof raw !== 'object' || Array.isArray(raw))
|
|
599
|
+
return undefined;
|
|
600
|
+
const snapshot = raw;
|
|
601
|
+
return typeof snapshot.threadId === 'string' && Array.isArray(snapshot.recentTurns)
|
|
602
|
+
? snapshot
|
|
603
|
+
: undefined;
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Deterministic stale-context protection: when the persisted working state says
|
|
607
|
+
* the new question is a topic SHIFT, prior-turn filters must not be forced into
|
|
608
|
+
* the follow-up (a "by X" phrasing can regex-classify as drilldown even on a
|
|
609
|
+
* genuinely new topic). Question-derived filters are kept; carried ones drop.
|
|
610
|
+
*/
|
|
611
|
+
function applyTopicShiftGuard(followUp, snapshot) {
|
|
612
|
+
if (!followUp || snapshot?.topicRelation !== 'shift')
|
|
613
|
+
return followUp;
|
|
614
|
+
if (followUp.kind !== 'drilldown')
|
|
615
|
+
return followUp;
|
|
616
|
+
return {
|
|
617
|
+
...followUp,
|
|
618
|
+
kind: 'contextual',
|
|
619
|
+
filters: undefined,
|
|
620
|
+
dimensions: undefined,
|
|
621
|
+
priorResultValues: undefined,
|
|
622
|
+
priorMeasures: undefined,
|
|
623
|
+
priorLimit: undefined,
|
|
624
|
+
};
|
|
625
|
+
}
|
|
374
626
|
function followUpFromConversationContext(req, question) {
|
|
375
627
|
const context = req.conversationContext;
|
|
376
628
|
if (!context)
|
|
377
629
|
return undefined;
|
|
378
|
-
const
|
|
379
|
-
|
|
630
|
+
const turns = conversationTurnsFromContext(context);
|
|
631
|
+
const activeTurn = activeConversationTurn(context, turns, question);
|
|
632
|
+
const activeResult = activeTurn?.result && typeof activeTurn.result === 'object' && !Array.isArray(activeTurn.result)
|
|
633
|
+
? activeTurn.result
|
|
634
|
+
: undefined;
|
|
635
|
+
const sourceBlockName = cleanOptionalString(activeTurn?.sourceCertifiedBlock) ?? cleanOptionalString(context.sourceCertifiedBlock);
|
|
636
|
+
const priorResultValues = cleanStringRecordArray(activeResult?.dimensionValues) ?? cleanStringRecordArray(context.resultDimensionValues);
|
|
637
|
+
const priorResultColumns = mergeStrings(arrayValue(activeResult?.columns), context.resultColumns, context.outputColumns);
|
|
638
|
+
const priorResultRef = priorResultRefFromTurn(activeTurn, activeResult, priorResultColumns);
|
|
639
|
+
const priorDqlArtifact = cleanDqlArtifactReference(activeTurn?.dqlArtifact) ?? cleanDqlArtifactReference(context.dqlArtifact);
|
|
640
|
+
const resolvedReferences = resolveConversationReferences(question, turns, priorResultValues);
|
|
641
|
+
const hasUsefulContext = Boolean(sourceBlockName || priorResultColumns?.length || priorResultValues || priorDqlArtifact);
|
|
642
|
+
if (!hasUsefulContext)
|
|
380
643
|
return undefined;
|
|
381
644
|
const inferredKind = isGenericFollowUp(question)
|
|
382
645
|
? 'generic'
|
|
383
646
|
: isDrilldownFollowUp(question)
|
|
384
647
|
? 'drilldown'
|
|
385
648
|
: null;
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
649
|
+
// Always-on carry: the regexes only CLASSIFY the follow-up kind — they no longer
|
|
650
|
+
// gate whether conversation context exists at all. A question that matches neither
|
|
651
|
+
// pattern still carries the prior turn as advisory 'contextual' state; the model
|
|
652
|
+
// (not a regex) decides whether it's relevant to the new question.
|
|
653
|
+
const kind = inferredKind
|
|
654
|
+
?? (context.followupKind === 'generic' || context.followupKind === 'drilldown' ? context.followupKind : null)
|
|
655
|
+
?? 'contextual';
|
|
389
656
|
return {
|
|
390
657
|
kind,
|
|
658
|
+
sourceTurnId: cleanOptionalString(activeTurn?.id) ?? cleanOptionalString(context.sourceAnswerId),
|
|
391
659
|
sourceBlockName,
|
|
392
|
-
sourceQuestion: cleanOptionalString(context.sourceQuestion),
|
|
393
|
-
sourceAnswer: cleanOptionalString(context.sourceAnswerSummary),
|
|
660
|
+
sourceQuestion: cleanOptionalString(activeTurn?.question) ?? cleanOptionalString(context.sourceQuestion),
|
|
661
|
+
sourceAnswer: cleanOptionalString(activeTurn?.answerSummary) ?? cleanOptionalString(context.sourceAnswerSummary),
|
|
394
662
|
filters: kind === 'drilldown'
|
|
395
|
-
? mergeStrings(context.requestedFilters, extractDrilldownFilters(question))
|
|
663
|
+
? mergeStrings(activeTurnStringArray(activeTurn, 'requestedFilters'), context.requestedFilters, extractDrilldownFilters(question), resolvedReferences.filters)
|
|
396
664
|
: undefined,
|
|
397
665
|
dimensions: kind === 'drilldown'
|
|
398
|
-
? mergeStrings(context.requestedDimensions, extractDrilldownDimensions(question))
|
|
666
|
+
? mergeStrings(activeTurnStringArray(activeTurn, 'requestedDimensions'), context.requestedDimensions, extractDrilldownDimensions(question), resolvedReferences.dimensions)
|
|
399
667
|
: undefined,
|
|
668
|
+
priorResultColumns,
|
|
669
|
+
priorResultValues,
|
|
670
|
+
priorResultRef,
|
|
671
|
+
priorDqlArtifact,
|
|
672
|
+
priorLimit: activeTurnNumber(activeTurn, 'topN') ?? (typeof context.priorLimit === 'number' ? context.priorLimit : undefined),
|
|
673
|
+
priorMeasures: mergeStrings(activeTurnStringArray(activeTurn, 'requestedMeasures'), arrayValue(activeResult?.measureColumns), context.priorMeasures, inferredMeasuresFromAnswerContract(context.answerContract), inferredMeasureColumns(priorResultColumns)),
|
|
674
|
+
resolvedReferences: resolvedReferences.labels,
|
|
675
|
+
unresolvedReferences: resolvedReferences.unresolved,
|
|
400
676
|
};
|
|
401
677
|
}
|
|
678
|
+
function conversationTurnsFromContext(context) {
|
|
679
|
+
const explicit = Array.isArray(context?.turns)
|
|
680
|
+
? context.turns.map(cleanRecord).filter((turn) => Boolean(turn))
|
|
681
|
+
: [];
|
|
682
|
+
const snapshot = conversationSnapshotFromContext(context);
|
|
683
|
+
const snapshotTurns = [
|
|
684
|
+
...(snapshot?.recalledTurns ?? []).map((turn) => snapshotTurnToConversationRecord(turn, 'recalled')),
|
|
685
|
+
...(snapshot?.recentTurns ?? []).map((turn) => snapshotTurnToConversationRecord(turn, 'recent')),
|
|
686
|
+
].filter((turn) => Boolean(turn));
|
|
687
|
+
const merged = mergeConversationTurnRecords([...snapshotTurns, ...explicit]);
|
|
688
|
+
if (merged.length > 0)
|
|
689
|
+
return merged.slice(-12);
|
|
690
|
+
const legacy = cleanRecord({
|
|
691
|
+
id: context?.sourceAnswerId,
|
|
692
|
+
question: context?.sourceQuestion,
|
|
693
|
+
answerSummary: context?.sourceAnswerSummary,
|
|
694
|
+
sourceCertifiedBlock: context?.sourceCertifiedBlock,
|
|
695
|
+
requestedFilters: context?.requestedFilters,
|
|
696
|
+
requestedDimensions: context?.requestedDimensions,
|
|
697
|
+
requestedMeasures: context?.priorMeasures,
|
|
698
|
+
topN: context?.priorLimit,
|
|
699
|
+
result: {
|
|
700
|
+
columns: context?.resultColumns ?? context?.outputColumns,
|
|
701
|
+
rowsSample: context?.resultRowsSample,
|
|
702
|
+
dimensionValues: context?.resultDimensionValues,
|
|
703
|
+
measureColumns: context?.priorMeasures,
|
|
704
|
+
},
|
|
705
|
+
route: context?.route,
|
|
706
|
+
trustLabel: context?.trustLabel,
|
|
707
|
+
reviewStatus: context?.reviewStatus,
|
|
708
|
+
certification: context?.certification,
|
|
709
|
+
contextPackId: context?.contextPackId,
|
|
710
|
+
sourceSql: context?.sourceSql,
|
|
711
|
+
dqlArtifact: context?.dqlArtifact,
|
|
712
|
+
cascade: context?.cascade,
|
|
713
|
+
});
|
|
714
|
+
return legacy ? [legacy] : [];
|
|
715
|
+
}
|
|
716
|
+
function snapshotTurnToConversationRecord(turn, snapshotSource) {
|
|
717
|
+
return compactConversationRecord({
|
|
718
|
+
id: turn.id,
|
|
719
|
+
question: turn.question,
|
|
720
|
+
answerSummary: turn.answerSummary,
|
|
721
|
+
sourceCertifiedBlock: turn.sourceCertifiedBlock,
|
|
722
|
+
route: turn.route,
|
|
723
|
+
contextPackId: turn.contextPackId,
|
|
724
|
+
sourceSql: turn.sourceSql,
|
|
725
|
+
dqlArtifact: turn.dqlArtifact,
|
|
726
|
+
cascade: turn.cascade,
|
|
727
|
+
snapshotSource,
|
|
728
|
+
result: compactConversationRecord({
|
|
729
|
+
columns: turn.resultColumns,
|
|
730
|
+
dimensionValues: turn.resultDimensionValues,
|
|
731
|
+
rowCount: turn.resultRowCount,
|
|
732
|
+
}),
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
function mergeConversationTurnRecords(turns) {
|
|
736
|
+
const byId = new Map();
|
|
737
|
+
const anonymous = [];
|
|
738
|
+
for (const turn of turns) {
|
|
739
|
+
const id = cleanOptionalString(turn.id);
|
|
740
|
+
if (!id) {
|
|
741
|
+
anonymous.push(turn);
|
|
742
|
+
continue;
|
|
743
|
+
}
|
|
744
|
+
const current = byId.get(id);
|
|
745
|
+
byId.set(id, current ? { ...current, ...turn } : turn);
|
|
746
|
+
}
|
|
747
|
+
return [...anonymous, ...byId.values()];
|
|
748
|
+
}
|
|
749
|
+
function activeConversationTurn(context, turns, question) {
|
|
750
|
+
if (turns.length === 0)
|
|
751
|
+
return undefined;
|
|
752
|
+
const activeId = cleanOptionalString(context?.activeTurnId);
|
|
753
|
+
const activeMatch = activeId
|
|
754
|
+
? turns.find((turn) => cleanOptionalString(turn.id) === activeId)
|
|
755
|
+
: undefined;
|
|
756
|
+
const recalled = turns.filter((turn) => cleanOptionalString(turn.snapshotSource) === 'recalled' && turnHasUsefulResult(turn));
|
|
757
|
+
if (recalled.length > 0) {
|
|
758
|
+
const bestRecalled = recalled
|
|
759
|
+
.map((turn) => ({ turn, score: scoreTurnForQuestion(turn, question) }))
|
|
760
|
+
.sort((a, b) => b.score - a.score)[0];
|
|
761
|
+
const activeScore = activeMatch ? scoreTurnForQuestion(activeMatch, question) : 0;
|
|
762
|
+
if (bestRecalled && shouldPreferRecalledPriorResult(question, bestRecalled.score, activeScore)) {
|
|
763
|
+
return bestRecalled.turn;
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
if (activeId) {
|
|
767
|
+
if (activeMatch)
|
|
768
|
+
return activeMatch;
|
|
769
|
+
}
|
|
770
|
+
return [...turns].reverse().find((turn) => {
|
|
771
|
+
return turnHasUsefulResult(turn);
|
|
772
|
+
}) ?? turns[turns.length - 1];
|
|
773
|
+
}
|
|
774
|
+
function turnHasUsefulResult(turn) {
|
|
775
|
+
const result = cleanRecord(turn.result);
|
|
776
|
+
const columns = arrayValue(result?.columns);
|
|
777
|
+
const rows = arrayValue(result?.rowsSample);
|
|
778
|
+
const values = cleanStringRecordArray(result?.dimensionValues);
|
|
779
|
+
return Boolean(columns?.length || rows?.length || values);
|
|
780
|
+
}
|
|
781
|
+
function scoreTurnForQuestion(turn, question) {
|
|
782
|
+
const queryTokens = tokenSet(question);
|
|
783
|
+
if (queryTokens.size === 0)
|
|
784
|
+
return 0;
|
|
785
|
+
const result = cleanRecord(turn.result);
|
|
786
|
+
const values = cleanStringRecordArray(result?.dimensionValues);
|
|
787
|
+
const text = [
|
|
788
|
+
cleanOptionalString(turn.question),
|
|
789
|
+
cleanOptionalString(turn.answerSummary),
|
|
790
|
+
...(arrayValue(result?.columns) ?? []).map((value) => cleanOptionalString(value)).filter((value) => Boolean(value)),
|
|
791
|
+
...Object.entries(values ?? {}).flatMap(([key, list]) => [key, ...list.slice(0, 8)]),
|
|
792
|
+
].filter(Boolean).join(' ');
|
|
793
|
+
let score = 0;
|
|
794
|
+
const turnTokens = tokenSet(text);
|
|
795
|
+
for (const token of queryTokens) {
|
|
796
|
+
if (turnTokens.has(token))
|
|
797
|
+
score += 1;
|
|
798
|
+
}
|
|
799
|
+
return score;
|
|
800
|
+
}
|
|
801
|
+
function shouldPreferRecalledPriorResult(question, recalledScore, activeScore) {
|
|
802
|
+
if (recalledScore >= Math.max(activeScore + 2, 2))
|
|
803
|
+
return true;
|
|
804
|
+
if (!wantsPriorResultReference(question))
|
|
805
|
+
return false;
|
|
806
|
+
return recalledScore > 0 && recalledScore > activeScore;
|
|
807
|
+
}
|
|
808
|
+
function wantsPriorResultReference(question) {
|
|
809
|
+
return /\b(?:previous|prior|earlier|above)\s+(?:results?|outputs?|rows?|turns?)\b/i.test(question)
|
|
810
|
+
|| /\b(?:with|from|using|include)\s+(?:the\s+)?(?:previous|prior|earlier|above)\b/i.test(question);
|
|
811
|
+
}
|
|
812
|
+
function tokenSet(value) {
|
|
813
|
+
const tokens = value
|
|
814
|
+
.toLowerCase()
|
|
815
|
+
.match(/[a-z0-9_]+/g) ?? [];
|
|
816
|
+
return new Set(tokens
|
|
817
|
+
.map((token) => token.replace(/s$/, ''))
|
|
818
|
+
.filter((token) => token.length > 2 && !GENERIC_FOLLOW_UP_WORDS.has(token)));
|
|
819
|
+
}
|
|
820
|
+
function priorResultRefFromTurn(activeTurn, activeResult, priorResultColumns) {
|
|
821
|
+
const columns = priorResultColumns?.slice(0, 32) ?? [];
|
|
822
|
+
if (columns.length === 0)
|
|
823
|
+
return undefined;
|
|
824
|
+
const id = cleanOptionalString(activeTurn?.id) ?? 'previous';
|
|
825
|
+
const rowCount = typeof activeResult?.rowCount === 'number' && Number.isFinite(activeResult.rowCount)
|
|
826
|
+
? activeResult.rowCount
|
|
827
|
+
: Array.isArray(activeResult?.rowsSample)
|
|
828
|
+
? activeResult.rowsSample.length
|
|
829
|
+
: undefined;
|
|
830
|
+
const sourceSql = cleanOptionalString(activeTurn?.sourceSql);
|
|
831
|
+
return {
|
|
832
|
+
id,
|
|
833
|
+
question: cleanOptionalString(activeTurn?.question),
|
|
834
|
+
columns,
|
|
835
|
+
rowCount,
|
|
836
|
+
sourceSql: sourceSql ? sourceSql.slice(0, 1200) : undefined,
|
|
837
|
+
};
|
|
838
|
+
}
|
|
839
|
+
function resolveConversationReferences(question, turns, activeValues) {
|
|
840
|
+
const dimensions = resolveDeicticDimensions(question, activeValues) ?? [];
|
|
841
|
+
let filters = resolveDeicticFilters(question, activeValues) ?? [];
|
|
842
|
+
const labels = [];
|
|
843
|
+
if (dimensions.length > 0 && filters.length === 0) {
|
|
844
|
+
for (const turn of [...turns].reverse()) {
|
|
845
|
+
const values = cleanStringRecordArray(cleanRecord(turn.result)?.dimensionValues);
|
|
846
|
+
if (!values)
|
|
847
|
+
continue;
|
|
848
|
+
filters = dimensions.flatMap((dimension) => valuesForPriorDimension(values, dimension));
|
|
849
|
+
if (filters.length > 0)
|
|
850
|
+
break;
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
for (const dimension of dimensions) {
|
|
854
|
+
const values = (activeValues ? valuesForPriorDimension(activeValues, dimension) : []).slice(0, 5);
|
|
855
|
+
labels.push(values.length ? `${dimension}: ${values.join(', ')}` : `${dimension}: unresolved`);
|
|
856
|
+
}
|
|
857
|
+
const unresolved = referencesNeedValues(question) && filters.length === 0
|
|
858
|
+
? ['Could not resolve the referenced prior result values from conversation state.']
|
|
859
|
+
: undefined;
|
|
860
|
+
return {
|
|
861
|
+
filters: filters.length > 0 ? Array.from(new Set(filters)).slice(0, 24) : undefined,
|
|
862
|
+
dimensions: dimensions.length > 0 ? dimensions : undefined,
|
|
863
|
+
labels: labels.length > 0 ? labels : undefined,
|
|
864
|
+
unresolved,
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
function cleanRecord(value) {
|
|
868
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
869
|
+
? value
|
|
870
|
+
: undefined;
|
|
871
|
+
}
|
|
872
|
+
function compactConversationRecord(record) {
|
|
873
|
+
const entries = Object.entries(record).filter(([, value]) => value !== undefined && value !== null);
|
|
874
|
+
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
875
|
+
}
|
|
876
|
+
function cleanDqlArtifactReference(value) {
|
|
877
|
+
const artifact = normalizeDqlArtifactReference(value);
|
|
878
|
+
if (!artifact)
|
|
879
|
+
return undefined;
|
|
880
|
+
return {
|
|
881
|
+
...artifact,
|
|
882
|
+
source: artifact.source.slice(0, 3000),
|
|
883
|
+
metrics: uniqueStringList(artifact.metrics),
|
|
884
|
+
dimensions: uniqueStringList(artifact.dimensions),
|
|
885
|
+
filters: artifact.filters
|
|
886
|
+
?.filter((filter) => filter.values.length > 0)
|
|
887
|
+
.slice(0, 12)
|
|
888
|
+
.map((filter) => ({ ...filter, values: uniqueStringList(filter.values)?.slice(0, 12) ?? [] })),
|
|
889
|
+
orderBy: artifact.orderBy?.slice(0, 12),
|
|
890
|
+
};
|
|
891
|
+
}
|
|
892
|
+
function uniqueStringList(value) {
|
|
893
|
+
if (!value?.length)
|
|
894
|
+
return undefined;
|
|
895
|
+
const unique = Array.from(new Set(value)).slice(0, 24);
|
|
896
|
+
return unique.length > 0 ? unique : undefined;
|
|
897
|
+
}
|
|
898
|
+
function activeTurnStringArray(turn, key) {
|
|
899
|
+
return arrayValue(turn?.[key]);
|
|
900
|
+
}
|
|
901
|
+
function activeTurnNumber(turn, key) {
|
|
902
|
+
const value = turn?.[key];
|
|
903
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
904
|
+
}
|
|
905
|
+
function referencesNeedValues(question) {
|
|
906
|
+
return /\b(?:this|that|these|those|same|above|previous|prior)\b/i.test(question);
|
|
907
|
+
}
|
|
402
908
|
function extractCertifiedBlockName(content) {
|
|
403
909
|
const fromAnswer = content.match(/Answered by certified block \*\*([^*]+)\*\*/i)?.[1];
|
|
910
|
+
const fromRoute = content.match(/Answered from certified block\s+([A-Za-z0-9_.-]+)/i)?.[1];
|
|
404
911
|
const fromCitation = content.match(/^- block:\s*([A-Za-z0-9_.-]+)/im)?.[1];
|
|
405
|
-
|
|
912
|
+
// The name pattern admits dots, so a sentence period lands in the match
|
|
913
|
+
// ("... block food_vs_drink_revenue.") — strip trailing punctuation.
|
|
914
|
+
return (fromAnswer ?? fromRoute ?? fromCitation)?.trim().replace(/[.,;:]+$/, '') || undefined;
|
|
406
915
|
}
|
|
407
916
|
function cleanOptionalString(value) {
|
|
408
917
|
return typeof value === 'string' && value.trim() ? value.trim() : undefined;
|
|
@@ -415,6 +924,168 @@ function mergeStrings(...groups) {
|
|
|
415
924
|
const unique = Array.from(new Set(values)).slice(0, 24);
|
|
416
925
|
return unique.length > 0 ? unique : undefined;
|
|
417
926
|
}
|
|
927
|
+
function inferredMeasuresFromAnswerContract(value) {
|
|
928
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
929
|
+
return undefined;
|
|
930
|
+
const record = value;
|
|
931
|
+
const requestedShape = record.requestedShape && typeof record.requestedShape === 'object' && !Array.isArray(record.requestedShape)
|
|
932
|
+
? record.requestedShape
|
|
933
|
+
: undefined;
|
|
934
|
+
return mergeStrings(arrayValue(record.measures), arrayValue(requestedShape?.measures));
|
|
935
|
+
}
|
|
936
|
+
function inferredMeasureColumns(columns) {
|
|
937
|
+
if (!columns?.length)
|
|
938
|
+
return undefined;
|
|
939
|
+
const measures = columns.filter((column) => /\b(revenue|sales|amount|total|count|average|avg|sum|spend|cost|margin|profit|value|points?|score|quantity|rate|volume)\b/i.test(column.replace(/_/g, ' ')));
|
|
940
|
+
return measures.length > 0 ? measures : undefined;
|
|
941
|
+
}
|
|
942
|
+
function arrayValue(value) {
|
|
943
|
+
return Array.isArray(value) ? value : undefined;
|
|
944
|
+
}
|
|
945
|
+
function cleanStringRecordArray(value) {
|
|
946
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
947
|
+
return undefined;
|
|
948
|
+
const out = {};
|
|
949
|
+
for (const [key, raw] of Object.entries(value)) {
|
|
950
|
+
const values = Array.isArray(raw)
|
|
951
|
+
? raw.map(cleanOptionalString).filter((item) => Boolean(item)).slice(0, 24)
|
|
952
|
+
: [];
|
|
953
|
+
if (key.trim() && values.length > 0)
|
|
954
|
+
out[key.trim()] = values;
|
|
955
|
+
}
|
|
956
|
+
return Object.keys(out).length > 0 ? out : undefined;
|
|
957
|
+
}
|
|
958
|
+
function resolveDeicticFilters(question, priorValues) {
|
|
959
|
+
if (!priorValues)
|
|
960
|
+
return undefined;
|
|
961
|
+
const hasExplicitPluralReference = /\b(?:these|those|same)\s+(?:customers?|products?|cat(?:egor|agor|ogor)(?:y|ies)|segments?|regions?)\b/i.test(question);
|
|
962
|
+
if (!hasExplicitPluralReference) {
|
|
963
|
+
const singular = resolveSingularDeicticDimension(question, priorValues);
|
|
964
|
+
if (singular) {
|
|
965
|
+
const values = valuesForPriorDimension(priorValues, singular).slice(0, 1);
|
|
966
|
+
return values.length > 0 ? values : undefined;
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
const dims = resolveDeicticDimensions(question, priorValues) ?? [];
|
|
970
|
+
const values = dims.flatMap((dim) => valuesForPriorDimension(priorValues, dim));
|
|
971
|
+
return values.length > 0 ? Array.from(new Set(values)).slice(0, 24) : undefined;
|
|
972
|
+
}
|
|
973
|
+
function resolveDeicticDimensions(question, priorValues) {
|
|
974
|
+
if (!priorValues)
|
|
975
|
+
return undefined;
|
|
976
|
+
const lower = question.toLowerCase();
|
|
977
|
+
const dims = [];
|
|
978
|
+
const candidates = [
|
|
979
|
+
[/\b(?:these|those|the)\s+cat(?:egor|agor|ogor)(?:y|ies)\b/, 'category'],
|
|
980
|
+
[/\b(?:this|that|these|those|the)\s+products?\b/, 'product'],
|
|
981
|
+
[/\b(?:this|that|these|those|the)\s+customers?\b/, 'customer'],
|
|
982
|
+
[/\b(?:above|previous|prior)\s+(?:orders?|results?|rows?)\b/, 'customer'],
|
|
983
|
+
[/\b(?:this|that|these|those|the)\s+segments?\b/, 'segment'],
|
|
984
|
+
[/\b(?:this|that|these|those|the)\s+regions?\b/, 'region'],
|
|
985
|
+
];
|
|
986
|
+
for (const [pattern, dim] of candidates) {
|
|
987
|
+
if (!pattern.test(lower))
|
|
988
|
+
continue;
|
|
989
|
+
if (dim === 'product' && /\bthe\s+product\s+cat(?:egor|agor|ogor)(?:y|ies)\b/.test(lower))
|
|
990
|
+
continue;
|
|
991
|
+
if (valuesForPriorDimension(priorValues, dim).length)
|
|
992
|
+
dims.push(dim);
|
|
993
|
+
}
|
|
994
|
+
if (dims.length === 0 && /\b(?:these|those|that|them|same|above|previous|prior)\b/.test(lower)) {
|
|
995
|
+
const single = singlePriorValueDimension(priorValues);
|
|
996
|
+
if (single)
|
|
997
|
+
dims.push(single);
|
|
998
|
+
}
|
|
999
|
+
return dims.length > 0 ? Array.from(new Set(dims)) : undefined;
|
|
1000
|
+
}
|
|
1001
|
+
function resolveSingularDeicticDimension(question, priorValues) {
|
|
1002
|
+
const lower = question.toLowerCase();
|
|
1003
|
+
const candidates = [
|
|
1004
|
+
[/\b(?:this|that|the)\s+product\b/, 'product'],
|
|
1005
|
+
[/\b(?:this|that|the)\s+customer\b/, 'customer'],
|
|
1006
|
+
[/\b(?:this|that|the)\s+category\b/, 'category'],
|
|
1007
|
+
[/\b(?:this|that|the)\s+segment\b/, 'segment'],
|
|
1008
|
+
[/\b(?:this|that|the)\s+region\b/, 'region'],
|
|
1009
|
+
];
|
|
1010
|
+
for (const [pattern, dim] of candidates) {
|
|
1011
|
+
if (pattern.test(lower) && valuesForPriorDimension(priorValues, dim).length)
|
|
1012
|
+
return dim;
|
|
1013
|
+
}
|
|
1014
|
+
return undefined;
|
|
1015
|
+
}
|
|
1016
|
+
function valuesForPriorDimension(values, dim) {
|
|
1017
|
+
const aliases = [dim, `${dim}_name`];
|
|
1018
|
+
if (dim === 'product')
|
|
1019
|
+
aliases.push('sku');
|
|
1020
|
+
if (dim === 'category')
|
|
1021
|
+
aliases.push('product_type', 'category_name');
|
|
1022
|
+
return Array.from(new Set(aliases.flatMap((alias) => values[alias] ?? []))).filter(Boolean);
|
|
1023
|
+
}
|
|
1024
|
+
function formatResultDimensionValues(value) {
|
|
1025
|
+
return Object.entries(value)
|
|
1026
|
+
.slice(0, 8)
|
|
1027
|
+
.map(([key, values]) => `${key}=[${values.slice(0, 8).join(', ')}]`)
|
|
1028
|
+
.join('; ');
|
|
1029
|
+
}
|
|
1030
|
+
function formatConversationTurnsForPrompt(turns) {
|
|
1031
|
+
return turns
|
|
1032
|
+
.map(cleanRecord)
|
|
1033
|
+
.filter((turn) => Boolean(turn))
|
|
1034
|
+
.slice(-4)
|
|
1035
|
+
.map((turn, index) => {
|
|
1036
|
+
const result = cleanRecord(turn.result);
|
|
1037
|
+
const columns = arrayValue(result?.columns)?.map(cleanOptionalString).filter(Boolean).slice(0, 6) ?? [];
|
|
1038
|
+
const values = cleanStringRecordArray(result?.dimensionValues);
|
|
1039
|
+
const valueText = values ? formatResultDimensionValues(values) : '';
|
|
1040
|
+
const cascade = cleanRecord(turn.cascade);
|
|
1041
|
+
const cascadeText = [
|
|
1042
|
+
cleanOptionalString(cascade?.terminalLane),
|
|
1043
|
+
cleanOptionalString(cascade?.routeTier),
|
|
1044
|
+
].filter(Boolean).join('/');
|
|
1045
|
+
return [
|
|
1046
|
+
`${index + 1}. ${cleanOptionalString(turn.question) ?? 'prior turn'}`,
|
|
1047
|
+
cascadeText ? `cascade=${cascadeText}` : '',
|
|
1048
|
+
cleanOptionalString(turn.answerSummary) ? `summary=${cleanOptionalString(turn.answerSummary)}` : '',
|
|
1049
|
+
columns.length ? `columns=${columns.join(', ')}` : '',
|
|
1050
|
+
valueText ? `values=${valueText}` : '',
|
|
1051
|
+
].filter(Boolean).join(' | ');
|
|
1052
|
+
})
|
|
1053
|
+
.join('\n');
|
|
1054
|
+
}
|
|
1055
|
+
function singlePriorValueDimension(priorValues) {
|
|
1056
|
+
const dims = Array.from(new Set(Object.keys(priorValues).map(normalizePriorValueDimension).filter(Boolean)));
|
|
1057
|
+
return dims.length === 1 ? dims[0] : undefined;
|
|
1058
|
+
}
|
|
1059
|
+
function normalizePriorValueDimension(value) {
|
|
1060
|
+
const lower = value.toLowerCase();
|
|
1061
|
+
if (lower.includes('category'))
|
|
1062
|
+
return 'category';
|
|
1063
|
+
if (lower.includes('product'))
|
|
1064
|
+
return 'product';
|
|
1065
|
+
if (lower.includes('customer'))
|
|
1066
|
+
return 'customer';
|
|
1067
|
+
if (lower.includes('account'))
|
|
1068
|
+
return 'account';
|
|
1069
|
+
if (lower.includes('user'))
|
|
1070
|
+
return 'user';
|
|
1071
|
+
if (lower.includes('region'))
|
|
1072
|
+
return 'region';
|
|
1073
|
+
if (lower.includes('segment'))
|
|
1074
|
+
return 'segment';
|
|
1075
|
+
if (lower.includes('channel'))
|
|
1076
|
+
return 'channel';
|
|
1077
|
+
return lower.replace(/[_\s-]+name$/, '').replace(/[^a-z0-9_ -]+/g, '').trim();
|
|
1078
|
+
}
|
|
1079
|
+
export const __test__ = {
|
|
1080
|
+
applyTopicShiftGuard,
|
|
1081
|
+
buildAnswerLoopTools,
|
|
1082
|
+
createCertifiedFitConfirmation,
|
|
1083
|
+
followUpFromConversationContext,
|
|
1084
|
+
inferFollowUpContext,
|
|
1085
|
+
formatCascadeOutcome,
|
|
1086
|
+
parseCertifiedFitConfirmation,
|
|
1087
|
+
rewriteFollowUpQuestion,
|
|
1088
|
+
};
|
|
418
1089
|
const GENERIC_FOLLOW_UP_WORDS = new Set([
|
|
419
1090
|
'a', 'about', 'again', 'all', 'also', 'and', 'as', 'be', 'block', 'can', 'could', 'data', 'do',
|
|
420
1091
|
'execute', 'for', 'from', 'get', 'give', 'import', 'it', 'its', 'let', 'lets', 'me', 'metrics',
|
|
@@ -423,6 +1094,9 @@ const GENERIC_FOLLOW_UP_WORDS = new Set([
|
|
|
423
1094
|
]);
|
|
424
1095
|
function isGenericFollowUp(question) {
|
|
425
1096
|
const lower = question.toLowerCase();
|
|
1097
|
+
if (/\b(?:combine|merge|join|final|summari[sz]e)\b/.test(lower) && /\b(?:previous|prior|above|these|those|results?|outputs?|turns?)\b/.test(lower)) {
|
|
1098
|
+
return true;
|
|
1099
|
+
}
|
|
426
1100
|
if (!/\b(block|data|execute|import|it|result|results|run|solution|that|this)\b/.test(lower))
|
|
427
1101
|
return false;
|
|
428
1102
|
const tokens = lower.match(/[a-z0-9_]+/g) ?? [];
|
|
@@ -431,8 +1105,9 @@ function isGenericFollowUp(question) {
|
|
|
431
1105
|
}
|
|
432
1106
|
function isDrilldownFollowUp(question) {
|
|
433
1107
|
const lower = question.toLowerCase();
|
|
434
|
-
|
|
435
|
-
|
|
1108
|
+
const deicticDrilldown = /\b(?:this|that|these|those|same|above|previous|prior)\s+(?:orders?|results?|rows?|customers?|products?|cat(?:egor|agor|ogor)(?:y|ies)|segments?|regions?)\b/.test(lower);
|
|
1109
|
+
return /\b(drill|break\s*down|slice|segment|filter|compare|split|why|changed?|change|driver|root cause|increase|decrease|drop|spike|variance|by|for|only|where|last week|this week|last month|this month|enterprise|region|customer|channel|product|category|categories|catagor(?:y|ies)|catogor(?:y|ies))\b/.test(lower)
|
|
1110
|
+
&& (deicticDrilldown || !/\b(what is|what are|define|definition|meaning of)\b/.test(lower));
|
|
436
1111
|
}
|
|
437
1112
|
function extractDrilldownFilters(question) {
|
|
438
1113
|
const filters = [];
|
|
@@ -462,7 +1137,7 @@ function extractDrilldownDimensions(question) {
|
|
|
462
1137
|
if (value)
|
|
463
1138
|
dims.push(value);
|
|
464
1139
|
}
|
|
465
|
-
for (const dim of ['segment', 'region', 'customer', 'channel', 'product', 'week', 'month']) {
|
|
1140
|
+
for (const dim of ['segment', 'region', 'customer', 'channel', 'product', 'category', 'week', 'month']) {
|
|
466
1141
|
if (new RegExp(`\\b${dim}\\b`, 'i').test(question))
|
|
467
1142
|
dims.push(dim);
|
|
468
1143
|
}
|
|
@@ -479,9 +1154,58 @@ function formatAgentAnswer(result) {
|
|
|
479
1154
|
const citations = result.citations.length > 0
|
|
480
1155
|
? '\n\nCitations:\n' + result.citations.map((c) => `- ${c.kind}: ${c.name}${c.provenance ? ` (${c.provenance})` : ''}`).join('\n')
|
|
481
1156
|
: '';
|
|
1157
|
+
const cascade = formatCascadeOutcome(result.cascade);
|
|
1158
|
+
const cascadeLine = cascade ? `\nCascade: ${cascade}` : '';
|
|
482
1159
|
const resultPreview = formatResultPreview(result.result);
|
|
483
|
-
const
|
|
484
|
-
|
|
1160
|
+
const dql = result.dqlArtifact?.source?.trim()
|
|
1161
|
+
? `\n\nDQL Artifact (${result.dqlArtifact.kind}):\n\`\`\`dql\n${result.dqlArtifact.source.trim()}\n\`\`\``
|
|
1162
|
+
: '';
|
|
1163
|
+
const sql = result.proposedSql ? `\n\nCompiled SQL preview:\n\`\`\`sql\n${result.proposedSql}\n\`\`\`` : '';
|
|
1164
|
+
return `[${badge}]${cascadeLine}\n\n${result.text}${resultPreview}${citations}${dql}${sql}`;
|
|
1165
|
+
}
|
|
1166
|
+
function formatCascadeOutcome(cascade) {
|
|
1167
|
+
if (!cascade?.terminalLane && !cascade?.routeTier)
|
|
1168
|
+
return undefined;
|
|
1169
|
+
const lane = formatCascadeLane(cascade.terminalLane);
|
|
1170
|
+
const tier = formatCascadeTier(cascade.routeTier);
|
|
1171
|
+
return [lane, tier].filter(Boolean).join(' · ') || undefined;
|
|
1172
|
+
}
|
|
1173
|
+
function formatCascadeLane(value) {
|
|
1174
|
+
switch (value) {
|
|
1175
|
+
case 'certified':
|
|
1176
|
+
return 'Lane 1 certified';
|
|
1177
|
+
case 'semantic':
|
|
1178
|
+
return 'Lane 2 semantic';
|
|
1179
|
+
case 'generated':
|
|
1180
|
+
return 'Lane 3 generated';
|
|
1181
|
+
case 'refusal':
|
|
1182
|
+
return 'Lane 4 refusal';
|
|
1183
|
+
default:
|
|
1184
|
+
return value ? formatLabel(value) : undefined;
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
function formatCascadeTier(value) {
|
|
1188
|
+
switch (value) {
|
|
1189
|
+
case 'certified_block':
|
|
1190
|
+
return 'Certified block';
|
|
1191
|
+
case 'semantic_metric':
|
|
1192
|
+
return 'Semantic metric';
|
|
1193
|
+
case 'generated_sql':
|
|
1194
|
+
return 'Generated SQL';
|
|
1195
|
+
case 'business_context':
|
|
1196
|
+
return 'Business context';
|
|
1197
|
+
case 'no_answer':
|
|
1198
|
+
return 'No answer';
|
|
1199
|
+
default:
|
|
1200
|
+
return value ? formatLabel(value) : undefined;
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
1203
|
+
function formatLabel(value) {
|
|
1204
|
+
return value
|
|
1205
|
+
.replace(/[_-]+/g, ' ')
|
|
1206
|
+
.replace(/\s+/g, ' ')
|
|
1207
|
+
.trim()
|
|
1208
|
+
.replace(/\b\w/g, (char) => char.toUpperCase());
|
|
485
1209
|
}
|
|
486
1210
|
function formatResultPreview(result) {
|
|
487
1211
|
if (!result)
|
|
@@ -528,13 +1252,34 @@ function escapeMarkdownTable(value) {
|
|
|
528
1252
|
}
|
|
529
1253
|
function emitDraftProposal(result, question, emit) {
|
|
530
1254
|
const isDrilldown = result.evidence?.route.some((step) => step.tool === 'propose_drilldown' && step.status === 'checked') ?? false;
|
|
1255
|
+
const dqlArtifact = result.dqlArtifact;
|
|
1256
|
+
const semanticArtifact = dqlArtifact?.kind === 'semantic_block' ? dqlArtifact : undefined;
|
|
531
1257
|
const proposal = {
|
|
532
1258
|
name: slugify(question).slice(0, 56) || 'ai_generated_analysis',
|
|
533
1259
|
domain: inferProposalDomain(result) ?? '',
|
|
534
1260
|
owner: `${process.env.USER ?? 'analyst'}@local`,
|
|
535
1261
|
description: result.text.slice(0, 240),
|
|
536
1262
|
sql: result.proposedSql,
|
|
537
|
-
|
|
1263
|
+
blockType: semanticArtifact ? 'semantic' : 'custom',
|
|
1264
|
+
...(dqlArtifact
|
|
1265
|
+
? {
|
|
1266
|
+
dqlSource: dqlArtifact.source,
|
|
1267
|
+
}
|
|
1268
|
+
: {}),
|
|
1269
|
+
...(semanticArtifact
|
|
1270
|
+
? {
|
|
1271
|
+
metrics: semanticArtifact.metrics,
|
|
1272
|
+
dimensions: semanticArtifact.dimensions,
|
|
1273
|
+
...(semanticArtifact.filters ? { filters: semanticArtifact.filters } : {}),
|
|
1274
|
+
...(semanticArtifact.timeDimension ? { timeDimension: semanticArtifact.timeDimension } : {}),
|
|
1275
|
+
}
|
|
1276
|
+
: {}),
|
|
1277
|
+
tags: [
|
|
1278
|
+
'ai-generated',
|
|
1279
|
+
'needs-review',
|
|
1280
|
+
semanticArtifact ? 'semantic' : result.sourceTier ?? 'dbt_manifest',
|
|
1281
|
+
...(isDrilldown ? ['drilldown'] : []),
|
|
1282
|
+
],
|
|
538
1283
|
chartType: result.suggestedViz,
|
|
539
1284
|
};
|
|
540
1285
|
emit({
|