@duckcodeailabs/dql-agent 1.6.8 → 1.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/answer-loop.d.ts +12 -0
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +1020 -53
- package/dist/answer-loop.js.map +1 -1
- package/dist/app-builder.d.ts +27 -0
- package/dist/app-builder.d.ts.map +1 -1
- package/dist/app-builder.js +210 -43
- package/dist/app-builder.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metadata/analysis-planner.d.ts +52 -0
- package/dist/metadata/analysis-planner.d.ts.map +1 -0
- package/dist/metadata/analysis-planner.js +632 -0
- package/dist/metadata/analysis-planner.js.map +1 -0
- package/dist/metadata/catalog.d.ts +32 -0
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +705 -22
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/sql-context-validation.d.ts.map +1 -1
- package/dist/metadata/sql-context-validation.js +41 -13
- package/dist/metadata/sql-context-validation.js.map +1 -1
- package/dist/metadata/sql-shape.d.ts +13 -0
- package/dist/metadata/sql-shape.d.ts.map +1 -0
- package/dist/metadata/sql-shape.js +92 -0
- package/dist/metadata/sql-shape.js.map +1 -0
- package/package.json +4 -4
package/dist/answer-loop.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { buildSkillBlockHints, buildSkillsPrompt } from './skills/loader.js';
|
|
18
18
|
import { validateSqlAgainstLocalContext } from './metadata/sql-context-validation.js';
|
|
19
|
+
import { compactSqlSnippet, extractSimpleSelectShape, selectExpressionOutputName, } from './metadata/sql-shape.js';
|
|
19
20
|
const CERTIFIED_HIT_THRESHOLD = 0.18;
|
|
20
21
|
const HARD_NEGATIVE_RATIO = 0.5;
|
|
21
22
|
const EXECUTABLE_ARTIFACT_KINDS = ['block', 'dashboard', 'app', 'notebook'];
|
|
@@ -41,6 +42,7 @@ export async function answer(input) {
|
|
|
41
42
|
const semanticHits = kg.search({ query: question, domain, kinds: SEMANTIC_KINDS, limit: 12 });
|
|
42
43
|
const manifestHits = kg.search({ query: question, domain, kinds: MANIFEST_KINDS, limit: 12 });
|
|
43
44
|
const considered = mergeHits(artifactHits, semanticHits, manifestHits, kg.search({ query: question, domain, limit: 10 })).slice(0, 30);
|
|
45
|
+
const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext ?? [], input.contextPack);
|
|
44
46
|
const catalogRoute = input.contextPack?.routeDecision;
|
|
45
47
|
const fallbackIntent = classifyAgentIntent({
|
|
46
48
|
question,
|
|
@@ -48,7 +50,7 @@ export async function answer(input) {
|
|
|
48
50
|
artifactHits,
|
|
49
51
|
semanticHits,
|
|
50
52
|
manifestHits,
|
|
51
|
-
schemaContext
|
|
53
|
+
schemaContext,
|
|
52
54
|
});
|
|
53
55
|
const intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
|
|
54
56
|
// Stage 1: certified artifact match. Blocks can be executed; dashboards,
|
|
@@ -104,7 +106,7 @@ export async function answer(input) {
|
|
|
104
106
|
intent,
|
|
105
107
|
routeReason: catalogRoute?.reason ?? 'The question matched a certified DQL artifact closely enough to answer without generating new SQL.',
|
|
106
108
|
selectedNodes: [artifactHit.node],
|
|
107
|
-
schemaContext
|
|
109
|
+
schemaContext,
|
|
108
110
|
sql: result?.sql,
|
|
109
111
|
suggestedViz: result?.chartConfig ? chartNameFromConfig(result.chartConfig) : undefined,
|
|
110
112
|
});
|
|
@@ -147,13 +149,13 @@ export async function answer(input) {
|
|
|
147
149
|
};
|
|
148
150
|
}
|
|
149
151
|
if (intent === 'clarify' || catalogRoute?.route === 'clarify') {
|
|
150
|
-
const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered,
|
|
152
|
+
const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered, schemaContext);
|
|
151
153
|
const analysisPlan = buildAnalysisPlan({
|
|
152
154
|
question,
|
|
153
155
|
intent,
|
|
154
156
|
routeReason: catalogRoute?.reason ?? 'No certified artifact, semantic object, dbt/source table, or runtime schema match was strong enough to safely generate SQL.',
|
|
155
157
|
selectedNodes: considered.slice(0, 4).map((hit) => hit.node),
|
|
156
|
-
schemaContext
|
|
158
|
+
schemaContext,
|
|
157
159
|
assumptions: catalogRoute?.missingContext.length
|
|
158
160
|
? catalogRoute.missingContext.map((item) => item.message)
|
|
159
161
|
: ['Need a clearer business object, measure, or grain before querying.'],
|
|
@@ -195,7 +197,7 @@ export async function answer(input) {
|
|
|
195
197
|
const reviewRequiredArtifactHits = artifactHits
|
|
196
198
|
.filter((hit) => hit.score >= CERTIFIED_HIT_THRESHOLD && !isCertifiedHit(hit, kg))
|
|
197
199
|
.slice(0, 4);
|
|
198
|
-
const trustedArtifactContext = rankGeneratedContextHits(executableArtifactHits.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId)),
|
|
200
|
+
const trustedArtifactContext = rankGeneratedContextHits(executableArtifactHits.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId)), schemaContext, question)
|
|
199
201
|
.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId))
|
|
200
202
|
.slice(0, 5);
|
|
201
203
|
const contextHits = activeTier === 'semantic_layer'
|
|
@@ -213,13 +215,13 @@ export async function answer(input) {
|
|
|
213
215
|
messages.push({ role: 'system', content: skillsPrompt });
|
|
214
216
|
messages.push({
|
|
215
217
|
role: 'system',
|
|
216
|
-
content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp,
|
|
218
|
+
content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack),
|
|
217
219
|
});
|
|
218
220
|
messages.push({ role: 'user', content: question });
|
|
219
221
|
const localProposal = buildSchemaAwareProposal({
|
|
220
222
|
question,
|
|
221
223
|
intent,
|
|
222
|
-
schemaContext
|
|
224
|
+
schemaContext,
|
|
223
225
|
followUp: input.followUp,
|
|
224
226
|
contextPack: input.contextPack,
|
|
225
227
|
}) ?? buildContextPackAwareProposal({
|
|
@@ -304,7 +306,7 @@ export async function answer(input) {
|
|
|
304
306
|
intent,
|
|
305
307
|
routeReason: catalogRoute?.reason ?? 'Generated SQL failed metadata context validation before preview execution or draft capture.',
|
|
306
308
|
selectedNodes: contextNodes,
|
|
307
|
-
schemaContext
|
|
309
|
+
schemaContext,
|
|
308
310
|
sql: parsed.sql,
|
|
309
311
|
suggestedViz: parsed.viz ?? 'table',
|
|
310
312
|
assumptions: [
|
|
@@ -363,7 +365,7 @@ export async function answer(input) {
|
|
|
363
365
|
sourceTier: 'memory',
|
|
364
366
|
provenance: m.source,
|
|
365
367
|
})),
|
|
366
|
-
...schemaCitations(
|
|
368
|
+
...schemaCitations(schemaContext, Math.max(0, 4 - contextNodes.length)),
|
|
367
369
|
];
|
|
368
370
|
let result;
|
|
369
371
|
let executionError;
|
|
@@ -375,7 +377,7 @@ export async function answer(input) {
|
|
|
375
377
|
catch (err) {
|
|
376
378
|
executionError = err instanceof Error ? err.message : String(err);
|
|
377
379
|
if (isRetryableGeneratedSqlError(executionError)) {
|
|
378
|
-
const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError,
|
|
380
|
+
const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError, schemaContext);
|
|
379
381
|
if (localRepairSql) {
|
|
380
382
|
repairAttempts = 1;
|
|
381
383
|
parsed.sql = localRepairSql;
|
|
@@ -394,7 +396,7 @@ export async function answer(input) {
|
|
|
394
396
|
question,
|
|
395
397
|
parsed,
|
|
396
398
|
executionError,
|
|
397
|
-
schemaContext
|
|
399
|
+
schemaContext,
|
|
398
400
|
signal: input.signal,
|
|
399
401
|
});
|
|
400
402
|
const repaired = parseProposal(repairedRaw);
|
|
@@ -422,18 +424,20 @@ export async function answer(input) {
|
|
|
422
424
|
? 'The user asked for a drill-through or follow-up, so DQL generated review-required SQL from the prior context and current metadata.'
|
|
423
425
|
: 'The question asks for a custom analysis, ranking, breakdown, comparison, or grain that should not be answered by a loose certified block match.'),
|
|
424
426
|
selectedNodes: contextNodes,
|
|
425
|
-
schemaContext
|
|
427
|
+
schemaContext,
|
|
426
428
|
sql: parsed.sql,
|
|
427
429
|
suggestedViz: parsed.viz ?? 'table',
|
|
428
430
|
assumptions: [
|
|
429
431
|
'Generated SQL is an uncertified preview until an analyst reviews and promotes it.',
|
|
430
432
|
...(localProposal ? ['A local metadata planner selected a review-required SQL grain before provider generation.'] : []),
|
|
433
|
+
...contextValidation.warnings,
|
|
431
434
|
...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
|
|
432
435
|
],
|
|
433
436
|
repairAttempts,
|
|
434
437
|
});
|
|
435
438
|
const validationWarnings = [
|
|
436
439
|
...(input.contextPack?.warnings ?? []),
|
|
440
|
+
...contextValidation.warnings,
|
|
437
441
|
...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
|
|
438
442
|
];
|
|
439
443
|
let draftBlock;
|
|
@@ -494,7 +498,7 @@ export async function answer(input) {
|
|
|
494
498
|
activeTier,
|
|
495
499
|
intent,
|
|
496
500
|
contextNodes,
|
|
497
|
-
schemaContext
|
|
501
|
+
schemaContext,
|
|
498
502
|
followUp: input.followUp,
|
|
499
503
|
businessHits,
|
|
500
504
|
semanticHits,
|
|
@@ -527,9 +531,10 @@ Rules:
|
|
|
527
531
|
4. Provide a one-paragraph natural-language summary BEFORE the SQL block.
|
|
528
532
|
5. Suggest a visualization type from this list, on a line starting with "Viz:":
|
|
529
533
|
line, bar, area, pie, single_value, table, pivot, kpi.
|
|
530
|
-
6. NEVER fabricate column names that are not present in the supplied schema
|
|
531
|
-
|
|
532
|
-
|
|
534
|
+
6. NEVER fabricate column names that are not present in the supplied schema,
|
|
535
|
+
dbt metadata, or certified source SQL shape context. If a requested filter
|
|
536
|
+
value is supplied as a matched value, prefer the table and column that
|
|
537
|
+
matched that value.
|
|
533
538
|
7. Return one read-only SELECT or WITH query for the local warehouse/runtime.
|
|
534
539
|
Do NOT use dbt/Jinja macros such as {{ ref(...) }} or {{ source(...) }} in
|
|
535
540
|
proposed SQL. Do not emit INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, COPY,
|
|
@@ -593,6 +598,20 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
|
|
|
593
598
|
return `${intentSection}\n\n${blockSection}${businessSection}${otherSection}${schemaSection}${contextPackSection}${memorySection}${extraSection}${followUpSection}`;
|
|
594
599
|
}
|
|
595
600
|
function renderContextPackForPrompt(contextPack) {
|
|
601
|
+
const questionPlan = contextPack.questionPlan
|
|
602
|
+
? [
|
|
603
|
+
`Question plan: ${contextPack.questionPlan.mode} -> ${contextPack.questionPlan.routeIntent}`,
|
|
604
|
+
contextPack.questionPlan.entities.length
|
|
605
|
+
? `Entities: ${contextPack.questionPlan.entities.map((entity) => entity.text).join(', ')}`
|
|
606
|
+
: '',
|
|
607
|
+
contextPack.questionPlan.metricTerms.length ? `Metric terms: ${contextPack.questionPlan.metricTerms.join(', ')}` : '',
|
|
608
|
+
contextPack.questionPlan.dimensionTerms.length ? `Dimension terms: ${contextPack.questionPlan.dimensionTerms.join(', ')}` : '',
|
|
609
|
+
`Answer shape: ${contextPack.questionPlan.outputShape}`,
|
|
610
|
+
].filter(Boolean).join('\n')
|
|
611
|
+
: '';
|
|
612
|
+
const certifiedApplicability = contextPack.routeDecision?.certifiedApplicability
|
|
613
|
+
? `\nCertified applicability: ${contextPack.routeDecision.certifiedApplicability.name} is ${contextPack.routeDecision.certifiedApplicability.kind} (${contextPack.routeDecision.certifiedApplicability.reasons.join('; ')})`
|
|
614
|
+
: '';
|
|
596
615
|
const warnings = contextPack.warnings.length
|
|
597
616
|
? `Warnings:\n${contextPack.warnings.slice(0, 8).map((warning) => `- ${warning}`).join('\n')}\n`
|
|
598
617
|
: '';
|
|
@@ -609,21 +628,101 @@ function renderContextPackForPrompt(contextPack) {
|
|
|
609
628
|
? `\nCandidate conflicts:\n${contextPack.retrievalDiagnostics.candidateConflicts.slice(0, 4).map((conflict) => `- ${conflict.reason} ${conflict.prompt}`).join('\n')}`
|
|
610
629
|
: '';
|
|
611
630
|
const route = contextPack.routeDecision
|
|
612
|
-
? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
|
|
631
|
+
? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
|
|
613
632
|
: '';
|
|
614
|
-
const allowed = contextPack
|
|
615
|
-
|
|
633
|
+
const allowed = renderAllowedSqlRelationsForPrompt(contextPack);
|
|
634
|
+
const joins = renderCandidateJoinsForPrompt(contextPack);
|
|
635
|
+
const relationDiagnostics = contextPack.retrievalDiagnostics?.selectedRelations?.length
|
|
636
|
+
? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, 8).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
|
|
616
637
|
: '';
|
|
638
|
+
const sourceSql = renderSourceBlockSqlContext(contextPack);
|
|
617
639
|
return [
|
|
618
640
|
`context_pack_id: ${contextPack.id}`,
|
|
619
641
|
`trust_label: ${contextPack.trustLabel}`,
|
|
642
|
+
questionPlan.trim(),
|
|
620
643
|
route.trim(),
|
|
621
644
|
warnings.trim(),
|
|
622
645
|
`Selected evidence:\n${objects || '- none'}`,
|
|
623
646
|
allowed.trim(),
|
|
647
|
+
joins.trim(),
|
|
648
|
+
relationDiagnostics.trim(),
|
|
649
|
+
sourceSql.trim(),
|
|
624
650
|
conflicts.trim(),
|
|
625
651
|
].filter(Boolean).join('\n');
|
|
626
652
|
}
|
|
653
|
+
function renderCandidateJoinsForPrompt(contextPack) {
|
|
654
|
+
const joins = contextPack.retrievalDiagnostics.selectedJoinPaths?.length
|
|
655
|
+
? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, 8).map((join) => ({
|
|
656
|
+
leftRelation: join.leftRelation,
|
|
657
|
+
leftColumn: join.leftColumn,
|
|
658
|
+
rightRelation: join.rightRelation,
|
|
659
|
+
rightColumn: join.rightColumn,
|
|
660
|
+
reason: join.reason,
|
|
661
|
+
}))
|
|
662
|
+
: buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, 8);
|
|
663
|
+
if (joins.length === 0)
|
|
664
|
+
return '';
|
|
665
|
+
return [
|
|
666
|
+
'Suggested join paths from selected metadata:',
|
|
667
|
+
...joins.map((join) => `- ${join.leftRelation}.${join.leftColumn} -> ${join.rightRelation}.${join.rightColumn}${join.reason ? ` (${join.reason})` : ''}`),
|
|
668
|
+
].join('\n');
|
|
669
|
+
}
|
|
670
|
+
function renderAllowedSqlRelationsForPrompt(contextPack) {
|
|
671
|
+
const relations = contextPack.allowedSqlContext?.relations ?? [];
|
|
672
|
+
if (relations.length === 0)
|
|
673
|
+
return '';
|
|
674
|
+
const selectedLookup = selectedRelationLookup(contextPack);
|
|
675
|
+
const cards = relations.slice(0, 12).map((relation, index) => {
|
|
676
|
+
const selection = relationSelectionFor(relation.relation, selectedLookup);
|
|
677
|
+
const rank = selection?.rank ?? index + 1;
|
|
678
|
+
const score = typeof selection?.score === 'number' ? `, score ${selection.score.toFixed(1)}` : '';
|
|
679
|
+
const reason = selection?.reason ? `\n why selected: ${selection.reason}` : '';
|
|
680
|
+
const columns = relation.columns.length
|
|
681
|
+
? relation.columns.slice(0, 32).map(formatRelationColumnForPrompt).join(', ')
|
|
682
|
+
: '(columns unavailable; use certified source SQL shape or inspect metadata before inventing columns)';
|
|
683
|
+
return [
|
|
684
|
+
`- [rank ${rank}${score}] ${relation.relation} (${relation.source})`,
|
|
685
|
+
reason,
|
|
686
|
+
`\n columns: ${columns}`,
|
|
687
|
+
].join('');
|
|
688
|
+
});
|
|
689
|
+
return [
|
|
690
|
+
'Selected SQL relation context:',
|
|
691
|
+
'Use these ranked relations and columns as the primary SQL-generation boundary. Prefer lower rank when multiple relations look plausible.',
|
|
692
|
+
...cards,
|
|
693
|
+
].join('\n');
|
|
694
|
+
}
|
|
695
|
+
function formatRelationColumnForPrompt(column) {
|
|
696
|
+
const type = column.type ? ` ${column.type}` : '';
|
|
697
|
+
const description = column.description ? ` - ${column.description.replace(/\s+/g, ' ').trim()}` : '';
|
|
698
|
+
const samples = column.sampleValues?.length
|
|
699
|
+
? `; matched values: ${column.sampleValues.slice(0, 4).map(formatPromptValue).join(', ')}`
|
|
700
|
+
: '';
|
|
701
|
+
return `${column.name}${type}${description}${samples}`;
|
|
702
|
+
}
|
|
703
|
+
function renderSourceBlockSqlContext(contextPack) {
|
|
704
|
+
const sources = contextPack.allowedSqlContext?.sourceBlockSql ?? [];
|
|
705
|
+
if (sources.length === 0)
|
|
706
|
+
return '';
|
|
707
|
+
const lines = sources.slice(0, 5).map((source) => {
|
|
708
|
+
const shape = extractSimpleSelectShape(source.sql);
|
|
709
|
+
const projectedColumns = shape
|
|
710
|
+
? shape.selectExpressions
|
|
711
|
+
.map(selectExpressionOutputName)
|
|
712
|
+
.filter((value) => Boolean(value))
|
|
713
|
+
.slice(0, 24)
|
|
714
|
+
.join(', ')
|
|
715
|
+
: '';
|
|
716
|
+
const snippet = compactSqlSnippet(source.sql, 280);
|
|
717
|
+
return [
|
|
718
|
+
`- ${source.name}${source.status ? ` (${source.status})` : ''}`,
|
|
719
|
+
shape?.relation ? ` relation: ${shape.relation}` : '',
|
|
720
|
+
projectedColumns ? ` projected columns: ${projectedColumns}` : '',
|
|
721
|
+
snippet ? ` sql: ${snippet}` : '',
|
|
722
|
+
].filter(Boolean).join('\n');
|
|
723
|
+
});
|
|
724
|
+
return `Certified source SQL shape context:\n${lines.join('\n')}`;
|
|
725
|
+
}
|
|
627
726
|
function contextPackCitations(contextPack, limit) {
|
|
628
727
|
if (!contextPack)
|
|
629
728
|
return [];
|
|
@@ -789,61 +888,794 @@ function buildSchemaAwareProposal(input) {
|
|
|
789
888
|
const drilldownProposal = buildMatchedEntityDrilldownProposal({ ...input, schemaContext });
|
|
790
889
|
if (drilldownProposal)
|
|
791
890
|
return drilldownProposal;
|
|
891
|
+
const profileProposal = buildEntityProfileProposal({ ...input, schemaContext });
|
|
892
|
+
if (profileProposal)
|
|
893
|
+
return profileProposal;
|
|
792
894
|
if (isFilteredEntityQuestion(input.question))
|
|
793
895
|
return undefined;
|
|
794
896
|
const lower = input.question.toLowerCase();
|
|
795
897
|
const asksForCustomerPerformance = /\bcustomers?\b/.test(lower)
|
|
796
|
-
&& /\border|orders|spend|revenue|perform|performed|better|top|best|rank|ranking
|
|
898
|
+
&& /\border|orders|spend|revenue|perform|performed|better|top|best|rank|ranking|bottom|least|fewest|lowest|less|worst|underperform/.test(lower)
|
|
797
899
|
&& !/\b(order details|specific orders|each order|all orders|order line|line item)\b/.test(lower);
|
|
798
|
-
if (
|
|
900
|
+
if (asksForCustomerPerformance) {
|
|
901
|
+
const direction = customerRankingDirectionFromText(input.question);
|
|
902
|
+
const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
|
|
903
|
+
const orderFocused = /\border|orders|order count|ordered\b/.test(lower)
|
|
904
|
+
&& !/\b(spend|revenue|sales|amount|lifetime spend|value)\b/.test(lower);
|
|
905
|
+
const customers = findSchemaTable(schemaContext, ['customers', 'customer']);
|
|
906
|
+
if (customers) {
|
|
907
|
+
const customerName = findSchemaColumn(customers, ['customer_name', 'name', 'full_name']);
|
|
908
|
+
const orderCount = findSchemaColumn(customers, ['count_lifetime_orders', 'lifetime_orders', 'order_count', 'orders_count', 'orders']);
|
|
909
|
+
const spend = findSchemaColumn(customers, ['lifetime_spend', 'total_lifetime_spend', 'customer_lifetime_value', 'total_revenue', 'revenue']);
|
|
910
|
+
if (customerName && orderCount && spend) {
|
|
911
|
+
const primarySort = orderFocused ? orderCount : spend;
|
|
912
|
+
const secondarySort = orderFocused ? spend : orderCount;
|
|
913
|
+
const rankingLabel = direction === 'bottom'
|
|
914
|
+
? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
|
|
915
|
+
: orderFocused ? 'Top customers by order count' : 'Top performing customers';
|
|
916
|
+
return {
|
|
917
|
+
text: `${rankingLabel}, with ${businessMeasurePhrase(orderCount)} and ${businessMeasurePhrase(spend)} for context. This is AI-generated and needs analyst review before certification.`,
|
|
918
|
+
sql: [
|
|
919
|
+
'SELECT',
|
|
920
|
+
` ${sqlIdentifier(customerName)} AS customer_name,`,
|
|
921
|
+
` ${sqlIdentifier(orderCount)} AS orders,`,
|
|
922
|
+
` ROUND(${sqlIdentifier(spend)}, 2) AS lifetime_spend`,
|
|
923
|
+
`FROM ${sqlRelation(customers.relation)}`,
|
|
924
|
+
`ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
|
|
925
|
+
'LIMIT 10',
|
|
926
|
+
].join('\n'),
|
|
927
|
+
viz: 'table',
|
|
928
|
+
};
|
|
929
|
+
}
|
|
930
|
+
const customerId = findSchemaColumn(customers, ['customer_id', 'id']);
|
|
931
|
+
const orders = findSchemaTable(schemaContext, ['orders', 'order']);
|
|
932
|
+
const orderCustomerId = orders ? findSchemaColumn(orders, ['customer_id', 'customer']) : undefined;
|
|
933
|
+
const orderTotal = orders ? findSchemaColumn(orders, ['order_total', 'total_order_amount', 'total_amount', 'amount', 'subtotal']) : undefined;
|
|
934
|
+
const orderId = orders ? findSchemaColumn(orders, ['order_id', 'id']) : undefined;
|
|
935
|
+
if (orders && customerName && customerId && orderCustomerId && orderTotal) {
|
|
936
|
+
const countExpression = orderId ? `COUNT(DISTINCT o.${sqlIdentifier(orderId)})` : 'COUNT(*)';
|
|
937
|
+
const primarySort = orderFocused ? 'orders' : 'lifetime_spend';
|
|
938
|
+
const secondarySort = orderFocused ? 'lifetime_spend' : 'orders';
|
|
939
|
+
const rankingLabel = direction === 'bottom'
|
|
940
|
+
? orderFocused ? 'Customers with the fewest orders' : 'Lowest performing customers'
|
|
941
|
+
: orderFocused ? 'Top customers by order count' : 'Top performing customers';
|
|
942
|
+
return {
|
|
943
|
+
text: `${rankingLabel} from order totals, with order count for context. This is AI-generated and needs analyst review before certification.`,
|
|
944
|
+
sql: [
|
|
945
|
+
'SELECT',
|
|
946
|
+
` c.${sqlIdentifier(customerName)} AS customer_name,`,
|
|
947
|
+
` ${countExpression} AS orders,`,
|
|
948
|
+
` ROUND(SUM(o.${sqlIdentifier(orderTotal)}), 2) AS lifetime_spend`,
|
|
949
|
+
`FROM ${sqlRelation(orders.relation)} AS o`,
|
|
950
|
+
`JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
|
|
951
|
+
`GROUP BY c.${sqlIdentifier(customerName)}`,
|
|
952
|
+
`ORDER BY ${sqlIdentifier(primarySort)} ${orderDirection}, ${sqlIdentifier(secondarySort)} ${orderDirection}`,
|
|
953
|
+
'LIMIT 10',
|
|
954
|
+
].join('\n'),
|
|
955
|
+
viz: 'table',
|
|
956
|
+
};
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
const genericJoinProposal = buildGenericJoinProposal({ ...input, schemaContext });
|
|
961
|
+
if (genericJoinProposal)
|
|
962
|
+
return genericJoinProposal;
|
|
963
|
+
const genericProposal = buildGenericSingleTableProposal({ ...input, schemaContext });
|
|
964
|
+
if (genericProposal)
|
|
965
|
+
return genericProposal;
|
|
966
|
+
return undefined;
|
|
967
|
+
}
|
|
968
|
+
function customerRankingDirectionFromText(text) {
|
|
969
|
+
const lower = text.toLowerCase();
|
|
970
|
+
if (/\b(less|lesser)\s+(?:order|orders|ordering)\b/.test(lower) || /\borders?\s+(?:less|least|fewest|lowest)\b/.test(lower)) {
|
|
971
|
+
return 'bottom';
|
|
972
|
+
}
|
|
973
|
+
return rankingDirectionFromText(text) ?? 'top';
|
|
974
|
+
}
|
|
975
|
+
function buildGenericJoinProposal(input) {
|
|
976
|
+
const planMode = input.contextPack?.questionPlan?.mode;
|
|
977
|
+
if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
|
|
799
978
|
return undefined;
|
|
800
|
-
|
|
801
|
-
|
|
979
|
+
}
|
|
980
|
+
const lower = input.question.toLowerCase();
|
|
981
|
+
if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
|
|
982
|
+
return undefined;
|
|
983
|
+
}
|
|
984
|
+
if (!/\b(by\s+[a-z]|per\s+[a-z]|for each|group(?:ed)? by|split|segment|break\s*down|breakdown|compare)\b/i.test(lower)) {
|
|
985
|
+
return undefined;
|
|
986
|
+
}
|
|
987
|
+
const candidate = pickGenericJoinCandidate(input.schemaContext, input.question);
|
|
988
|
+
if (!candidate)
|
|
802
989
|
return undefined;
|
|
803
|
-
const
|
|
804
|
-
const
|
|
805
|
-
const
|
|
806
|
-
|
|
990
|
+
const direction = rankingDirectionFromText(input.question);
|
|
991
|
+
const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
|
|
992
|
+
const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
|
|
993
|
+
const chart = isTimeLikeGenericColumn(candidate.dimensionColumn) ? 'line' : 'bar';
|
|
994
|
+
const metricExpression = qualifiedMetricExpression(candidate.metric, 'f');
|
|
995
|
+
const metricAlias = candidate.metric.alias;
|
|
996
|
+
const dimensionAlias = candidate.dimensionColumn;
|
|
997
|
+
return {
|
|
998
|
+
text: `Prepared a review-required ${businessMeasurePhrase(candidate.metric.column)} breakdown by ${humanizeIdentifier(candidate.dimensionColumn)} by joining ${candidate.metricTable.relation} to ${candidate.dimensionTable.relation}. This result is uncertified until reviewed and promoted.`,
|
|
999
|
+
sql: [
|
|
1000
|
+
'SELECT',
|
|
1001
|
+
` d.${sqlIdentifier(candidate.dimensionColumn)} AS ${sqlIdentifier(dimensionAlias)},`,
|
|
1002
|
+
` ${metricExpression} AS ${sqlIdentifier(metricAlias)}`,
|
|
1003
|
+
`FROM ${sqlRelation(candidate.metricTable.relation)} AS f`,
|
|
1004
|
+
`JOIN ${sqlRelation(candidate.dimensionTable.relation)} AS d ON f.${sqlIdentifier(candidate.metricJoinColumn)} = d.${sqlIdentifier(candidate.dimensionJoinColumn)}`,
|
|
1005
|
+
`GROUP BY d.${sqlIdentifier(candidate.dimensionColumn)}`,
|
|
1006
|
+
`ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
|
|
1007
|
+
`LIMIT ${limit}`,
|
|
1008
|
+
].join('\n'),
|
|
1009
|
+
viz: chart,
|
|
1010
|
+
};
|
|
1011
|
+
}
|
|
1012
|
+
function buildGenericSingleTableProposal(input) {
|
|
1013
|
+
const planMode = input.contextPack?.questionPlan?.mode;
|
|
1014
|
+
if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
|
|
1015
|
+
return undefined;
|
|
1016
|
+
}
|
|
1017
|
+
const lower = input.question.toLowerCase();
|
|
1018
|
+
if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
|
|
1019
|
+
return undefined;
|
|
1020
|
+
}
|
|
1021
|
+
if (!/\b(show|list|top|bottom|best|worst|highest|lowest|least|fewest|rank|ranking|compare|trend|by\s+[a-z]|how many|number of|revenue|sales|orders?|points?|score|count|total|average|avg|sum)\b/i.test(lower)) {
|
|
1022
|
+
return undefined;
|
|
1023
|
+
}
|
|
1024
|
+
const table = pickGenericAnalysisTable(input.schemaContext, input.question);
|
|
1025
|
+
if (!table)
|
|
1026
|
+
return undefined;
|
|
1027
|
+
const metric = inferGenericMetric(table, input.question);
|
|
1028
|
+
if (!metric)
|
|
1029
|
+
return undefined;
|
|
1030
|
+
const dimensions = augmentGenericRankingDimensions(table, inferGenericDimensions(table, input.question, metric.column), metric, input.question).slice(0, 2);
|
|
1031
|
+
const direction = rankingDirectionFromText(input.question);
|
|
1032
|
+
const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
|
|
1033
|
+
const metricAlias = metric.alias;
|
|
1034
|
+
if (dimensions.length === 0) {
|
|
1035
|
+
return {
|
|
1036
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} summary from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
1037
|
+
sql: [
|
|
1038
|
+
'SELECT',
|
|
1039
|
+
` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
|
|
1040
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1041
|
+
].join('\n'),
|
|
1042
|
+
viz: 'single_value',
|
|
1043
|
+
};
|
|
1044
|
+
}
|
|
1045
|
+
if (direction && metric.preAggregated) {
|
|
807
1046
|
return {
|
|
808
|
-
text: `
|
|
1047
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} ranking by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
809
1048
|
sql: [
|
|
810
1049
|
'SELECT',
|
|
811
|
-
` ${sqlIdentifier(
|
|
812
|
-
` ${sqlIdentifier(
|
|
813
|
-
`
|
|
814
|
-
`
|
|
815
|
-
`
|
|
816
|
-
'LIMIT 10',
|
|
1050
|
+
...dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`),
|
|
1051
|
+
` ${sqlIdentifier(metric.column)} AS ${sqlIdentifier(metric.column)}`,
|
|
1052
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1053
|
+
`ORDER BY ${sqlIdentifier(metric.column)} ${direction === 'bottom' ? 'ASC' : 'DESC'}`,
|
|
1054
|
+
`LIMIT ${limit}`,
|
|
817
1055
|
].join('\n'),
|
|
818
1056
|
viz: 'table',
|
|
819
1057
|
};
|
|
820
1058
|
}
|
|
821
|
-
const
|
|
822
|
-
const
|
|
823
|
-
|
|
1059
|
+
const selectDimensions = dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`);
|
|
1060
|
+
const groupBy = dimensions.map(sqlIdentifier).join(', ');
|
|
1061
|
+
const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
|
|
1062
|
+
const chart = dimensions.some(isTimeLikeGenericColumn) ? 'line' : 'bar';
|
|
1063
|
+
return {
|
|
1064
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} breakdown by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
1065
|
+
sql: [
|
|
1066
|
+
'SELECT',
|
|
1067
|
+
...selectDimensions,
|
|
1068
|
+
` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
|
|
1069
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1070
|
+
`GROUP BY ${groupBy}`,
|
|
1071
|
+
`ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
|
|
1072
|
+
`LIMIT ${limit}`,
|
|
1073
|
+
].join('\n'),
|
|
1074
|
+
viz: chart,
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
function pickGenericAnalysisTable(schemaContext, question) {
|
|
1078
|
+
return schemaContext
|
|
1079
|
+
.filter((table) => table.columns.length > 0)
|
|
1080
|
+
.map((table, index) => ({
|
|
1081
|
+
table,
|
|
1082
|
+
metric: inferGenericMetric(table, question),
|
|
1083
|
+
dimensions: inferGenericDimensions(table, question),
|
|
1084
|
+
index,
|
|
1085
|
+
}))
|
|
1086
|
+
.filter((candidate) => candidate.metric)
|
|
1087
|
+
.sort((a, b) => genericTableScore(b.table, question, b.metric, b.dimensions) -
|
|
1088
|
+
genericTableScore(a.table, question, a.metric, a.dimensions) ||
|
|
1089
|
+
a.index - b.index)[0]?.table;
|
|
1090
|
+
}
|
|
1091
|
+
function genericTableScore(table, question, metric, dimensions) {
|
|
1092
|
+
const questionTokens = meaningfulTokens(question);
|
|
1093
|
+
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1094
|
+
const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
|
|
1095
|
+
const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
|
|
1096
|
+
const selectedRelationBoost = table.selectionRank
|
|
1097
|
+
? Math.max(0, 56 - table.selectionRank * 3) + Math.min(table.selectionScore ?? 0, 24)
|
|
1098
|
+
: 0;
|
|
1099
|
+
return selectedRelationBoost + overlap * 8 + (metric?.score ?? 0) + dimensions.length * 8 + Math.min(table.columns.length, 20) * 0.3;
|
|
1100
|
+
}
|
|
1101
|
+
function pickGenericJoinCandidate(schemaContext, question) {
|
|
1102
|
+
const requestedDimensions = requestedDimensionPhrases(question);
|
|
1103
|
+
if (requestedDimensions.length === 0)
|
|
1104
|
+
return undefined;
|
|
1105
|
+
const metricCandidates = schemaContext
|
|
1106
|
+
.filter((table) => table.columns.length > 0)
|
|
1107
|
+
.map((table, index) => ({
|
|
1108
|
+
table,
|
|
1109
|
+
metric: inferGenericMetric(table, question),
|
|
1110
|
+
localDimensions: inferGenericDimensions(table, question),
|
|
1111
|
+
index,
|
|
1112
|
+
}))
|
|
1113
|
+
.filter((candidate) => Boolean(candidate.metric))
|
|
1114
|
+
.filter((candidate) => !candidate.localDimensions.some((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension)));
|
|
1115
|
+
const candidates = [];
|
|
1116
|
+
for (const metricCandidate of metricCandidates) {
|
|
1117
|
+
for (const dimensionTable of schemaContext) {
|
|
1118
|
+
if (normalizeRelationKey(dimensionTable.relation) === normalizeRelationKey(metricCandidate.table.relation))
|
|
1119
|
+
continue;
|
|
1120
|
+
const dimensionColumn = pickRequestedDimensionColumn(dimensionTable, question, requestedDimensions);
|
|
1121
|
+
if (!dimensionColumn || namesEqualLoose(dimensionColumn, metricCandidate.metric.column))
|
|
1122
|
+
continue;
|
|
1123
|
+
const join = pickJoinColumns(metricCandidate.table, dimensionTable);
|
|
1124
|
+
if (!join)
|
|
1125
|
+
continue;
|
|
1126
|
+
const dimensionScore = genericDimensionScore({ name: dimensionColumn, type: findColumnType(dimensionTable, dimensionColumn) }, question, Math.max(0, dimensionTable.columns.findIndex((column) => namesEqualLoose(column.name, dimensionColumn))), metricCandidate.metric.column);
|
|
1127
|
+
candidates.push({
|
|
1128
|
+
metricTable: metricCandidate.table,
|
|
1129
|
+
dimensionTable,
|
|
1130
|
+
metric: metricCandidate.metric,
|
|
1131
|
+
dimensionColumn,
|
|
1132
|
+
metricJoinColumn: join.leftColumn,
|
|
1133
|
+
dimensionJoinColumn: join.rightColumn,
|
|
1134
|
+
score: genericTableScore(metricCandidate.table, question, metricCandidate.metric, metricCandidate.localDimensions) +
|
|
1135
|
+
dimensionScore +
|
|
1136
|
+
join.score +
|
|
1137
|
+
selectedRelationWeight(dimensionTable),
|
|
1138
|
+
});
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
return candidates
|
|
1142
|
+
.sort((a, b) => b.score - a.score ||
|
|
1143
|
+
(a.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
1144
|
+
a.metricTable.relation.localeCompare(b.metricTable.relation))[0];
|
|
1145
|
+
}
|
|
1146
|
+
function buildCandidateJoinPaths(schemaContext) {
|
|
1147
|
+
const tables = [...schemaContext]
|
|
1148
|
+
.filter((table) => table.columns.some((column) => isJoinKeyColumn(column.name)))
|
|
1149
|
+
.sort((a, b) => (a.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
1150
|
+
a.relation.localeCompare(b.relation))
|
|
1151
|
+
.slice(0, 16);
|
|
1152
|
+
const joins = [];
|
|
1153
|
+
const seen = new Set();
|
|
1154
|
+
for (let leftIndex = 0; leftIndex < tables.length; leftIndex += 1) {
|
|
1155
|
+
for (let rightIndex = leftIndex + 1; rightIndex < tables.length; rightIndex += 1) {
|
|
1156
|
+
const left = tables[leftIndex];
|
|
1157
|
+
const right = tables[rightIndex];
|
|
1158
|
+
const join = pickJoinColumns(left, right);
|
|
1159
|
+
if (!join)
|
|
1160
|
+
continue;
|
|
1161
|
+
const key = [
|
|
1162
|
+
normalizeRelationKey(left.relation),
|
|
1163
|
+
join.leftColumn.toLowerCase(),
|
|
1164
|
+
normalizeRelationKey(right.relation),
|
|
1165
|
+
join.rightColumn.toLowerCase(),
|
|
1166
|
+
].join('|');
|
|
1167
|
+
if (seen.has(key))
|
|
1168
|
+
continue;
|
|
1169
|
+
seen.add(key);
|
|
1170
|
+
joins.push({
|
|
1171
|
+
leftRelation: left.relation,
|
|
1172
|
+
leftColumn: join.leftColumn,
|
|
1173
|
+
rightRelation: right.relation,
|
|
1174
|
+
rightColumn: join.rightColumn,
|
|
1175
|
+
reason: joinReason(join.leftColumn, join.rightColumn, join.score),
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
}
|
|
1179
|
+
return joins.slice(0, 12);
|
|
1180
|
+
}
|
|
1181
|
+
function joinReason(leftColumn, rightColumn, score) {
|
|
1182
|
+
if (namesEqualLoose(leftColumn, rightColumn))
|
|
1183
|
+
return `shared key ${leftColumn}`;
|
|
1184
|
+
const leftSubject = joinSubjectForColumn(leftColumn);
|
|
1185
|
+
const rightSubject = joinSubjectForColumn(rightColumn);
|
|
1186
|
+
if (leftSubject && rightSubject && leftSubject === rightSubject)
|
|
1187
|
+
return `matching ${leftSubject} key`;
|
|
1188
|
+
if (score >= 55)
|
|
1189
|
+
return 'foreign-key style id match';
|
|
1190
|
+
return 'join-key style column match';
|
|
1191
|
+
}
|
|
1192
|
+
function requestedDimensionPhrases(question) {
|
|
1193
|
+
const lower = question.toLowerCase();
|
|
1194
|
+
const phrases = [
|
|
1195
|
+
...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1196
|
+
...Array.from(lower.matchAll(/\bper\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1197
|
+
...Array.from(lower.matchAll(/\bfor each\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1198
|
+
];
|
|
1199
|
+
return uniqueDrilldownStrings(phrases
|
|
1200
|
+
.flatMap((phrase) => phrase.split(/\band\b|,|\//i))
|
|
1201
|
+
.map((phrase) => phrase
|
|
1202
|
+
.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time|where|for|with|from|in|during|last|this|next)\b.*$/gi, '')
|
|
1203
|
+
.replace(/[^a-z0-9_ -]+/gi, ' ')
|
|
1204
|
+
.replace(/\s+/g, ' ')
|
|
1205
|
+
.trim())
|
|
1206
|
+
.filter((phrase) => phrase.length > 0));
|
|
1207
|
+
}
|
|
1208
|
+
function pickRequestedDimensionColumn(table, question, requestedDimensions) {
|
|
1209
|
+
for (const phrase of requestedDimensions) {
|
|
1210
|
+
const direct = findSchemaColumn(table, dimensionColumnCandidatesForPhrase(phrase));
|
|
1211
|
+
if (direct && !isJoinKeyColumn(direct) && !isNumericLikeColumn({ name: direct, type: findColumnType(table, direct) })) {
|
|
1212
|
+
return direct;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
const inferred = inferGenericDimensions(table, question)
|
|
1216
|
+
.find((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension));
|
|
1217
|
+
return inferred;
|
|
1218
|
+
}
|
|
1219
|
+
function dimensionColumnCandidatesForPhrase(phrase) {
|
|
1220
|
+
const tokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
|
|
1221
|
+
.flatMap((token) => token.split('_'))
|
|
1222
|
+
.map(normalizeToken)
|
|
1223
|
+
.filter((token) => token.length > 0);
|
|
1224
|
+
if (tokens.length === 0)
|
|
1225
|
+
return [];
|
|
1226
|
+
const joined = tokens.join('_');
|
|
1227
|
+
const tail = tokens.slice(1).join('_');
|
|
1228
|
+
return uniqueDrilldownStrings([
|
|
1229
|
+
joined,
|
|
1230
|
+
phrase.replace(/\s+/g, '_').toLowerCase(),
|
|
1231
|
+
tail,
|
|
1232
|
+
tokens.at(-1) ?? '',
|
|
1233
|
+
...tokens,
|
|
1234
|
+
].filter(Boolean));
|
|
1235
|
+
}
|
|
1236
|
+
function requestedDimensionMatchesColumn(requestedDimensions, column) {
|
|
1237
|
+
const columnTokens = exactMatchTokens(column);
|
|
1238
|
+
return requestedDimensions.some((phrase) => {
|
|
1239
|
+
const phraseTokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
|
|
1240
|
+
.flatMap((token) => token.split('_'))
|
|
1241
|
+
.map(normalizeToken)
|
|
1242
|
+
.filter((token) => token.length > 0);
|
|
1243
|
+
return phraseTokens.some((token) => columnTokens.has(token));
|
|
1244
|
+
});
|
|
1245
|
+
}
|
|
1246
|
+
function selectedRelationWeight(table) {
|
|
1247
|
+
return table.selectionRank
|
|
1248
|
+
? Math.max(0, 30 - table.selectionRank * 2) + Math.min(table.selectionScore ?? 0, 16)
|
|
1249
|
+
: 0;
|
|
1250
|
+
}
|
|
1251
|
+
function pickJoinColumns(left, right) {
|
|
1252
|
+
const candidates = [];
|
|
1253
|
+
const leftColumns = left.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
|
|
1254
|
+
const rightColumns = right.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
|
|
1255
|
+
for (const leftColumn of leftColumns) {
|
|
1256
|
+
for (const rightColumn of rightColumns) {
|
|
1257
|
+
const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
|
|
1258
|
+
if (score <= 0)
|
|
1259
|
+
continue;
|
|
1260
|
+
candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
|
|
1261
|
+
}
|
|
1262
|
+
}
|
|
1263
|
+
return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
|
|
1264
|
+
}
|
|
1265
|
+
function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
|
|
1266
|
+
const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
|
|
1267
|
+
const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
|
|
1268
|
+
const leftJoinLike = isJoinKeyColumn(leftColumn);
|
|
1269
|
+
const rightJoinLike = isJoinKeyColumn(rightColumn);
|
|
1270
|
+
if (!leftJoinLike || !rightJoinLike)
|
|
1271
|
+
return 0;
|
|
1272
|
+
if (leftKey === rightKey)
|
|
1273
|
+
return 70;
|
|
1274
|
+
const leftSubject = joinSubjectForColumn(leftColumn);
|
|
1275
|
+
const rightSubject = joinSubjectForColumn(rightColumn);
|
|
1276
|
+
if (leftSubject && rightSubject && leftSubject === rightSubject)
|
|
1277
|
+
return 62;
|
|
1278
|
+
const leftTableTokens = tableEntityTokens(leftTable);
|
|
1279
|
+
const rightTableTokens = tableEntityTokens(rightTable);
|
|
1280
|
+
if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
|
|
1281
|
+
return 58;
|
|
1282
|
+
if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
|
|
1283
|
+
return 58;
|
|
1284
|
+
if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
|
|
1285
|
+
return 42;
|
|
1286
|
+
if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
|
|
1287
|
+
return 42;
|
|
1288
|
+
return 0;
|
|
1289
|
+
}
|
|
1290
|
+
function isJoinKeyColumn(column) {
|
|
1291
|
+
const normalized = column.toLowerCase();
|
|
1292
|
+
return normalized === 'id' ||
|
|
1293
|
+
/(^|_)(id|key|uuid|sk)$/.test(normalized) ||
|
|
1294
|
+
/_(id|key|uuid|sk)$/.test(normalized);
|
|
1295
|
+
}
|
|
1296
|
+
function joinSubjectForColumn(column) {
|
|
1297
|
+
const normalized = column.toLowerCase();
|
|
1298
|
+
const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
|
|
1299
|
+
if (!subject || subject === normalized || subject === 'id' || subject === 'key')
|
|
1300
|
+
return undefined;
|
|
1301
|
+
return normalizeToken(subject.split('_').at(-1) ?? subject);
|
|
1302
|
+
}
|
|
1303
|
+
function tableEntityTokens(table) {
|
|
1304
|
+
const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
|
|
1305
|
+
for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
|
|
1306
|
+
tokens.delete(generic);
|
|
1307
|
+
return tokens;
|
|
1308
|
+
}
|
|
1309
|
+
function findColumnType(table, columnName) {
|
|
1310
|
+
return table.columns.find((column) => namesEqualLoose(column.name, columnName))?.type;
|
|
1311
|
+
}
|
|
1312
|
+
function qualifiedMetricExpression(metric, tableAlias) {
|
|
1313
|
+
const column = `${tableAlias}.${sqlIdentifier(metric.column)}`;
|
|
1314
|
+
if (metric.column === 'rows')
|
|
1315
|
+
return 'COUNT(*)';
|
|
1316
|
+
if (/^COUNT\s*\(\s*DISTINCT\b/i.test(metric.expression))
|
|
1317
|
+
return `COUNT(DISTINCT ${column})`;
|
|
1318
|
+
if (metric.preAggregated)
|
|
1319
|
+
return column;
|
|
1320
|
+
const aggregate = metric.expression.match(/^(SUM|AVG|MIN|MAX|MEDIAN)\s*\(/i)?.[1]?.toUpperCase()
|
|
1321
|
+
?? (/avg|average|mean/i.test(metric.alias) ? 'AVG' : 'SUM');
|
|
1322
|
+
return `${aggregate}(${column})`;
|
|
1323
|
+
}
|
|
1324
|
+
function inferGenericMetric(table, question) {
|
|
1325
|
+
const lower = question.toLowerCase();
|
|
1326
|
+
const wantsCount = /\b(count|how many|number of|volume)\b/i.test(lower);
|
|
1327
|
+
const candidates = table.columns
|
|
1328
|
+
.map((column) => {
|
|
1329
|
+
const name = column.name.toLowerCase();
|
|
1330
|
+
const tokens = meaningfulTokens(column.name);
|
|
1331
|
+
let score = isNumericLikeColumn(column) ? 20 : 0;
|
|
1332
|
+
for (const token of meaningfulTokens(question)) {
|
|
1333
|
+
if (tokens.has(token))
|
|
1334
|
+
score += 18;
|
|
1335
|
+
}
|
|
1336
|
+
if (/\b(revenue|sales|amount|total|spend|cost|margin|profit|value|points?|score|orders?|count|quantity|duration|minutes?|rate|average|avg)\b/i.test(name.replace(/_/g, ' '))) {
|
|
1337
|
+
score += 18;
|
|
1338
|
+
}
|
|
1339
|
+
if (/\b(id|key|code|zip|postal|phone|season|year|month|week|day|date|time)\b/i.test(name.replace(/_/g, ' '))) {
|
|
1340
|
+
score -= 14;
|
|
1341
|
+
}
|
|
1342
|
+
return { column: column.name, score };
|
|
1343
|
+
})
|
|
1344
|
+
.filter((candidate) => candidate.score > 0)
|
|
1345
|
+
.sort((a, b) => b.score - a.score);
|
|
1346
|
+
const countSubject = wantsCount ? inferCountSubject(question) : undefined;
|
|
1347
|
+
if (wantsCount) {
|
|
1348
|
+
const precomputed = candidates.find((candidate) => isPrecomputedCountMetric(candidate.column, countSubject));
|
|
1349
|
+
if (precomputed) {
|
|
1350
|
+
return {
|
|
1351
|
+
column: precomputed.column,
|
|
1352
|
+
expression: isPreAggregatedMetricColumn(precomputed.column) ? sqlIdentifier(precomputed.column) : `SUM(${sqlIdentifier(precomputed.column)})`,
|
|
1353
|
+
alias: isPreAggregatedMetricColumn(precomputed.column) ? precomputed.column : `${precomputed.column}_sum`,
|
|
1354
|
+
score: precomputed.score + 8,
|
|
1355
|
+
preAggregated: isPreAggregatedMetricColumn(precomputed.column),
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
const distinctColumn = countSubject ? findDistinctCountColumn(table, countSubject) : undefined;
|
|
1359
|
+
if (distinctColumn) {
|
|
1360
|
+
return {
|
|
1361
|
+
column: distinctColumn,
|
|
1362
|
+
expression: `COUNT(DISTINCT ${sqlIdentifier(distinctColumn)})`,
|
|
1363
|
+
alias: `${countSubject}_count`,
|
|
1364
|
+
score: 34,
|
|
1365
|
+
preAggregated: false,
|
|
1366
|
+
};
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
const selected = candidates[0];
|
|
1370
|
+
if (!selected && !wantsCount)
|
|
1371
|
+
return undefined;
|
|
1372
|
+
if (!selected || wantsCount && selected.score < 26) {
|
|
1373
|
+
return {
|
|
1374
|
+
column: 'rows',
|
|
1375
|
+
expression: 'COUNT(*)',
|
|
1376
|
+
alias: 'row_count',
|
|
1377
|
+
score: 18,
|
|
1378
|
+
preAggregated: true,
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
const aggregate = /\b(avg|average|mean)\b/i.test(lower) ? 'AVG' : 'SUM';
|
|
1382
|
+
const aliasBase = `${selected.column}_${aggregate.toLowerCase()}`;
|
|
1383
|
+
const preAggregated = isPreAggregatedMetricColumn(selected.column);
|
|
1384
|
+
return {
|
|
1385
|
+
column: selected.column,
|
|
1386
|
+
expression: preAggregated ? sqlIdentifier(selected.column) : `${aggregate}(${sqlIdentifier(selected.column)})`,
|
|
1387
|
+
alias: preAggregated ? selected.column : aliasBase,
|
|
1388
|
+
score: selected.score,
|
|
1389
|
+
preAggregated,
|
|
1390
|
+
};
|
|
1391
|
+
}
|
|
1392
|
+
function inferCountSubject(question) {
|
|
1393
|
+
const lower = question.toLowerCase();
|
|
1394
|
+
const match = lower.match(/\b(?:how many|number of|count of|count|total)\s+(?:distinct\s+|unique\s+)?([a-z][a-z0-9_-]*)/);
|
|
1395
|
+
const raw = match?.[1] ?? lower.match(/\b(customers?|accounts?|users?|members?|orders?|products?|players?|teams?|transactions?|sessions?|events?|records?|rows?)\b/)?.[1];
|
|
1396
|
+
if (!raw)
|
|
1397
|
+
return undefined;
|
|
1398
|
+
const normalized = normalizeToken(raw.replace(/[^a-z0-9_]/g, ''));
|
|
1399
|
+
if (!normalized || normalized === 'row' || normalized === 'record')
|
|
824
1400
|
return undefined;
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1401
|
+
return normalized;
|
|
1402
|
+
}
|
|
1403
|
+
function findDistinctCountColumn(table, subject) {
|
|
1404
|
+
const preferred = [
|
|
1405
|
+
`${subject}_id`,
|
|
1406
|
+
`${subject}_key`,
|
|
1407
|
+
`${subject}_uuid`,
|
|
1408
|
+
`${subject}_sk`,
|
|
1409
|
+
`${subject}_name`,
|
|
1410
|
+
subject,
|
|
1411
|
+
];
|
|
1412
|
+
const direct = findSchemaColumn(table, preferred);
|
|
1413
|
+
if (direct)
|
|
1414
|
+
return direct;
|
|
1415
|
+
const tableTokens = meaningfulTokens([table.name, table.relation].join(' '));
|
|
1416
|
+
if (!tableTokens.has(subject))
|
|
829
1417
|
return undefined;
|
|
830
|
-
|
|
1418
|
+
return findSchemaColumn(table, ['id', 'key', 'uuid']);
|
|
1419
|
+
}
|
|
1420
|
+
function isPrecomputedCountMetric(column, subject) {
|
|
1421
|
+
const normalized = column.toLowerCase();
|
|
1422
|
+
const text = normalized.replace(/_/g, ' ');
|
|
1423
|
+
if (/\b(id|key|uuid|code)\b/i.test(text))
|
|
1424
|
+
return false;
|
|
1425
|
+
if (/\b(count|cnt|number|num)\b/i.test(text)) {
|
|
1426
|
+
return !subject || text.includes(subject) || text.includes(pluralizeSimple(subject));
|
|
1427
|
+
}
|
|
1428
|
+
return Boolean(subject && (normalized === `total_${subject}` ||
|
|
1429
|
+
normalized === `total_${pluralizeSimple(subject)}`));
|
|
1430
|
+
}
|
|
1431
|
+
function pluralizeSimple(value) {
|
|
1432
|
+
if (value.endsWith('y'))
|
|
1433
|
+
return `${value.slice(0, -1)}ies`;
|
|
1434
|
+
if (value.endsWith('s'))
|
|
1435
|
+
return value;
|
|
1436
|
+
return `${value}s`;
|
|
1437
|
+
}
|
|
1438
|
+
function augmentGenericRankingDimensions(table, dimensions, metric, question) {
|
|
1439
|
+
const direction = rankingDirectionFromText(question);
|
|
1440
|
+
if (!direction || !metric.preAggregated || /\bby\s+/i.test(question))
|
|
1441
|
+
return dimensions;
|
|
1442
|
+
const timeColumn = table.columns
|
|
1443
|
+
.map((column) => column.name)
|
|
1444
|
+
.find((name) => isTimeLikeGenericColumn(name) && !dimensions.some((dimension) => namesEqualLoose(dimension, name)));
|
|
1445
|
+
return timeColumn ? [...dimensions, timeColumn] : dimensions;
|
|
1446
|
+
}
|
|
1447
|
+
function inferGenericDimensions(table, question, metricColumn) {
|
|
1448
|
+
const lower = question.toLowerCase();
|
|
1449
|
+
const requested = Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g))
|
|
1450
|
+
.map((match) => match[1] ?? '')
|
|
1451
|
+
.flatMap((value) => value.split(/\band\b|,|\//i))
|
|
1452
|
+
.map((value) => value.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time)\b/gi, '').trim())
|
|
1453
|
+
.filter(Boolean);
|
|
1454
|
+
const direct = requested
|
|
1455
|
+
.map((value) => findSchemaColumn(table, [value, value.replace(/\s+/g, '_')]))
|
|
1456
|
+
.filter((value) => Boolean(value));
|
|
1457
|
+
if (direct.length > 0)
|
|
1458
|
+
return uniqueDrilldownStrings(direct);
|
|
1459
|
+
const domainHints = [
|
|
1460
|
+
/\bcustomers?\b/.test(lower) ? ['customer_name', 'customer', 'customer_id'] : [],
|
|
1461
|
+
/\bproducts?\b/.test(lower) ? ['product_name', 'product', 'sku', 'product_id'] : [],
|
|
1462
|
+
/\bplayers?\b/.test(lower) ? ['player_name', 'player', 'player_id'] : [],
|
|
1463
|
+
/\bteams?\b/.test(lower) ? ['team_name', 'team', 'team_id'] : [],
|
|
1464
|
+
/\bregions?\b|\bmarkets?\b/.test(lower) ? ['region', 'market', 'geo'] : [],
|
|
1465
|
+
/\bsegments?\b/.test(lower) ? ['segment', 'customer_segment', 'type'] : [],
|
|
1466
|
+
/\bchannels?\b/.test(lower) ? ['channel', 'source_channel'] : [],
|
|
1467
|
+
/\b(month|monthly)\b/.test(lower) ? ['month', 'order_month', 'created_month'] : [],
|
|
1468
|
+
/\b(week|weekly)\b/.test(lower) ? ['week', 'order_week', 'created_week'] : [],
|
|
1469
|
+
/\b(year|season|yearly)\b/.test(lower) ? ['season', 'year', 'order_year'] : [],
|
|
1470
|
+
].flat();
|
|
1471
|
+
const hinted = findSchemaColumn(table, domainHints);
|
|
1472
|
+
if (hinted)
|
|
1473
|
+
return [hinted];
|
|
1474
|
+
const fallback = table.columns
|
|
1475
|
+
.map((column, index) => ({
|
|
1476
|
+
column: column.name,
|
|
1477
|
+
score: genericDimensionScore(column, question, index, metricColumn),
|
|
1478
|
+
}))
|
|
1479
|
+
.filter((candidate) => candidate.score > 0)
|
|
1480
|
+
.sort((a, b) => b.score - a.score)
|
|
1481
|
+
.map((candidate) => candidate.column);
|
|
1482
|
+
return uniqueDrilldownStrings(fallback).slice(0, 1);
|
|
1483
|
+
}
|
|
1484
|
+
function genericDimensionScore(column, question, index, metricColumn) {
|
|
1485
|
+
if (metricColumn && namesEqualLoose(column.name, metricColumn))
|
|
1486
|
+
return -100;
|
|
1487
|
+
const name = column.name.toLowerCase();
|
|
1488
|
+
const text = name.replace(/_/g, ' ');
|
|
1489
|
+
let score = Math.max(0, 10 - index * 0.05);
|
|
1490
|
+
if (/\b(name|category|type|segment|region|market|channel|status|team|player|customer|product|vendor|account|month|week|year|season|date)\b/i.test(text)) {
|
|
1491
|
+
score += 24;
|
|
1492
|
+
}
|
|
1493
|
+
if (isNumericLikeColumn(column) && !isTimeLikeGenericColumn(column.name))
|
|
1494
|
+
score -= 18;
|
|
1495
|
+
for (const token of meaningfulTokens(question)) {
|
|
1496
|
+
if (meaningfulTokens(column.name).has(token))
|
|
1497
|
+
score += 10;
|
|
1498
|
+
}
|
|
1499
|
+
return score;
|
|
1500
|
+
}
|
|
1501
|
+
function isNumericLikeColumn(column) {
|
|
1502
|
+
const type = column.type ?? '';
|
|
1503
|
+
const name = column.name.toLowerCase();
|
|
1504
|
+
return /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(type)
|
|
1505
|
+
|| /\b(total|count|amount|revenue|sales|spend|cost|margin|profit|value|points?|score|quantity|duration|minutes?|rate|avg|average)\b/i.test(name.replace(/_/g, ' '));
|
|
1506
|
+
}
|
|
1507
|
+
function isPreAggregatedMetricColumn(name) {
|
|
1508
|
+
return /^(total|count|avg|average|median|min|max)_/i.test(name)
|
|
1509
|
+
|| /_(total|count|avg|average|median|min|max)$/i.test(name);
|
|
1510
|
+
}
|
|
1511
|
+
function isTimeLikeGenericColumn(name) {
|
|
1512
|
+
return /\b(date|time|day|week|month|quarter|year|season|period)\b/i.test(name);
|
|
1513
|
+
}
|
|
1514
|
+
function buildEntityProfileProposal(input) {
|
|
1515
|
+
const entityTexts = entityMentionsForProfile(input.question, input.contextPack);
|
|
1516
|
+
if (entityTexts.length === 0 || !isEntityProfileQuestion(input.question, input.contextPack))
|
|
1517
|
+
return undefined;
|
|
1518
|
+
const candidates = input.schemaContext
|
|
1519
|
+
.map((table) => profileTableCandidate(table, input.question, input.followUp, entityTexts))
|
|
1520
|
+
.filter((candidate) => Boolean(candidate))
|
|
1521
|
+
.sort((a, b) => b.score - a.score);
|
|
1522
|
+
const selected = candidates[0];
|
|
1523
|
+
if (!selected)
|
|
1524
|
+
return undefined;
|
|
1525
|
+
const where = selected.filterValue
|
|
1526
|
+
? [`WHERE ${sqlIdentifier(selected.entityColumn)} = ${sqlStringLiteral(selected.filterValue)}`]
|
|
1527
|
+
: [];
|
|
1528
|
+
const columns = selected.selectColumns.map((column) => ` ${sqlIdentifier(column)}`).join(',\n');
|
|
1529
|
+
const text = [
|
|
1530
|
+
`Prepared a review-required profile query for ${selected.filterValue ?? entityTexts[0]} from ${selected.table.relation}.`,
|
|
1531
|
+
selected.usedSampleValue
|
|
1532
|
+
? `The entity filter uses an inspected value match on ${selected.entityColumn}.`
|
|
1533
|
+
: `The entity filter uses the likely entity column ${selected.entityColumn} from inspected schema metadata.`,
|
|
1534
|
+
'This result is uncertified until reviewed and promoted.',
|
|
1535
|
+
].join(' ');
|
|
831
1536
|
return {
|
|
832
|
-
text
|
|
1537
|
+
text,
|
|
833
1538
|
sql: [
|
|
834
1539
|
'SELECT',
|
|
835
|
-
|
|
836
|
-
`
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
`JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
|
|
840
|
-
`GROUP BY c.${sqlIdentifier(customerName)}`,
|
|
841
|
-
'ORDER BY lifetime_spend DESC, orders DESC',
|
|
842
|
-
'LIMIT 10',
|
|
1540
|
+
columns,
|
|
1541
|
+
`FROM ${sqlRelation(selected.table.relation)}`,
|
|
1542
|
+
...where,
|
|
1543
|
+
'LIMIT 50',
|
|
843
1544
|
].join('\n'),
|
|
844
1545
|
viz: 'table',
|
|
845
1546
|
};
|
|
846
1547
|
}
|
|
1548
|
+
function profileTableCandidate(table, question, followUp, entityTexts) {
|
|
1549
|
+
if (table.columns.length === 0)
|
|
1550
|
+
return undefined;
|
|
1551
|
+
const matchedFilters = matchedEntityFiltersForQuestion(table, question, followUp);
|
|
1552
|
+
const sampled = matchedFilters.find((filter) => entityTexts.some((entity) => normalizeForEntityMatch(entity) === normalizeForEntityMatch(filter.value))) ?? matchedFilters[0];
|
|
1553
|
+
const entityColumn = sampled?.column ?? pickEntityProfileColumn(table, question);
|
|
1554
|
+
if (!entityColumn)
|
|
1555
|
+
return undefined;
|
|
1556
|
+
const filterValue = sampled?.value ?? entityTexts[0];
|
|
1557
|
+
if (!filterValue)
|
|
1558
|
+
return undefined;
|
|
1559
|
+
const selectColumns = orderedProfileColumns(table, entityColumn, question);
|
|
1560
|
+
if (selectColumns.length === 0)
|
|
1561
|
+
return undefined;
|
|
1562
|
+
const questionTokens = meaningfulTokens(question);
|
|
1563
|
+
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1564
|
+
const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
|
|
1565
|
+
const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
|
|
1566
|
+
const profileSignal = selectColumns.filter((column) => isProfileMeasureColumn(column, table)).length;
|
|
1567
|
+
const score = (sampled ? 80 : 42) +
|
|
1568
|
+
overlap * 6 +
|
|
1569
|
+
profileSignal * 3 +
|
|
1570
|
+
(/\b(player|customer|account|user|member|person|entity)\b/i.test(table.name) ? 8 : 0);
|
|
1571
|
+
return {
|
|
1572
|
+
table,
|
|
1573
|
+
entityColumn,
|
|
1574
|
+
filterValue,
|
|
1575
|
+
selectColumns,
|
|
1576
|
+
usedSampleValue: Boolean(sampled),
|
|
1577
|
+
score,
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1580
|
+
function isEntityProfileQuestion(question, contextPack) {
|
|
1581
|
+
const mode = contextPack?.questionPlan?.mode;
|
|
1582
|
+
if (mode === 'entity_profile')
|
|
1583
|
+
return true;
|
|
1584
|
+
return /\b(profile|overview|360|complete\s+(?:stats|statistics|view)|full\s+(?:stats|statistics|view)|all\s+(?:stats|statistics|metrics)|research|reserach)\b/i.test(question);
|
|
1585
|
+
}
|
|
1586
|
+
function entityMentionsForProfile(question, contextPack) {
|
|
1587
|
+
const values = [
|
|
1588
|
+
...(contextPack?.questionPlan?.entities ?? []).map((entity) => entity.text),
|
|
1589
|
+
...Array.from(question.matchAll(/\b(?:for|on|profile\s+for|research\s+on|reserach\s+on)\s+([A-Z][A-Za-z0-9&.'-]+(?:\s+[A-Z][A-Za-z0-9&.'-]+){0,5})/g))
|
|
1590
|
+
.map((match) => match[1] ?? ''),
|
|
1591
|
+
];
|
|
1592
|
+
if (isEntityProfileQuestion(question, contextPack)) {
|
|
1593
|
+
values.push(...Array.from(question.matchAll(/\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,5})\b/g)).map((match) => match[1] ?? ''));
|
|
1594
|
+
}
|
|
1595
|
+
return uniqueDrilldownStrings(values
|
|
1596
|
+
.map((value) => value.replace(/\b(profile|stats|statistics|details|summary)\b.*$/i, '').trim())
|
|
1597
|
+
.filter((value) => value.length > 1 && !/^(Can You|Could You|Tell Me|Show Me)$/i.test(value)))
|
|
1598
|
+
.slice(0, 6);
|
|
1599
|
+
}
|
|
1600
|
+
function pickEntityProfileColumn(table, question) {
|
|
1601
|
+
const lower = question.toLowerCase();
|
|
1602
|
+
const domainHints = [
|
|
1603
|
+
/\bplayers?\b/.test(lower) ? 'player' : '',
|
|
1604
|
+
/\bcustomers?\b/.test(lower) ? 'customer' : '',
|
|
1605
|
+
/\baccounts?\b/.test(lower) ? 'account' : '',
|
|
1606
|
+
/\busers?\b/.test(lower) ? 'user' : '',
|
|
1607
|
+
/\bmembers?\b/.test(lower) ? 'member' : '',
|
|
1608
|
+
/\bteams?\b/.test(lower) ? 'team' : '',
|
|
1609
|
+
/\bproducts?\b/.test(lower) ? 'product' : '',
|
|
1610
|
+
].filter(Boolean);
|
|
1611
|
+
const preferred = [
|
|
1612
|
+
...domainHints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
|
|
1613
|
+
'name',
|
|
1614
|
+
'full_name',
|
|
1615
|
+
'display_name',
|
|
1616
|
+
'title',
|
|
1617
|
+
];
|
|
1618
|
+
const direct = findSchemaColumn(table, preferred);
|
|
1619
|
+
if (direct)
|
|
1620
|
+
return direct;
|
|
1621
|
+
const scored = table.columns
|
|
1622
|
+
.map((column) => {
|
|
1623
|
+
const name = column.name.toLowerCase();
|
|
1624
|
+
let score = 0;
|
|
1625
|
+
if (name.endsWith('_name') || name === 'name')
|
|
1626
|
+
score += 30;
|
|
1627
|
+
if (/\b(name|title|email)\b/i.test(name.replace(/_/g, ' ')))
|
|
1628
|
+
score += 20;
|
|
1629
|
+
if (domainHints.some((hint) => name.includes(hint)))
|
|
1630
|
+
score += 16;
|
|
1631
|
+
if (/\b(id|key|code|date|time|amount|total|count|score|points|revenue)\b/i.test(name.replace(/_/g, ' ')))
|
|
1632
|
+
score -= 12;
|
|
1633
|
+
return { column: column.name, score };
|
|
1634
|
+
})
|
|
1635
|
+
.filter((item) => item.score > 0)
|
|
1636
|
+
.sort((a, b) => b.score - a.score);
|
|
1637
|
+
return scored[0]?.column;
|
|
1638
|
+
}
|
|
1639
|
+
function orderedProfileColumns(table, entityColumn, question) {
|
|
1640
|
+
const scored = table.columns
|
|
1641
|
+
.map((column, index) => ({
|
|
1642
|
+
column: column.name,
|
|
1643
|
+
score: profileColumnScore(column, question, index),
|
|
1644
|
+
index,
|
|
1645
|
+
}))
|
|
1646
|
+
.sort((a, b) => b.score - a.score || a.index - b.index);
|
|
1647
|
+
const selected = uniqueDrilldownStrings([
|
|
1648
|
+
entityColumn,
|
|
1649
|
+
...scored.map((item) => item.column),
|
|
1650
|
+
]).slice(0, 24);
|
|
1651
|
+
return selected.length > 0 ? selected : [entityColumn];
|
|
1652
|
+
}
|
|
1653
|
+
function profileColumnScore(column, question, index) {
|
|
1654
|
+
const name = column.name.toLowerCase();
|
|
1655
|
+
const text = name.replace(/_/g, ' ');
|
|
1656
|
+
const lowerQuestion = question.toLowerCase();
|
|
1657
|
+
let score = Math.max(0, 12 - index * 0.1);
|
|
1658
|
+
if (isProfileMeasureColumn(column.name, { relation: '', name: '', columns: [column] }))
|
|
1659
|
+
score += 35;
|
|
1660
|
+
if (/\b(name|title|team|season|year|date|type|category|segment|status)\b/i.test(text))
|
|
1661
|
+
score += 18;
|
|
1662
|
+
if (/\b(id|key|code)\b/i.test(text))
|
|
1663
|
+
score += 2;
|
|
1664
|
+
for (const token of meaningfulTokens(lowerQuestion)) {
|
|
1665
|
+
if (meaningfulTokens(column.name).has(token))
|
|
1666
|
+
score += 12;
|
|
1667
|
+
}
|
|
1668
|
+
if (column.description && meaningfulTokens(column.description).size > 0)
|
|
1669
|
+
score += 4;
|
|
1670
|
+
return score;
|
|
1671
|
+
}
|
|
1672
|
+
function isProfileMeasureColumn(columnName, table) {
|
|
1673
|
+
const column = table.columns.find((item) => namesEqualLoose(item.name, columnName));
|
|
1674
|
+
const name = columnName.toLowerCase();
|
|
1675
|
+
const text = name.replace(/_/g, ' ');
|
|
1676
|
+
return /\b(total|count|avg|average|sum|rate|score|points|pts|rebounds?|assists?|steals?|blocks?|turnovers?|minutes?|games?|wins?|losses?|revenue|amount|spend|orders?|quantity|margin|cost|value|stat)\b/i.test(text)
|
|
1677
|
+
|| /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(column?.type ?? '');
|
|
1678
|
+
}
|
|
847
1679
|
function buildMatchedEntityDrilldownProposal(input) {
|
|
848
1680
|
if (input.intent !== 'entity_drilldown' && input.followUp?.kind !== 'drilldown')
|
|
849
1681
|
return undefined;
|
|
@@ -896,15 +1728,21 @@ function buildMatchedEntityDrilldownProposal(input) {
|
|
|
896
1728
|
}
|
|
897
1729
|
function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
898
1730
|
const byRelation = new Map();
|
|
1731
|
+
const relationSelections = selectedRelationLookup(contextPack);
|
|
899
1732
|
for (const table of schemaContext) {
|
|
1733
|
+
const selection = relationSelectionFor(table.relation, relationSelections);
|
|
900
1734
|
byRelation.set(normalizeRelationKey(table.relation), {
|
|
901
1735
|
...table,
|
|
902
1736
|
columns: table.columns.map((column) => ({ ...column, sampleValues: column.sampleValues?.slice() })),
|
|
1737
|
+
selectionRank: selection?.rank ?? table.selectionRank,
|
|
1738
|
+
selectionScore: selection?.score ?? table.selectionScore,
|
|
1739
|
+
selectionReason: selection?.reason ?? table.selectionReason,
|
|
903
1740
|
});
|
|
904
1741
|
}
|
|
905
1742
|
for (const relation of contextPack?.allowedSqlContext?.relations ?? []) {
|
|
906
1743
|
const key = normalizeRelationKey(relation.relation);
|
|
907
1744
|
const existing = byRelation.get(key);
|
|
1745
|
+
const selection = relationSelectionFor(relation.relation, relationSelections);
|
|
908
1746
|
if (!existing) {
|
|
909
1747
|
byRelation.set(key, {
|
|
910
1748
|
relation: relation.relation,
|
|
@@ -916,9 +1754,15 @@ function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
|
916
1754
|
sampleValues: column.sampleValues?.slice(),
|
|
917
1755
|
})),
|
|
918
1756
|
source: relation.source,
|
|
1757
|
+
selectionRank: selection?.rank,
|
|
1758
|
+
selectionScore: selection?.score,
|
|
1759
|
+
selectionReason: selection?.reason,
|
|
919
1760
|
});
|
|
920
1761
|
continue;
|
|
921
1762
|
}
|
|
1763
|
+
existing.selectionRank = selection?.rank ?? existing.selectionRank;
|
|
1764
|
+
existing.selectionScore = selection?.score ?? existing.selectionScore;
|
|
1765
|
+
existing.selectionReason = selection?.reason ?? existing.selectionReason;
|
|
922
1766
|
const columns = new Map(existing.columns.map((column) => [column.name.toLowerCase(), column]));
|
|
923
1767
|
for (const column of relation.columns) {
|
|
924
1768
|
const existingColumn = columns.get(column.name.toLowerCase());
|
|
@@ -941,6 +1785,23 @@ function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
|
941
1785
|
}
|
|
942
1786
|
return Array.from(byRelation.values());
|
|
943
1787
|
}
|
|
1788
|
+
function selectedRelationLookup(contextPack) {
|
|
1789
|
+
const lookup = new Map();
|
|
1790
|
+
for (const relation of contextPack?.retrievalDiagnostics?.selectedRelations ?? []) {
|
|
1791
|
+
for (const key of relationLookupKeys(relation.relation)) {
|
|
1792
|
+
lookup.set(key, relation);
|
|
1793
|
+
}
|
|
1794
|
+
}
|
|
1795
|
+
return lookup;
|
|
1796
|
+
}
|
|
1797
|
+
function relationSelectionFor(relation, lookup) {
|
|
1798
|
+
for (const key of relationLookupKeys(relation)) {
|
|
1799
|
+
const selection = lookup.get(key);
|
|
1800
|
+
if (selection)
|
|
1801
|
+
return selection;
|
|
1802
|
+
}
|
|
1803
|
+
return undefined;
|
|
1804
|
+
}
|
|
944
1805
|
function pickDrilldownTable(schemaContext, question, followUp) {
|
|
945
1806
|
const scored = schemaContext
|
|
946
1807
|
.map((table) => ({
|
|
@@ -1113,6 +1974,18 @@ function namesEqualLoose(a, b) {
|
|
|
1113
1974
|
function normalizeRelationKey(relation) {
|
|
1114
1975
|
return relation.replace(/["`]/g, '').replace(/\s*\.\s*/g, '.').toLowerCase().trim();
|
|
1115
1976
|
}
|
|
1977
|
+
function relationLookupKeys(relation) {
|
|
1978
|
+
const normalized = normalizeRelationKey(relation);
|
|
1979
|
+
const parts = normalized.split('.').filter(Boolean);
|
|
1980
|
+
const keys = new Set();
|
|
1981
|
+
if (normalized)
|
|
1982
|
+
keys.add(normalized);
|
|
1983
|
+
if (parts.length >= 2)
|
|
1984
|
+
keys.add(parts.slice(-2).join('.'));
|
|
1985
|
+
if (parts.length >= 1)
|
|
1986
|
+
keys.add(parts[parts.length - 1]);
|
|
1987
|
+
return Array.from(keys);
|
|
1988
|
+
}
|
|
1116
1989
|
function normalizeForEntityMatch(value) {
|
|
1117
1990
|
return value.toLowerCase().replace(/[^a-z0-9.%+-]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
1118
1991
|
}
|
|
@@ -1142,6 +2015,9 @@ function buildContextPackAwareProposal(input) {
|
|
|
1142
2015
|
return undefined;
|
|
1143
2016
|
if (!input.contextPack)
|
|
1144
2017
|
return undefined;
|
|
2018
|
+
const profileProposal = buildContextPackProfileProposal(input.question, input.contextPack);
|
|
2019
|
+
if (profileProposal)
|
|
2020
|
+
return profileProposal;
|
|
1145
2021
|
const lower = input.question.toLowerCase();
|
|
1146
2022
|
if (!/\b(least|lowest|fewest|bottom|min(?:imum)?)\b/.test(lower))
|
|
1147
2023
|
return undefined;
|
|
@@ -1162,6 +2038,86 @@ function buildContextPackAwareProposal(input) {
|
|
|
1162
2038
|
}
|
|
1163
2039
|
return undefined;
|
|
1164
2040
|
}
|
|
2041
|
+
function buildContextPackProfileProposal(question, contextPack) {
|
|
2042
|
+
const entityTexts = entityMentionsForProfile(question, contextPack);
|
|
2043
|
+
if (entityTexts.length === 0 || !isEntityProfileQuestion(question, contextPack))
|
|
2044
|
+
return undefined;
|
|
2045
|
+
const sources = [...(contextPack.allowedSqlContext?.sourceBlockSql ?? [])]
|
|
2046
|
+
.sort((a, b) => {
|
|
2047
|
+
const preferred = contextPack.routeDecision.certifiedApplicability?.objectKey;
|
|
2048
|
+
if (preferred && a.objectKey === preferred)
|
|
2049
|
+
return -1;
|
|
2050
|
+
if (preferred && b.objectKey === preferred)
|
|
2051
|
+
return 1;
|
|
2052
|
+
return 0;
|
|
2053
|
+
});
|
|
2054
|
+
for (const source of sources) {
|
|
2055
|
+
const shape = extractSimpleSelectShape(source.sql);
|
|
2056
|
+
if (!shape)
|
|
2057
|
+
continue;
|
|
2058
|
+
const entityColumn = pickEntityColumnFromSelectExpressions(shape.selectExpressions, question);
|
|
2059
|
+
if (!entityColumn)
|
|
2060
|
+
continue;
|
|
2061
|
+
const selectExpressions = uniqueSqlSelectExpressions(shape.selectExpressions).slice(0, 24);
|
|
2062
|
+
if (selectExpressions.length === 0)
|
|
2063
|
+
continue;
|
|
2064
|
+
const orderColumn = selectExpressions
|
|
2065
|
+
.map(selectExpressionOutputName)
|
|
2066
|
+
.find((column) => column && /\b(season|year|date|month|week|game_date|created_at)\b/i.test(column));
|
|
2067
|
+
const sql = [
|
|
2068
|
+
'SELECT',
|
|
2069
|
+
selectExpressions.map((expression) => ` ${expression}`).join(',\n'),
|
|
2070
|
+
`FROM ${shape.relation}`,
|
|
2071
|
+
`WHERE ${sqlIdentifier(entityColumn)} = ${sqlStringLiteral(entityTexts[0])}`,
|
|
2072
|
+
orderColumn ? `ORDER BY ${sqlIdentifier(orderColumn)} DESC` : '',
|
|
2073
|
+
'LIMIT 50',
|
|
2074
|
+
].filter(Boolean).join('\n');
|
|
2075
|
+
return {
|
|
2076
|
+
text: `Prepared a review-required profile query for ${entityTexts[0]} by using certified block "${source.name}" as SQL-shape context. This result is uncertified until reviewed and promoted.`,
|
|
2077
|
+
sql,
|
|
2078
|
+
viz: 'table',
|
|
2079
|
+
};
|
|
2080
|
+
}
|
|
2081
|
+
return undefined;
|
|
2082
|
+
}
|
|
2083
|
+
function pickEntityColumnFromSelectExpressions(expressions, question) {
|
|
2084
|
+
const outputNames = expressions
|
|
2085
|
+
.map(selectExpressionOutputName)
|
|
2086
|
+
.filter((value) => Boolean(value));
|
|
2087
|
+
const lower = question.toLowerCase();
|
|
2088
|
+
const hints = [
|
|
2089
|
+
/\bplayers?\b/.test(lower) ? 'player' : '',
|
|
2090
|
+
/\bcustomers?\b/.test(lower) ? 'customer' : '',
|
|
2091
|
+
/\baccounts?\b/.test(lower) ? 'account' : '',
|
|
2092
|
+
/\busers?\b/.test(lower) ? 'user' : '',
|
|
2093
|
+
/\bteams?\b/.test(lower) ? 'team' : '',
|
|
2094
|
+
/\bproducts?\b/.test(lower) ? 'product' : '',
|
|
2095
|
+
].filter(Boolean);
|
|
2096
|
+
const preferred = [
|
|
2097
|
+
...hints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
|
|
2098
|
+
'name',
|
|
2099
|
+
'full_name',
|
|
2100
|
+
'display_name',
|
|
2101
|
+
];
|
|
2102
|
+
for (const wanted of preferred) {
|
|
2103
|
+
const match = outputNames.find((name) => namesEqualLoose(name, wanted));
|
|
2104
|
+
if (match)
|
|
2105
|
+
return match;
|
|
2106
|
+
}
|
|
2107
|
+
return outputNames.find((name) => /(^|_)(name|title|email)$/.test(name.toLowerCase()));
|
|
2108
|
+
}
|
|
2109
|
+
function uniqueSqlSelectExpressions(expressions) {
|
|
2110
|
+
const seen = new Set();
|
|
2111
|
+
const unique = [];
|
|
2112
|
+
for (const expression of expressions) {
|
|
2113
|
+
const normalized = expression.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
2114
|
+
if (!normalized || seen.has(normalized))
|
|
2115
|
+
continue;
|
|
2116
|
+
seen.add(normalized);
|
|
2117
|
+
unique.push(expression.replace(/\s+/g, ' ').trim());
|
|
2118
|
+
}
|
|
2119
|
+
return unique;
|
|
2120
|
+
}
|
|
1165
2121
|
function invertRankingSql(sql) {
|
|
1166
2122
|
const withoutTrailingSemicolon = sql.replace(/;\s*$/, '').trim();
|
|
1167
2123
|
const inverted = withoutTrailingSemicolon.replace(/\border\s+by\s+([\s\S]*?)(\blimit\b|$)/i, (match, orderExpr, limitKeyword) => {
|
|
@@ -1579,7 +2535,10 @@ function isAdHocAnalysisQuestion(question) {
|
|
|
1579
2535
|
const lower = question.toLowerCase();
|
|
1580
2536
|
if (isBusinessDefinitionQuestion(question))
|
|
1581
2537
|
return false;
|
|
1582
|
-
|
|
2538
|
+
const asksCountBreakdown = /\b(how many|number of|count of|count)\b/i.test(lower)
|
|
2539
|
+
&& /\b(by\s+[a-z][\w\s-]{1,40}|per|for each|group(?:ed)? by|split|segment)\b/i.test(lower);
|
|
2540
|
+
return asksCountBreakdown
|
|
2541
|
+
|| /\b(break\s*down|breakdown|drill\s*(?:down|into)|slice|segment|split|compare|versus|vs\.?|trend|over time|top|bottom|best|worst|highest|lowest|least|fewest|minimum|min|smallest|rank|ranking|performed better|better performing|why|what drove|driver|drivers|top movers?|changed?|change|dropped?|drop|decreased?|decrease|declined?|decline|increased?|increase|anomal(?:y|ies)|exceptions?|root cause|contribut(?:e|ed|ion)|variance|delta|by\s+[a-z][\w\s-]{1,40})\b/i.test(lower)
|
|
1583
2542
|
|| /\b(show|list|find|give)\b.+\b(account|accounts|customer|customers|product|products|order|orders|region|location|month|week|day|user|users)\b/i.test(lower);
|
|
1584
2543
|
}
|
|
1585
2544
|
function isFilteredEntityQuestion(question) {
|
|
@@ -1616,7 +2575,9 @@ function buildAnalysisPlan(input) {
|
|
|
1616
2575
|
const tokens = meaningfulTokens(input.question);
|
|
1617
2576
|
const dimensions = inferDimensions(input.question, input.selectedNodes, input.schemaContext);
|
|
1618
2577
|
const measures = inferMeasures(input.question, input.selectedNodes, input.schemaContext);
|
|
1619
|
-
const
|
|
2578
|
+
const candidateJoins = buildCandidateJoinPaths(input.schemaContext);
|
|
2579
|
+
const candidateTables = [...input.schemaContext].sort((a, b) => (a.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
2580
|
+
a.relation.localeCompare(b.relation)).slice(0, 8).map((table) => ({
|
|
1620
2581
|
relation: table.relation,
|
|
1621
2582
|
columns: table.columns.slice(0, 16).map((col) => col.name),
|
|
1622
2583
|
reason: tableReason(table, tokens),
|
|
@@ -1635,6 +2596,7 @@ function buildAnalysisPlan(input) {
|
|
|
1635
2596
|
measures,
|
|
1636
2597
|
dimensions,
|
|
1637
2598
|
candidateTables,
|
|
2599
|
+
candidateJoins,
|
|
1638
2600
|
trustedContext,
|
|
1639
2601
|
assumptions: input.assumptions ?? [],
|
|
1640
2602
|
sql: input.sql,
|
|
@@ -1694,6 +2656,11 @@ function inferMeasures(question, selectedNodes, schemaContext) {
|
|
|
1694
2656
|
return Array.from(measures).slice(0, 6);
|
|
1695
2657
|
}
|
|
1696
2658
|
function tableReason(table, questionTokens) {
|
|
2659
|
+
if (table.selectionReason) {
|
|
2660
|
+
return table.selectionRank
|
|
2661
|
+
? `metadata rank ${table.selectionRank}: ${table.selectionReason}`
|
|
2662
|
+
: table.selectionReason;
|
|
2663
|
+
}
|
|
1697
2664
|
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1698
2665
|
const columnTokens = meaningfulTokens(table.columns.map((col) => col.name).join(' '));
|
|
1699
2666
|
const matches = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token));
|