@duckcodeailabs/dql-agent 1.6.7 → 1.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/answer-loop.d.ts +12 -0
- package/dist/answer-loop.d.ts.map +1 -1
- package/dist/answer-loop.js +998 -52
- package/dist/answer-loop.js.map +1 -1
- package/dist/app-builder.d.ts +27 -0
- package/dist/app-builder.d.ts.map +1 -1
- package/dist/app-builder.js +191 -14
- package/dist/app-builder.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/metadata/analysis-planner.d.ts +52 -0
- package/dist/metadata/analysis-planner.d.ts.map +1 -0
- package/dist/metadata/analysis-planner.js +632 -0
- package/dist/metadata/analysis-planner.js.map +1 -0
- package/dist/metadata/catalog.d.ts +32 -0
- package/dist/metadata/catalog.d.ts.map +1 -1
- package/dist/metadata/catalog.js +705 -22
- package/dist/metadata/catalog.js.map +1 -1
- package/dist/metadata/sql-context-validation.d.ts.map +1 -1
- package/dist/metadata/sql-context-validation.js +41 -13
- package/dist/metadata/sql-context-validation.js.map +1 -1
- package/dist/metadata/sql-shape.d.ts +13 -0
- package/dist/metadata/sql-shape.d.ts.map +1 -0
- package/dist/metadata/sql-shape.js +92 -0
- package/dist/metadata/sql-shape.js.map +1 -0
- package/package.json +4 -4
package/dist/answer-loop.js
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
*/
|
|
17
17
|
import { buildSkillBlockHints, buildSkillsPrompt } from './skills/loader.js';
|
|
18
18
|
import { validateSqlAgainstLocalContext } from './metadata/sql-context-validation.js';
|
|
19
|
+
import { compactSqlSnippet, extractSimpleSelectShape, selectExpressionOutputName, } from './metadata/sql-shape.js';
|
|
19
20
|
const CERTIFIED_HIT_THRESHOLD = 0.18;
|
|
20
21
|
const HARD_NEGATIVE_RATIO = 0.5;
|
|
21
22
|
const EXECUTABLE_ARTIFACT_KINDS = ['block', 'dashboard', 'app', 'notebook'];
|
|
@@ -41,6 +42,7 @@ export async function answer(input) {
|
|
|
41
42
|
const semanticHits = kg.search({ query: question, domain, kinds: SEMANTIC_KINDS, limit: 12 });
|
|
42
43
|
const manifestHits = kg.search({ query: question, domain, kinds: MANIFEST_KINDS, limit: 12 });
|
|
43
44
|
const considered = mergeHits(artifactHits, semanticHits, manifestHits, kg.search({ query: question, domain, limit: 10 })).slice(0, 30);
|
|
45
|
+
const schemaContext = schemaContextWithAllowedSqlContext(input.schemaContext ?? [], input.contextPack);
|
|
44
46
|
const catalogRoute = input.contextPack?.routeDecision;
|
|
45
47
|
const fallbackIntent = classifyAgentIntent({
|
|
46
48
|
question,
|
|
@@ -48,7 +50,7 @@ export async function answer(input) {
|
|
|
48
50
|
artifactHits,
|
|
49
51
|
semanticHits,
|
|
50
52
|
manifestHits,
|
|
51
|
-
schemaContext
|
|
53
|
+
schemaContext,
|
|
52
54
|
});
|
|
53
55
|
const intent = catalogRoute ? agentIntentFromCatalogRoute(catalogRoute) : fallbackIntent;
|
|
54
56
|
// Stage 1: certified artifact match. Blocks can be executed; dashboards,
|
|
@@ -104,7 +106,7 @@ export async function answer(input) {
|
|
|
104
106
|
intent,
|
|
105
107
|
routeReason: catalogRoute?.reason ?? 'The question matched a certified DQL artifact closely enough to answer without generating new SQL.',
|
|
106
108
|
selectedNodes: [artifactHit.node],
|
|
107
|
-
schemaContext
|
|
109
|
+
schemaContext,
|
|
108
110
|
sql: result?.sql,
|
|
109
111
|
suggestedViz: result?.chartConfig ? chartNameFromConfig(result.chartConfig) : undefined,
|
|
110
112
|
});
|
|
@@ -147,13 +149,13 @@ export async function answer(input) {
|
|
|
147
149
|
};
|
|
148
150
|
}
|
|
149
151
|
if (intent === 'clarify' || catalogRoute?.route === 'clarify') {
|
|
150
|
-
const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered,
|
|
152
|
+
const text = composeCatalogClarificationText(question, catalogRoute) ?? composeClarificationText(question, considered, schemaContext);
|
|
151
153
|
const analysisPlan = buildAnalysisPlan({
|
|
152
154
|
question,
|
|
153
155
|
intent,
|
|
154
156
|
routeReason: catalogRoute?.reason ?? 'No certified artifact, semantic object, dbt/source table, or runtime schema match was strong enough to safely generate SQL.',
|
|
155
157
|
selectedNodes: considered.slice(0, 4).map((hit) => hit.node),
|
|
156
|
-
schemaContext
|
|
158
|
+
schemaContext,
|
|
157
159
|
assumptions: catalogRoute?.missingContext.length
|
|
158
160
|
? catalogRoute.missingContext.map((item) => item.message)
|
|
159
161
|
: ['Need a clearer business object, measure, or grain before querying.'],
|
|
@@ -195,7 +197,7 @@ export async function answer(input) {
|
|
|
195
197
|
const reviewRequiredArtifactHits = artifactHits
|
|
196
198
|
.filter((hit) => hit.score >= CERTIFIED_HIT_THRESHOLD && !isCertifiedHit(hit, kg))
|
|
197
199
|
.slice(0, 4);
|
|
198
|
-
const trustedArtifactContext = rankGeneratedContextHits(executableArtifactHits.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId)),
|
|
200
|
+
const trustedArtifactContext = rankGeneratedContextHits(executableArtifactHits.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId)), schemaContext, question)
|
|
199
201
|
.filter((hit) => !excludedArtifactIds?.has(hit.node.nodeId))
|
|
200
202
|
.slice(0, 5);
|
|
201
203
|
const contextHits = activeTier === 'semantic_layer'
|
|
@@ -213,13 +215,13 @@ export async function answer(input) {
|
|
|
213
215
|
messages.push({ role: 'system', content: skillsPrompt });
|
|
214
216
|
messages.push({
|
|
215
217
|
role: 'system',
|
|
216
|
-
content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp,
|
|
218
|
+
content: renderContextPrompt(contextBlocks, contextBusiness, contextOther, activeTier, input.memoryContext ?? [], input.extraContext, input.followUp, schemaContext, intent, input.contextPack),
|
|
217
219
|
});
|
|
218
220
|
messages.push({ role: 'user', content: question });
|
|
219
221
|
const localProposal = buildSchemaAwareProposal({
|
|
220
222
|
question,
|
|
221
223
|
intent,
|
|
222
|
-
schemaContext
|
|
224
|
+
schemaContext,
|
|
223
225
|
followUp: input.followUp,
|
|
224
226
|
contextPack: input.contextPack,
|
|
225
227
|
}) ?? buildContextPackAwareProposal({
|
|
@@ -304,7 +306,7 @@ export async function answer(input) {
|
|
|
304
306
|
intent,
|
|
305
307
|
routeReason: catalogRoute?.reason ?? 'Generated SQL failed metadata context validation before preview execution or draft capture.',
|
|
306
308
|
selectedNodes: contextNodes,
|
|
307
|
-
schemaContext
|
|
309
|
+
schemaContext,
|
|
308
310
|
sql: parsed.sql,
|
|
309
311
|
suggestedViz: parsed.viz ?? 'table',
|
|
310
312
|
assumptions: [
|
|
@@ -363,7 +365,7 @@ export async function answer(input) {
|
|
|
363
365
|
sourceTier: 'memory',
|
|
364
366
|
provenance: m.source,
|
|
365
367
|
})),
|
|
366
|
-
...schemaCitations(
|
|
368
|
+
...schemaCitations(schemaContext, Math.max(0, 4 - contextNodes.length)),
|
|
367
369
|
];
|
|
368
370
|
let result;
|
|
369
371
|
let executionError;
|
|
@@ -375,7 +377,7 @@ export async function answer(input) {
|
|
|
375
377
|
catch (err) {
|
|
376
378
|
executionError = err instanceof Error ? err.message : String(err);
|
|
377
379
|
if (isRetryableGeneratedSqlError(executionError)) {
|
|
378
|
-
const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError,
|
|
380
|
+
const localRepairSql = repairGeneratedSqlLocally(parsed.sql, executionError, schemaContext);
|
|
379
381
|
if (localRepairSql) {
|
|
380
382
|
repairAttempts = 1;
|
|
381
383
|
parsed.sql = localRepairSql;
|
|
@@ -394,7 +396,7 @@ export async function answer(input) {
|
|
|
394
396
|
question,
|
|
395
397
|
parsed,
|
|
396
398
|
executionError,
|
|
397
|
-
schemaContext
|
|
399
|
+
schemaContext,
|
|
398
400
|
signal: input.signal,
|
|
399
401
|
});
|
|
400
402
|
const repaired = parseProposal(repairedRaw);
|
|
@@ -422,18 +424,20 @@ export async function answer(input) {
|
|
|
422
424
|
? 'The user asked for a drill-through or follow-up, so DQL generated review-required SQL from the prior context and current metadata.'
|
|
423
425
|
: 'The question asks for a custom analysis, ranking, breakdown, comparison, or grain that should not be answered by a loose certified block match.'),
|
|
424
426
|
selectedNodes: contextNodes,
|
|
425
|
-
schemaContext
|
|
427
|
+
schemaContext,
|
|
426
428
|
sql: parsed.sql,
|
|
427
429
|
suggestedViz: parsed.viz ?? 'table',
|
|
428
430
|
assumptions: [
|
|
429
431
|
'Generated SQL is an uncertified preview until an analyst reviews and promotes it.',
|
|
430
432
|
...(localProposal ? ['A local metadata planner selected a review-required SQL grain before provider generation.'] : []),
|
|
433
|
+
...contextValidation.warnings,
|
|
431
434
|
...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
|
|
432
435
|
],
|
|
433
436
|
repairAttempts,
|
|
434
437
|
});
|
|
435
438
|
const validationWarnings = [
|
|
436
439
|
...(input.contextPack?.warnings ?? []),
|
|
440
|
+
...contextValidation.warnings,
|
|
437
441
|
...(executionError ? ['The preview execution error must be reviewed before reuse.'] : []),
|
|
438
442
|
];
|
|
439
443
|
let draftBlock;
|
|
@@ -494,7 +498,7 @@ export async function answer(input) {
|
|
|
494
498
|
activeTier,
|
|
495
499
|
intent,
|
|
496
500
|
contextNodes,
|
|
497
|
-
schemaContext
|
|
501
|
+
schemaContext,
|
|
498
502
|
followUp: input.followUp,
|
|
499
503
|
businessHits,
|
|
500
504
|
semanticHits,
|
|
@@ -527,9 +531,10 @@ Rules:
|
|
|
527
531
|
4. Provide a one-paragraph natural-language summary BEFORE the SQL block.
|
|
528
532
|
5. Suggest a visualization type from this list, on a line starting with "Viz:":
|
|
529
533
|
line, bar, area, pie, single_value, table, pivot, kpi.
|
|
530
|
-
6. NEVER fabricate column names that are not present in the supplied schema
|
|
531
|
-
|
|
532
|
-
|
|
534
|
+
6. NEVER fabricate column names that are not present in the supplied schema,
|
|
535
|
+
dbt metadata, or certified source SQL shape context. If a requested filter
|
|
536
|
+
value is supplied as a matched value, prefer the table and column that
|
|
537
|
+
matched that value.
|
|
533
538
|
7. Return one read-only SELECT or WITH query for the local warehouse/runtime.
|
|
534
539
|
Do NOT use dbt/Jinja macros such as {{ ref(...) }} or {{ source(...) }} in
|
|
535
540
|
proposed SQL. Do not emit INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, COPY,
|
|
@@ -593,6 +598,20 @@ function renderContextPrompt(blocks, businessContext, others, activeTier, memory
|
|
|
593
598
|
return `${intentSection}\n\n${blockSection}${businessSection}${otherSection}${schemaSection}${contextPackSection}${memorySection}${extraSection}${followUpSection}`;
|
|
594
599
|
}
|
|
595
600
|
function renderContextPackForPrompt(contextPack) {
|
|
601
|
+
const questionPlan = contextPack.questionPlan
|
|
602
|
+
? [
|
|
603
|
+
`Question plan: ${contextPack.questionPlan.mode} -> ${contextPack.questionPlan.routeIntent}`,
|
|
604
|
+
contextPack.questionPlan.entities.length
|
|
605
|
+
? `Entities: ${contextPack.questionPlan.entities.map((entity) => entity.text).join(', ')}`
|
|
606
|
+
: '',
|
|
607
|
+
contextPack.questionPlan.metricTerms.length ? `Metric terms: ${contextPack.questionPlan.metricTerms.join(', ')}` : '',
|
|
608
|
+
contextPack.questionPlan.dimensionTerms.length ? `Dimension terms: ${contextPack.questionPlan.dimensionTerms.join(', ')}` : '',
|
|
609
|
+
`Answer shape: ${contextPack.questionPlan.outputShape}`,
|
|
610
|
+
].filter(Boolean).join('\n')
|
|
611
|
+
: '';
|
|
612
|
+
const certifiedApplicability = contextPack.routeDecision?.certifiedApplicability
|
|
613
|
+
? `\nCertified applicability: ${contextPack.routeDecision.certifiedApplicability.name} is ${contextPack.routeDecision.certifiedApplicability.kind} (${contextPack.routeDecision.certifiedApplicability.reasons.join('; ')})`
|
|
614
|
+
: '';
|
|
596
615
|
const warnings = contextPack.warnings.length
|
|
597
616
|
? `Warnings:\n${contextPack.warnings.slice(0, 8).map((warning) => `- ${warning}`).join('\n')}\n`
|
|
598
617
|
: '';
|
|
@@ -609,21 +628,101 @@ function renderContextPackForPrompt(contextPack) {
|
|
|
609
628
|
? `\nCandidate conflicts:\n${contextPack.retrievalDiagnostics.candidateConflicts.slice(0, 4).map((conflict) => `- ${conflict.reason} ${conflict.prompt}`).join('\n')}`
|
|
610
629
|
: '';
|
|
611
630
|
const route = contextPack.routeDecision
|
|
612
|
-
? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
|
|
631
|
+
? `\nRoute decision: ${contextPack.routeDecision.route} / ${contextPack.routeDecision.intent}\nReason: ${contextPack.routeDecision.reason}${certifiedApplicability}\nMissing context: ${contextPack.routeDecision.missingContext.map((item) => item.message).join(' ') || 'none'}`
|
|
613
632
|
: '';
|
|
614
|
-
const allowed = contextPack
|
|
615
|
-
|
|
633
|
+
const allowed = renderAllowedSqlRelationsForPrompt(contextPack);
|
|
634
|
+
const joins = renderCandidateJoinsForPrompt(contextPack);
|
|
635
|
+
const relationDiagnostics = contextPack.retrievalDiagnostics?.selectedRelations?.length
|
|
636
|
+
? `\nSelected relation reasoning:\n${contextPack.retrievalDiagnostics.selectedRelations.slice(0, 8).map((relation) => `- ${relation.relation} (score ${relation.score.toFixed(1)}): ${relation.reason}`).join('\n')}`
|
|
616
637
|
: '';
|
|
638
|
+
const sourceSql = renderSourceBlockSqlContext(contextPack);
|
|
617
639
|
return [
|
|
618
640
|
`context_pack_id: ${contextPack.id}`,
|
|
619
641
|
`trust_label: ${contextPack.trustLabel}`,
|
|
642
|
+
questionPlan.trim(),
|
|
620
643
|
route.trim(),
|
|
621
644
|
warnings.trim(),
|
|
622
645
|
`Selected evidence:\n${objects || '- none'}`,
|
|
623
646
|
allowed.trim(),
|
|
647
|
+
joins.trim(),
|
|
648
|
+
relationDiagnostics.trim(),
|
|
649
|
+
sourceSql.trim(),
|
|
624
650
|
conflicts.trim(),
|
|
625
651
|
].filter(Boolean).join('\n');
|
|
626
652
|
}
|
|
653
|
+
function renderCandidateJoinsForPrompt(contextPack) {
|
|
654
|
+
const joins = contextPack.retrievalDiagnostics.selectedJoinPaths?.length
|
|
655
|
+
? contextPack.retrievalDiagnostics.selectedJoinPaths.slice(0, 8).map((join) => ({
|
|
656
|
+
leftRelation: join.leftRelation,
|
|
657
|
+
leftColumn: join.leftColumn,
|
|
658
|
+
rightRelation: join.rightRelation,
|
|
659
|
+
rightColumn: join.rightColumn,
|
|
660
|
+
reason: join.reason,
|
|
661
|
+
}))
|
|
662
|
+
: buildCandidateJoinPaths(schemaContextWithAllowedSqlContext([], contextPack)).slice(0, 8);
|
|
663
|
+
if (joins.length === 0)
|
|
664
|
+
return '';
|
|
665
|
+
return [
|
|
666
|
+
'Suggested join paths from selected metadata:',
|
|
667
|
+
...joins.map((join) => `- ${join.leftRelation}.${join.leftColumn} -> ${join.rightRelation}.${join.rightColumn}${join.reason ? ` (${join.reason})` : ''}`),
|
|
668
|
+
].join('\n');
|
|
669
|
+
}
|
|
670
|
+
function renderAllowedSqlRelationsForPrompt(contextPack) {
|
|
671
|
+
const relations = contextPack.allowedSqlContext?.relations ?? [];
|
|
672
|
+
if (relations.length === 0)
|
|
673
|
+
return '';
|
|
674
|
+
const selectedLookup = selectedRelationLookup(contextPack);
|
|
675
|
+
const cards = relations.slice(0, 12).map((relation, index) => {
|
|
676
|
+
const selection = relationSelectionFor(relation.relation, selectedLookup);
|
|
677
|
+
const rank = selection?.rank ?? index + 1;
|
|
678
|
+
const score = typeof selection?.score === 'number' ? `, score ${selection.score.toFixed(1)}` : '';
|
|
679
|
+
const reason = selection?.reason ? `\n why selected: ${selection.reason}` : '';
|
|
680
|
+
const columns = relation.columns.length
|
|
681
|
+
? relation.columns.slice(0, 32).map(formatRelationColumnForPrompt).join(', ')
|
|
682
|
+
: '(columns unavailable; use certified source SQL shape or inspect metadata before inventing columns)';
|
|
683
|
+
return [
|
|
684
|
+
`- [rank ${rank}${score}] ${relation.relation} (${relation.source})`,
|
|
685
|
+
reason,
|
|
686
|
+
`\n columns: ${columns}`,
|
|
687
|
+
].join('');
|
|
688
|
+
});
|
|
689
|
+
return [
|
|
690
|
+
'Selected SQL relation context:',
|
|
691
|
+
'Use these ranked relations and columns as the primary SQL-generation boundary. Prefer lower rank when multiple relations look plausible.',
|
|
692
|
+
...cards,
|
|
693
|
+
].join('\n');
|
|
694
|
+
}
|
|
695
|
+
function formatRelationColumnForPrompt(column) {
|
|
696
|
+
const type = column.type ? ` ${column.type}` : '';
|
|
697
|
+
const description = column.description ? ` - ${column.description.replace(/\s+/g, ' ').trim()}` : '';
|
|
698
|
+
const samples = column.sampleValues?.length
|
|
699
|
+
? `; matched values: ${column.sampleValues.slice(0, 4).map(formatPromptValue).join(', ')}`
|
|
700
|
+
: '';
|
|
701
|
+
return `${column.name}${type}${description}${samples}`;
|
|
702
|
+
}
|
|
703
|
+
function renderSourceBlockSqlContext(contextPack) {
|
|
704
|
+
const sources = contextPack.allowedSqlContext?.sourceBlockSql ?? [];
|
|
705
|
+
if (sources.length === 0)
|
|
706
|
+
return '';
|
|
707
|
+
const lines = sources.slice(0, 5).map((source) => {
|
|
708
|
+
const shape = extractSimpleSelectShape(source.sql);
|
|
709
|
+
const projectedColumns = shape
|
|
710
|
+
? shape.selectExpressions
|
|
711
|
+
.map(selectExpressionOutputName)
|
|
712
|
+
.filter((value) => Boolean(value))
|
|
713
|
+
.slice(0, 24)
|
|
714
|
+
.join(', ')
|
|
715
|
+
: '';
|
|
716
|
+
const snippet = compactSqlSnippet(source.sql, 280);
|
|
717
|
+
return [
|
|
718
|
+
`- ${source.name}${source.status ? ` (${source.status})` : ''}`,
|
|
719
|
+
shape?.relation ? ` relation: ${shape.relation}` : '',
|
|
720
|
+
projectedColumns ? ` projected columns: ${projectedColumns}` : '',
|
|
721
|
+
snippet ? ` sql: ${snippet}` : '',
|
|
722
|
+
].filter(Boolean).join('\n');
|
|
723
|
+
});
|
|
724
|
+
return `Certified source SQL shape context:\n${lines.join('\n')}`;
|
|
725
|
+
}
|
|
627
726
|
function contextPackCitations(contextPack, limit) {
|
|
628
727
|
if (!contextPack)
|
|
629
728
|
return [];
|
|
@@ -789,61 +888,773 @@ function buildSchemaAwareProposal(input) {
|
|
|
789
888
|
const drilldownProposal = buildMatchedEntityDrilldownProposal({ ...input, schemaContext });
|
|
790
889
|
if (drilldownProposal)
|
|
791
890
|
return drilldownProposal;
|
|
891
|
+
const profileProposal = buildEntityProfileProposal({ ...input, schemaContext });
|
|
892
|
+
if (profileProposal)
|
|
893
|
+
return profileProposal;
|
|
792
894
|
if (isFilteredEntityQuestion(input.question))
|
|
793
895
|
return undefined;
|
|
794
896
|
const lower = input.question.toLowerCase();
|
|
795
897
|
const asksForCustomerPerformance = /\bcustomers?\b/.test(lower)
|
|
796
898
|
&& /\border|orders|spend|revenue|perform|performed|better|top|best|rank|ranking\b/.test(lower)
|
|
797
899
|
&& !/\b(order details|specific orders|each order|all orders|order line|line item)\b/.test(lower);
|
|
798
|
-
if (
|
|
900
|
+
if (asksForCustomerPerformance) {
|
|
901
|
+
const customers = findSchemaTable(schemaContext, ['customers', 'customer']);
|
|
902
|
+
if (customers) {
|
|
903
|
+
const customerName = findSchemaColumn(customers, ['customer_name', 'name', 'full_name']);
|
|
904
|
+
const orderCount = findSchemaColumn(customers, ['count_lifetime_orders', 'lifetime_orders', 'order_count', 'orders_count', 'orders']);
|
|
905
|
+
const spend = findSchemaColumn(customers, ['lifetime_spend', 'total_lifetime_spend', 'customer_lifetime_value', 'total_revenue', 'revenue']);
|
|
906
|
+
if (customerName && orderCount && spend) {
|
|
907
|
+
return {
|
|
908
|
+
text: `Top performing customers ranked by ${businessMeasurePhrase(spend)} with ${businessMeasurePhrase(orderCount)} for context. This is AI-generated and needs analyst review before certification.`,
|
|
909
|
+
sql: [
|
|
910
|
+
'SELECT',
|
|
911
|
+
` ${sqlIdentifier(customerName)} AS customer_name,`,
|
|
912
|
+
` ${sqlIdentifier(orderCount)} AS orders,`,
|
|
913
|
+
` ROUND(${sqlIdentifier(spend)}, 2) AS lifetime_spend`,
|
|
914
|
+
`FROM ${sqlRelation(customers.relation)}`,
|
|
915
|
+
`ORDER BY ${sqlIdentifier(spend)} DESC, ${sqlIdentifier(orderCount)} DESC`,
|
|
916
|
+
'LIMIT 10',
|
|
917
|
+
].join('\n'),
|
|
918
|
+
viz: 'table',
|
|
919
|
+
};
|
|
920
|
+
}
|
|
921
|
+
const customerId = findSchemaColumn(customers, ['customer_id', 'id']);
|
|
922
|
+
const orders = findSchemaTable(schemaContext, ['orders', 'order']);
|
|
923
|
+
const orderCustomerId = orders ? findSchemaColumn(orders, ['customer_id', 'customer']) : undefined;
|
|
924
|
+
const orderTotal = orders ? findSchemaColumn(orders, ['order_total', 'total_order_amount', 'total_amount', 'amount', 'subtotal']) : undefined;
|
|
925
|
+
const orderId = orders ? findSchemaColumn(orders, ['order_id', 'id']) : undefined;
|
|
926
|
+
if (orders && customerName && customerId && orderCustomerId && orderTotal) {
|
|
927
|
+
const countExpression = orderId ? `COUNT(DISTINCT o.${sqlIdentifier(orderId)})` : 'COUNT(*)';
|
|
928
|
+
return {
|
|
929
|
+
text: `Top performing customers ranked from order totals with order count for context. This is AI-generated and needs analyst review before certification.`,
|
|
930
|
+
sql: [
|
|
931
|
+
'SELECT',
|
|
932
|
+
` c.${sqlIdentifier(customerName)} AS customer_name,`,
|
|
933
|
+
` ${countExpression} AS orders,`,
|
|
934
|
+
` ROUND(SUM(o.${sqlIdentifier(orderTotal)}), 2) AS lifetime_spend`,
|
|
935
|
+
`FROM ${sqlRelation(orders.relation)} AS o`,
|
|
936
|
+
`JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
|
|
937
|
+
`GROUP BY c.${sqlIdentifier(customerName)}`,
|
|
938
|
+
'ORDER BY lifetime_spend DESC, orders DESC',
|
|
939
|
+
'LIMIT 10',
|
|
940
|
+
].join('\n'),
|
|
941
|
+
viz: 'table',
|
|
942
|
+
};
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
946
|
+
const genericJoinProposal = buildGenericJoinProposal({ ...input, schemaContext });
|
|
947
|
+
if (genericJoinProposal)
|
|
948
|
+
return genericJoinProposal;
|
|
949
|
+
const genericProposal = buildGenericSingleTableProposal({ ...input, schemaContext });
|
|
950
|
+
if (genericProposal)
|
|
951
|
+
return genericProposal;
|
|
952
|
+
return undefined;
|
|
953
|
+
}
|
|
954
|
+
function buildGenericJoinProposal(input) {
|
|
955
|
+
const planMode = input.contextPack?.questionPlan?.mode;
|
|
956
|
+
if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
|
|
957
|
+
return undefined;
|
|
958
|
+
}
|
|
959
|
+
const lower = input.question.toLowerCase();
|
|
960
|
+
if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
|
|
961
|
+
return undefined;
|
|
962
|
+
}
|
|
963
|
+
if (!/\b(by\s+[a-z]|per\s+[a-z]|for each|group(?:ed)? by|split|segment|break\s*down|breakdown|compare)\b/i.test(lower)) {
|
|
964
|
+
return undefined;
|
|
965
|
+
}
|
|
966
|
+
const candidate = pickGenericJoinCandidate(input.schemaContext, input.question);
|
|
967
|
+
if (!candidate)
|
|
968
|
+
return undefined;
|
|
969
|
+
const direction = rankingDirectionFromText(input.question);
|
|
970
|
+
const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
|
|
971
|
+
const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
|
|
972
|
+
const chart = isTimeLikeGenericColumn(candidate.dimensionColumn) ? 'line' : 'bar';
|
|
973
|
+
const metricExpression = qualifiedMetricExpression(candidate.metric, 'f');
|
|
974
|
+
const metricAlias = candidate.metric.alias;
|
|
975
|
+
const dimensionAlias = candidate.dimensionColumn;
|
|
976
|
+
return {
|
|
977
|
+
text: `Prepared a review-required ${businessMeasurePhrase(candidate.metric.column)} breakdown by ${humanizeIdentifier(candidate.dimensionColumn)} by joining ${candidate.metricTable.relation} to ${candidate.dimensionTable.relation}. This result is uncertified until reviewed and promoted.`,
|
|
978
|
+
sql: [
|
|
979
|
+
'SELECT',
|
|
980
|
+
` d.${sqlIdentifier(candidate.dimensionColumn)} AS ${sqlIdentifier(dimensionAlias)},`,
|
|
981
|
+
` ${metricExpression} AS ${sqlIdentifier(metricAlias)}`,
|
|
982
|
+
`FROM ${sqlRelation(candidate.metricTable.relation)} AS f`,
|
|
983
|
+
`JOIN ${sqlRelation(candidate.dimensionTable.relation)} AS d ON f.${sqlIdentifier(candidate.metricJoinColumn)} = d.${sqlIdentifier(candidate.dimensionJoinColumn)}`,
|
|
984
|
+
`GROUP BY d.${sqlIdentifier(candidate.dimensionColumn)}`,
|
|
985
|
+
`ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
|
|
986
|
+
`LIMIT ${limit}`,
|
|
987
|
+
].join('\n'),
|
|
988
|
+
viz: chart,
|
|
989
|
+
};
|
|
990
|
+
}
|
|
991
|
+
function buildGenericSingleTableProposal(input) {
|
|
992
|
+
const planMode = input.contextPack?.questionPlan?.mode;
|
|
993
|
+
if (planMode === 'entity_profile' || planMode === 'diagnose_change' || planMode === 'anomaly' || planMode === 'trust_review') {
|
|
994
|
+
return undefined;
|
|
995
|
+
}
|
|
996
|
+
const lower = input.question.toLowerCase();
|
|
997
|
+
if (/\b(why|root cause|diagnos|changed?|drop|decline|increase|decrease|repair|fix|bad|details?|detail rows?|line items?|raw rows?)\b/i.test(lower)) {
|
|
998
|
+
return undefined;
|
|
999
|
+
}
|
|
1000
|
+
if (!/\b(show|list|top|bottom|best|worst|highest|lowest|least|fewest|rank|ranking|compare|trend|by\s+[a-z]|how many|number of|revenue|sales|orders?|points?|score|count|total|average|avg|sum)\b/i.test(lower)) {
|
|
799
1001
|
return undefined;
|
|
800
|
-
|
|
801
|
-
|
|
1002
|
+
}
|
|
1003
|
+
const table = pickGenericAnalysisTable(input.schemaContext, input.question);
|
|
1004
|
+
if (!table)
|
|
802
1005
|
return undefined;
|
|
803
|
-
const
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
1006
|
+
const metric = inferGenericMetric(table, input.question);
|
|
1007
|
+
if (!metric)
|
|
1008
|
+
return undefined;
|
|
1009
|
+
const dimensions = augmentGenericRankingDimensions(table, inferGenericDimensions(table, input.question, metric.column), metric, input.question).slice(0, 2);
|
|
1010
|
+
const direction = rankingDirectionFromText(input.question);
|
|
1011
|
+
const limit = /\b(all|complete|full)\b/i.test(input.question) ? 50 : 10;
|
|
1012
|
+
const metricAlias = metric.alias;
|
|
1013
|
+
if (dimensions.length === 0) {
|
|
807
1014
|
return {
|
|
808
|
-
text: `
|
|
1015
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} summary from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
809
1016
|
sql: [
|
|
810
1017
|
'SELECT',
|
|
811
|
-
` ${sqlIdentifier(
|
|
812
|
-
`
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
1018
|
+
` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
|
|
1019
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1020
|
+
].join('\n'),
|
|
1021
|
+
viz: 'single_value',
|
|
1022
|
+
};
|
|
1023
|
+
}
|
|
1024
|
+
if (direction && metric.preAggregated) {
|
|
1025
|
+
return {
|
|
1026
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} ranking by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
1027
|
+
sql: [
|
|
1028
|
+
'SELECT',
|
|
1029
|
+
...dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`),
|
|
1030
|
+
` ${sqlIdentifier(metric.column)} AS ${sqlIdentifier(metric.column)}`,
|
|
1031
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1032
|
+
`ORDER BY ${sqlIdentifier(metric.column)} ${direction === 'bottom' ? 'ASC' : 'DESC'}`,
|
|
1033
|
+
`LIMIT ${limit}`,
|
|
817
1034
|
].join('\n'),
|
|
818
1035
|
viz: 'table',
|
|
819
1036
|
};
|
|
820
1037
|
}
|
|
821
|
-
const
|
|
822
|
-
const
|
|
823
|
-
|
|
1038
|
+
const selectDimensions = dimensions.map((dimension) => ` ${sqlIdentifier(dimension)} AS ${sqlIdentifier(dimension)},`);
|
|
1039
|
+
const groupBy = dimensions.map(sqlIdentifier).join(', ');
|
|
1040
|
+
const orderDirection = direction === 'bottom' ? 'ASC' : 'DESC';
|
|
1041
|
+
const chart = dimensions.some(isTimeLikeGenericColumn) ? 'line' : 'bar';
|
|
1042
|
+
return {
|
|
1043
|
+
text: `Prepared a review-required ${businessMeasurePhrase(metric.column)} breakdown by ${dimensions.map(humanizeIdentifier).join(' and ')} from ${table.relation}. This result is uncertified until reviewed and promoted.`,
|
|
1044
|
+
sql: [
|
|
1045
|
+
'SELECT',
|
|
1046
|
+
...selectDimensions,
|
|
1047
|
+
` ${metric.expression} AS ${sqlIdentifier(metricAlias)}`,
|
|
1048
|
+
`FROM ${sqlRelation(table.relation)}`,
|
|
1049
|
+
`GROUP BY ${groupBy}`,
|
|
1050
|
+
`ORDER BY ${sqlIdentifier(metricAlias)} ${orderDirection}`,
|
|
1051
|
+
`LIMIT ${limit}`,
|
|
1052
|
+
].join('\n'),
|
|
1053
|
+
viz: chart,
|
|
1054
|
+
};
|
|
1055
|
+
}
|
|
1056
|
+
function pickGenericAnalysisTable(schemaContext, question) {
|
|
1057
|
+
return schemaContext
|
|
1058
|
+
.filter((table) => table.columns.length > 0)
|
|
1059
|
+
.map((table, index) => ({
|
|
1060
|
+
table,
|
|
1061
|
+
metric: inferGenericMetric(table, question),
|
|
1062
|
+
dimensions: inferGenericDimensions(table, question),
|
|
1063
|
+
index,
|
|
1064
|
+
}))
|
|
1065
|
+
.filter((candidate) => candidate.metric)
|
|
1066
|
+
.sort((a, b) => genericTableScore(b.table, question, b.metric, b.dimensions) -
|
|
1067
|
+
genericTableScore(a.table, question, a.metric, a.dimensions) ||
|
|
1068
|
+
a.index - b.index)[0]?.table;
|
|
1069
|
+
}
|
|
1070
|
+
function genericTableScore(table, question, metric, dimensions) {
|
|
1071
|
+
const questionTokens = meaningfulTokens(question);
|
|
1072
|
+
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1073
|
+
const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
|
|
1074
|
+
const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
|
|
1075
|
+
const selectedRelationBoost = table.selectionRank
|
|
1076
|
+
? Math.max(0, 56 - table.selectionRank * 3) + Math.min(table.selectionScore ?? 0, 24)
|
|
1077
|
+
: 0;
|
|
1078
|
+
return selectedRelationBoost + overlap * 8 + (metric?.score ?? 0) + dimensions.length * 8 + Math.min(table.columns.length, 20) * 0.3;
|
|
1079
|
+
}
|
|
1080
|
+
function pickGenericJoinCandidate(schemaContext, question) {
|
|
1081
|
+
const requestedDimensions = requestedDimensionPhrases(question);
|
|
1082
|
+
if (requestedDimensions.length === 0)
|
|
1083
|
+
return undefined;
|
|
1084
|
+
const metricCandidates = schemaContext
|
|
1085
|
+
.filter((table) => table.columns.length > 0)
|
|
1086
|
+
.map((table, index) => ({
|
|
1087
|
+
table,
|
|
1088
|
+
metric: inferGenericMetric(table, question),
|
|
1089
|
+
localDimensions: inferGenericDimensions(table, question),
|
|
1090
|
+
index,
|
|
1091
|
+
}))
|
|
1092
|
+
.filter((candidate) => Boolean(candidate.metric))
|
|
1093
|
+
.filter((candidate) => !candidate.localDimensions.some((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension)));
|
|
1094
|
+
const candidates = [];
|
|
1095
|
+
for (const metricCandidate of metricCandidates) {
|
|
1096
|
+
for (const dimensionTable of schemaContext) {
|
|
1097
|
+
if (normalizeRelationKey(dimensionTable.relation) === normalizeRelationKey(metricCandidate.table.relation))
|
|
1098
|
+
continue;
|
|
1099
|
+
const dimensionColumn = pickRequestedDimensionColumn(dimensionTable, question, requestedDimensions);
|
|
1100
|
+
if (!dimensionColumn || namesEqualLoose(dimensionColumn, metricCandidate.metric.column))
|
|
1101
|
+
continue;
|
|
1102
|
+
const join = pickJoinColumns(metricCandidate.table, dimensionTable);
|
|
1103
|
+
if (!join)
|
|
1104
|
+
continue;
|
|
1105
|
+
const dimensionScore = genericDimensionScore({ name: dimensionColumn, type: findColumnType(dimensionTable, dimensionColumn) }, question, Math.max(0, dimensionTable.columns.findIndex((column) => namesEqualLoose(column.name, dimensionColumn))), metricCandidate.metric.column);
|
|
1106
|
+
candidates.push({
|
|
1107
|
+
metricTable: metricCandidate.table,
|
|
1108
|
+
dimensionTable,
|
|
1109
|
+
metric: metricCandidate.metric,
|
|
1110
|
+
dimensionColumn,
|
|
1111
|
+
metricJoinColumn: join.leftColumn,
|
|
1112
|
+
dimensionJoinColumn: join.rightColumn,
|
|
1113
|
+
score: genericTableScore(metricCandidate.table, question, metricCandidate.metric, metricCandidate.localDimensions) +
|
|
1114
|
+
dimensionScore +
|
|
1115
|
+
join.score +
|
|
1116
|
+
selectedRelationWeight(dimensionTable),
|
|
1117
|
+
});
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
return candidates
|
|
1121
|
+
.sort((a, b) => b.score - a.score ||
|
|
1122
|
+
(a.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.metricTable.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
1123
|
+
a.metricTable.relation.localeCompare(b.metricTable.relation))[0];
|
|
1124
|
+
}
|
|
1125
|
+
function buildCandidateJoinPaths(schemaContext) {
|
|
1126
|
+
const tables = [...schemaContext]
|
|
1127
|
+
.filter((table) => table.columns.some((column) => isJoinKeyColumn(column.name)))
|
|
1128
|
+
.sort((a, b) => (a.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
1129
|
+
a.relation.localeCompare(b.relation))
|
|
1130
|
+
.slice(0, 16);
|
|
1131
|
+
const joins = [];
|
|
1132
|
+
const seen = new Set();
|
|
1133
|
+
for (let leftIndex = 0; leftIndex < tables.length; leftIndex += 1) {
|
|
1134
|
+
for (let rightIndex = leftIndex + 1; rightIndex < tables.length; rightIndex += 1) {
|
|
1135
|
+
const left = tables[leftIndex];
|
|
1136
|
+
const right = tables[rightIndex];
|
|
1137
|
+
const join = pickJoinColumns(left, right);
|
|
1138
|
+
if (!join)
|
|
1139
|
+
continue;
|
|
1140
|
+
const key = [
|
|
1141
|
+
normalizeRelationKey(left.relation),
|
|
1142
|
+
join.leftColumn.toLowerCase(),
|
|
1143
|
+
normalizeRelationKey(right.relation),
|
|
1144
|
+
join.rightColumn.toLowerCase(),
|
|
1145
|
+
].join('|');
|
|
1146
|
+
if (seen.has(key))
|
|
1147
|
+
continue;
|
|
1148
|
+
seen.add(key);
|
|
1149
|
+
joins.push({
|
|
1150
|
+
leftRelation: left.relation,
|
|
1151
|
+
leftColumn: join.leftColumn,
|
|
1152
|
+
rightRelation: right.relation,
|
|
1153
|
+
rightColumn: join.rightColumn,
|
|
1154
|
+
reason: joinReason(join.leftColumn, join.rightColumn, join.score),
|
|
1155
|
+
});
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
return joins.slice(0, 12);
|
|
1159
|
+
}
|
|
1160
|
+
function joinReason(leftColumn, rightColumn, score) {
|
|
1161
|
+
if (namesEqualLoose(leftColumn, rightColumn))
|
|
1162
|
+
return `shared key ${leftColumn}`;
|
|
1163
|
+
const leftSubject = joinSubjectForColumn(leftColumn);
|
|
1164
|
+
const rightSubject = joinSubjectForColumn(rightColumn);
|
|
1165
|
+
if (leftSubject && rightSubject && leftSubject === rightSubject)
|
|
1166
|
+
return `matching ${leftSubject} key`;
|
|
1167
|
+
if (score >= 55)
|
|
1168
|
+
return 'foreign-key style id match';
|
|
1169
|
+
return 'join-key style column match';
|
|
1170
|
+
}
|
|
1171
|
+
function requestedDimensionPhrases(question) {
|
|
1172
|
+
const lower = question.toLowerCase();
|
|
1173
|
+
const phrases = [
|
|
1174
|
+
...Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1175
|
+
...Array.from(lower.matchAll(/\bper\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1176
|
+
...Array.from(lower.matchAll(/\bfor each\s+([a-z][a-z0-9_ -]{1,50})/g)).map((match) => match[1] ?? ''),
|
|
1177
|
+
];
|
|
1178
|
+
return uniqueDrilldownStrings(phrases
|
|
1179
|
+
.flatMap((phrase) => phrase.split(/\band\b|,|\//i))
|
|
1180
|
+
.map((phrase) => phrase
|
|
1181
|
+
.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time|where|for|with|from|in|during|last|this|next)\b.*$/gi, '')
|
|
1182
|
+
.replace(/[^a-z0-9_ -]+/gi, ' ')
|
|
1183
|
+
.replace(/\s+/g, ' ')
|
|
1184
|
+
.trim())
|
|
1185
|
+
.filter((phrase) => phrase.length > 0));
|
|
1186
|
+
}
|
|
1187
|
+
function pickRequestedDimensionColumn(table, question, requestedDimensions) {
|
|
1188
|
+
for (const phrase of requestedDimensions) {
|
|
1189
|
+
const direct = findSchemaColumn(table, dimensionColumnCandidatesForPhrase(phrase));
|
|
1190
|
+
if (direct && !isJoinKeyColumn(direct) && !isNumericLikeColumn({ name: direct, type: findColumnType(table, direct) })) {
|
|
1191
|
+
return direct;
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1194
|
+
const inferred = inferGenericDimensions(table, question)
|
|
1195
|
+
.find((dimension) => !isJoinKeyColumn(dimension) && requestedDimensionMatchesColumn(requestedDimensions, dimension));
|
|
1196
|
+
return inferred;
|
|
1197
|
+
}
|
|
1198
|
+
function dimensionColumnCandidatesForPhrase(phrase) {
|
|
1199
|
+
const tokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
|
|
1200
|
+
.flatMap((token) => token.split('_'))
|
|
1201
|
+
.map(normalizeToken)
|
|
1202
|
+
.filter((token) => token.length > 0);
|
|
1203
|
+
if (tokens.length === 0)
|
|
1204
|
+
return [];
|
|
1205
|
+
const joined = tokens.join('_');
|
|
1206
|
+
const tail = tokens.slice(1).join('_');
|
|
1207
|
+
return uniqueDrilldownStrings([
|
|
1208
|
+
joined,
|
|
1209
|
+
phrase.replace(/\s+/g, '_').toLowerCase(),
|
|
1210
|
+
tail,
|
|
1211
|
+
tokens.at(-1) ?? '',
|
|
1212
|
+
...tokens,
|
|
1213
|
+
].filter(Boolean));
|
|
1214
|
+
}
|
|
1215
|
+
function requestedDimensionMatchesColumn(requestedDimensions, column) {
|
|
1216
|
+
const columnTokens = exactMatchTokens(column);
|
|
1217
|
+
return requestedDimensions.some((phrase) => {
|
|
1218
|
+
const phraseTokens = (phrase.toLowerCase().match(/[a-z0-9_]+/g) ?? [])
|
|
1219
|
+
.flatMap((token) => token.split('_'))
|
|
1220
|
+
.map(normalizeToken)
|
|
1221
|
+
.filter((token) => token.length > 0);
|
|
1222
|
+
return phraseTokens.some((token) => columnTokens.has(token));
|
|
1223
|
+
});
|
|
1224
|
+
}
|
|
1225
|
+
function selectedRelationWeight(table) {
|
|
1226
|
+
return table.selectionRank
|
|
1227
|
+
? Math.max(0, 30 - table.selectionRank * 2) + Math.min(table.selectionScore ?? 0, 16)
|
|
1228
|
+
: 0;
|
|
1229
|
+
}
|
|
1230
|
+
function pickJoinColumns(left, right) {
|
|
1231
|
+
const candidates = [];
|
|
1232
|
+
const leftColumns = left.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
|
|
1233
|
+
const rightColumns = right.columns.filter((column) => isJoinKeyColumn(column.name)).slice(0, 32);
|
|
1234
|
+
for (const leftColumn of leftColumns) {
|
|
1235
|
+
for (const rightColumn of rightColumns) {
|
|
1236
|
+
const score = joinColumnScore(leftColumn.name, rightColumn.name, left, right);
|
|
1237
|
+
if (score <= 0)
|
|
1238
|
+
continue;
|
|
1239
|
+
candidates.push({ leftColumn: leftColumn.name, rightColumn: rightColumn.name, score });
|
|
1240
|
+
}
|
|
1241
|
+
}
|
|
1242
|
+
return candidates.sort((a, b) => b.score - a.score || a.leftColumn.localeCompare(b.leftColumn))[0];
|
|
1243
|
+
}
|
|
1244
|
+
function joinColumnScore(leftColumn, rightColumn, leftTable, rightTable) {
|
|
1245
|
+
const leftKey = normalizeRelationKey(leftColumn).replace(/\./g, '_');
|
|
1246
|
+
const rightKey = normalizeRelationKey(rightColumn).replace(/\./g, '_');
|
|
1247
|
+
const leftJoinLike = isJoinKeyColumn(leftColumn);
|
|
1248
|
+
const rightJoinLike = isJoinKeyColumn(rightColumn);
|
|
1249
|
+
if (!leftJoinLike || !rightJoinLike)
|
|
1250
|
+
return 0;
|
|
1251
|
+
if (leftKey === rightKey)
|
|
1252
|
+
return 70;
|
|
1253
|
+
const leftSubject = joinSubjectForColumn(leftColumn);
|
|
1254
|
+
const rightSubject = joinSubjectForColumn(rightColumn);
|
|
1255
|
+
if (leftSubject && rightSubject && leftSubject === rightSubject)
|
|
1256
|
+
return 62;
|
|
1257
|
+
const leftTableTokens = tableEntityTokens(leftTable);
|
|
1258
|
+
const rightTableTokens = tableEntityTokens(rightTable);
|
|
1259
|
+
if (leftSubject && rightKey === 'id' && rightTableTokens.has(leftSubject))
|
|
1260
|
+
return 58;
|
|
1261
|
+
if (rightSubject && leftKey === 'id' && leftTableTokens.has(rightSubject))
|
|
1262
|
+
return 58;
|
|
1263
|
+
if (leftSubject && rightTableTokens.has(leftSubject) && rightKey.endsWith('_key'))
|
|
1264
|
+
return 42;
|
|
1265
|
+
if (rightSubject && leftTableTokens.has(rightSubject) && leftKey.endsWith('_key'))
|
|
1266
|
+
return 42;
|
|
1267
|
+
return 0;
|
|
1268
|
+
}
|
|
1269
|
+
function isJoinKeyColumn(column) {
|
|
1270
|
+
const normalized = column.toLowerCase();
|
|
1271
|
+
return normalized === 'id' ||
|
|
1272
|
+
/(^|_)(id|key|uuid|sk)$/.test(normalized) ||
|
|
1273
|
+
/_(id|key|uuid|sk)$/.test(normalized);
|
|
1274
|
+
}
|
|
1275
|
+
function joinSubjectForColumn(column) {
|
|
1276
|
+
const normalized = column.toLowerCase();
|
|
1277
|
+
const subject = normalized.replace(/_(id|key|uuid|sk)$/i, '');
|
|
1278
|
+
if (!subject || subject === normalized || subject === 'id' || subject === 'key')
|
|
1279
|
+
return undefined;
|
|
1280
|
+
return normalizeToken(subject.split('_').at(-1) ?? subject);
|
|
1281
|
+
}
|
|
1282
|
+
function tableEntityTokens(table) {
|
|
1283
|
+
const tokens = exactMatchTokens([table.name, table.relation.split('.').at(-1) ?? table.relation].join(' '));
|
|
1284
|
+
for (const generic of ['dim', 'fct', 'fact', 'stg', 'stage', 'model', 'table'])
|
|
1285
|
+
tokens.delete(generic);
|
|
1286
|
+
return tokens;
|
|
1287
|
+
}
|
|
1288
|
+
function findColumnType(table, columnName) {
|
|
1289
|
+
return table.columns.find((column) => namesEqualLoose(column.name, columnName))?.type;
|
|
1290
|
+
}
|
|
1291
|
+
function qualifiedMetricExpression(metric, tableAlias) {
|
|
1292
|
+
const column = `${tableAlias}.${sqlIdentifier(metric.column)}`;
|
|
1293
|
+
if (metric.column === 'rows')
|
|
1294
|
+
return 'COUNT(*)';
|
|
1295
|
+
if (/^COUNT\s*\(\s*DISTINCT\b/i.test(metric.expression))
|
|
1296
|
+
return `COUNT(DISTINCT ${column})`;
|
|
1297
|
+
if (metric.preAggregated)
|
|
1298
|
+
return column;
|
|
1299
|
+
const aggregate = metric.expression.match(/^(SUM|AVG|MIN|MAX|MEDIAN)\s*\(/i)?.[1]?.toUpperCase()
|
|
1300
|
+
?? (/avg|average|mean/i.test(metric.alias) ? 'AVG' : 'SUM');
|
|
1301
|
+
return `${aggregate}(${column})`;
|
|
1302
|
+
}
|
|
1303
|
+
function inferGenericMetric(table, question) {
|
|
1304
|
+
const lower = question.toLowerCase();
|
|
1305
|
+
const wantsCount = /\b(count|how many|number of|volume)\b/i.test(lower);
|
|
1306
|
+
const candidates = table.columns
|
|
1307
|
+
.map((column) => {
|
|
1308
|
+
const name = column.name.toLowerCase();
|
|
1309
|
+
const tokens = meaningfulTokens(column.name);
|
|
1310
|
+
let score = isNumericLikeColumn(column) ? 20 : 0;
|
|
1311
|
+
for (const token of meaningfulTokens(question)) {
|
|
1312
|
+
if (tokens.has(token))
|
|
1313
|
+
score += 18;
|
|
1314
|
+
}
|
|
1315
|
+
if (/\b(revenue|sales|amount|total|spend|cost|margin|profit|value|points?|score|orders?|count|quantity|duration|minutes?|rate|average|avg)\b/i.test(name.replace(/_/g, ' '))) {
|
|
1316
|
+
score += 18;
|
|
1317
|
+
}
|
|
1318
|
+
if (/\b(id|key|code|zip|postal|phone|season|year|month|week|day|date|time)\b/i.test(name.replace(/_/g, ' '))) {
|
|
1319
|
+
score -= 14;
|
|
1320
|
+
}
|
|
1321
|
+
return { column: column.name, score };
|
|
1322
|
+
})
|
|
1323
|
+
.filter((candidate) => candidate.score > 0)
|
|
1324
|
+
.sort((a, b) => b.score - a.score);
|
|
1325
|
+
const countSubject = wantsCount ? inferCountSubject(question) : undefined;
|
|
1326
|
+
if (wantsCount) {
|
|
1327
|
+
const precomputed = candidates.find((candidate) => isPrecomputedCountMetric(candidate.column, countSubject));
|
|
1328
|
+
if (precomputed) {
|
|
1329
|
+
return {
|
|
1330
|
+
column: precomputed.column,
|
|
1331
|
+
expression: isPreAggregatedMetricColumn(precomputed.column) ? sqlIdentifier(precomputed.column) : `SUM(${sqlIdentifier(precomputed.column)})`,
|
|
1332
|
+
alias: isPreAggregatedMetricColumn(precomputed.column) ? precomputed.column : `${precomputed.column}_sum`,
|
|
1333
|
+
score: precomputed.score + 8,
|
|
1334
|
+
preAggregated: isPreAggregatedMetricColumn(precomputed.column),
|
|
1335
|
+
};
|
|
1336
|
+
}
|
|
1337
|
+
const distinctColumn = countSubject ? findDistinctCountColumn(table, countSubject) : undefined;
|
|
1338
|
+
if (distinctColumn) {
|
|
1339
|
+
return {
|
|
1340
|
+
column: distinctColumn,
|
|
1341
|
+
expression: `COUNT(DISTINCT ${sqlIdentifier(distinctColumn)})`,
|
|
1342
|
+
alias: `${countSubject}_count`,
|
|
1343
|
+
score: 34,
|
|
1344
|
+
preAggregated: false,
|
|
1345
|
+
};
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
const selected = candidates[0];
|
|
1349
|
+
if (!selected && !wantsCount)
|
|
1350
|
+
return undefined;
|
|
1351
|
+
if (!selected || wantsCount && selected.score < 26) {
|
|
1352
|
+
return {
|
|
1353
|
+
column: 'rows',
|
|
1354
|
+
expression: 'COUNT(*)',
|
|
1355
|
+
alias: 'row_count',
|
|
1356
|
+
score: 18,
|
|
1357
|
+
preAggregated: true,
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
1360
|
+
const aggregate = /\b(avg|average|mean)\b/i.test(lower) ? 'AVG' : 'SUM';
|
|
1361
|
+
const aliasBase = `${selected.column}_${aggregate.toLowerCase()}`;
|
|
1362
|
+
const preAggregated = isPreAggregatedMetricColumn(selected.column);
|
|
1363
|
+
return {
|
|
1364
|
+
column: selected.column,
|
|
1365
|
+
expression: preAggregated ? sqlIdentifier(selected.column) : `${aggregate}(${sqlIdentifier(selected.column)})`,
|
|
1366
|
+
alias: preAggregated ? selected.column : aliasBase,
|
|
1367
|
+
score: selected.score,
|
|
1368
|
+
preAggregated,
|
|
1369
|
+
};
|
|
1370
|
+
}
|
|
1371
|
+
function inferCountSubject(question) {
|
|
1372
|
+
const lower = question.toLowerCase();
|
|
1373
|
+
const match = lower.match(/\b(?:how many|number of|count of|count|total)\s+(?:distinct\s+|unique\s+)?([a-z][a-z0-9_-]*)/);
|
|
1374
|
+
const raw = match?.[1] ?? lower.match(/\b(customers?|accounts?|users?|members?|orders?|products?|players?|teams?|transactions?|sessions?|events?|records?|rows?)\b/)?.[1];
|
|
1375
|
+
if (!raw)
|
|
1376
|
+
return undefined;
|
|
1377
|
+
const normalized = normalizeToken(raw.replace(/[^a-z0-9_]/g, ''));
|
|
1378
|
+
if (!normalized || normalized === 'row' || normalized === 'record')
|
|
1379
|
+
return undefined;
|
|
1380
|
+
return normalized;
|
|
1381
|
+
}
|
|
1382
|
+
function findDistinctCountColumn(table, subject) {
|
|
1383
|
+
const preferred = [
|
|
1384
|
+
`${subject}_id`,
|
|
1385
|
+
`${subject}_key`,
|
|
1386
|
+
`${subject}_uuid`,
|
|
1387
|
+
`${subject}_sk`,
|
|
1388
|
+
`${subject}_name`,
|
|
1389
|
+
subject,
|
|
1390
|
+
];
|
|
1391
|
+
const direct = findSchemaColumn(table, preferred);
|
|
1392
|
+
if (direct)
|
|
1393
|
+
return direct;
|
|
1394
|
+
const tableTokens = meaningfulTokens([table.name, table.relation].join(' '));
|
|
1395
|
+
if (!tableTokens.has(subject))
|
|
1396
|
+
return undefined;
|
|
1397
|
+
return findSchemaColumn(table, ['id', 'key', 'uuid']);
|
|
1398
|
+
}
|
|
1399
|
+
function isPrecomputedCountMetric(column, subject) {
|
|
1400
|
+
const normalized = column.toLowerCase();
|
|
1401
|
+
const text = normalized.replace(/_/g, ' ');
|
|
1402
|
+
if (/\b(id|key|uuid|code)\b/i.test(text))
|
|
1403
|
+
return false;
|
|
1404
|
+
if (/\b(count|cnt|number|num)\b/i.test(text)) {
|
|
1405
|
+
return !subject || text.includes(subject) || text.includes(pluralizeSimple(subject));
|
|
1406
|
+
}
|
|
1407
|
+
return Boolean(subject && (normalized === `total_${subject}` ||
|
|
1408
|
+
normalized === `total_${pluralizeSimple(subject)}`));
|
|
1409
|
+
}
|
|
1410
|
+
function pluralizeSimple(value) {
|
|
1411
|
+
if (value.endsWith('y'))
|
|
1412
|
+
return `${value.slice(0, -1)}ies`;
|
|
1413
|
+
if (value.endsWith('s'))
|
|
1414
|
+
return value;
|
|
1415
|
+
return `${value}s`;
|
|
1416
|
+
}
|
|
1417
|
+
function augmentGenericRankingDimensions(table, dimensions, metric, question) {
|
|
1418
|
+
const direction = rankingDirectionFromText(question);
|
|
1419
|
+
if (!direction || !metric.preAggregated || /\bby\s+/i.test(question))
|
|
1420
|
+
return dimensions;
|
|
1421
|
+
const timeColumn = table.columns
|
|
1422
|
+
.map((column) => column.name)
|
|
1423
|
+
.find((name) => isTimeLikeGenericColumn(name) && !dimensions.some((dimension) => namesEqualLoose(dimension, name)));
|
|
1424
|
+
return timeColumn ? [...dimensions, timeColumn] : dimensions;
|
|
1425
|
+
}
|
|
1426
|
+
function inferGenericDimensions(table, question, metricColumn) {
|
|
1427
|
+
const lower = question.toLowerCase();
|
|
1428
|
+
const requested = Array.from(lower.matchAll(/\bby\s+([a-z][a-z0-9_ -]{1,40})/g))
|
|
1429
|
+
.map((match) => match[1] ?? '')
|
|
1430
|
+
.flatMap((value) => value.split(/\band\b|,|\//i))
|
|
1431
|
+
.map((value) => value.replace(/\b(desc|asc|top|bottom|highest|lowest|least|most|over time)\b/gi, '').trim())
|
|
1432
|
+
.filter(Boolean);
|
|
1433
|
+
const direct = requested
|
|
1434
|
+
.map((value) => findSchemaColumn(table, [value, value.replace(/\s+/g, '_')]))
|
|
1435
|
+
.filter((value) => Boolean(value));
|
|
1436
|
+
if (direct.length > 0)
|
|
1437
|
+
return uniqueDrilldownStrings(direct);
|
|
1438
|
+
const domainHints = [
|
|
1439
|
+
/\bcustomers?\b/.test(lower) ? ['customer_name', 'customer', 'customer_id'] : [],
|
|
1440
|
+
/\bproducts?\b/.test(lower) ? ['product_name', 'product', 'sku', 'product_id'] : [],
|
|
1441
|
+
/\bplayers?\b/.test(lower) ? ['player_name', 'player', 'player_id'] : [],
|
|
1442
|
+
/\bteams?\b/.test(lower) ? ['team_name', 'team', 'team_id'] : [],
|
|
1443
|
+
/\bregions?\b|\bmarkets?\b/.test(lower) ? ['region', 'market', 'geo'] : [],
|
|
1444
|
+
/\bsegments?\b/.test(lower) ? ['segment', 'customer_segment', 'type'] : [],
|
|
1445
|
+
/\bchannels?\b/.test(lower) ? ['channel', 'source_channel'] : [],
|
|
1446
|
+
/\b(month|monthly)\b/.test(lower) ? ['month', 'order_month', 'created_month'] : [],
|
|
1447
|
+
/\b(week|weekly)\b/.test(lower) ? ['week', 'order_week', 'created_week'] : [],
|
|
1448
|
+
/\b(year|season|yearly)\b/.test(lower) ? ['season', 'year', 'order_year'] : [],
|
|
1449
|
+
].flat();
|
|
1450
|
+
const hinted = findSchemaColumn(table, domainHints);
|
|
1451
|
+
if (hinted)
|
|
1452
|
+
return [hinted];
|
|
1453
|
+
const fallback = table.columns
|
|
1454
|
+
.map((column, index) => ({
|
|
1455
|
+
column: column.name,
|
|
1456
|
+
score: genericDimensionScore(column, question, index, metricColumn),
|
|
1457
|
+
}))
|
|
1458
|
+
.filter((candidate) => candidate.score > 0)
|
|
1459
|
+
.sort((a, b) => b.score - a.score)
|
|
1460
|
+
.map((candidate) => candidate.column);
|
|
1461
|
+
return uniqueDrilldownStrings(fallback).slice(0, 1);
|
|
1462
|
+
}
|
|
1463
|
+
function genericDimensionScore(column, question, index, metricColumn) {
|
|
1464
|
+
if (metricColumn && namesEqualLoose(column.name, metricColumn))
|
|
1465
|
+
return -100;
|
|
1466
|
+
const name = column.name.toLowerCase();
|
|
1467
|
+
const text = name.replace(/_/g, ' ');
|
|
1468
|
+
let score = Math.max(0, 10 - index * 0.05);
|
|
1469
|
+
if (/\b(name|category|type|segment|region|market|channel|status|team|player|customer|product|vendor|account|month|week|year|season|date)\b/i.test(text)) {
|
|
1470
|
+
score += 24;
|
|
1471
|
+
}
|
|
1472
|
+
if (isNumericLikeColumn(column) && !isTimeLikeGenericColumn(column.name))
|
|
1473
|
+
score -= 18;
|
|
1474
|
+
for (const token of meaningfulTokens(question)) {
|
|
1475
|
+
if (meaningfulTokens(column.name).has(token))
|
|
1476
|
+
score += 10;
|
|
1477
|
+
}
|
|
1478
|
+
return score;
|
|
1479
|
+
}
|
|
1480
|
+
function isNumericLikeColumn(column) {
|
|
1481
|
+
const type = column.type ?? '';
|
|
1482
|
+
const name = column.name.toLowerCase();
|
|
1483
|
+
return /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(type)
|
|
1484
|
+
|| /\b(total|count|amount|revenue|sales|spend|cost|margin|profit|value|points?|score|quantity|duration|minutes?|rate|avg|average)\b/i.test(name.replace(/_/g, ' '));
|
|
1485
|
+
}
|
|
1486
|
+
function isPreAggregatedMetricColumn(name) {
|
|
1487
|
+
return /^(total|count|avg|average|median|min|max)_/i.test(name)
|
|
1488
|
+
|| /_(total|count|avg|average|median|min|max)$/i.test(name);
|
|
1489
|
+
}
|
|
1490
|
+
function isTimeLikeGenericColumn(name) {
|
|
1491
|
+
return /\b(date|time|day|week|month|quarter|year|season|period)\b/i.test(name);
|
|
1492
|
+
}
|
|
1493
|
+
function buildEntityProfileProposal(input) {
|
|
1494
|
+
const entityTexts = entityMentionsForProfile(input.question, input.contextPack);
|
|
1495
|
+
if (entityTexts.length === 0 || !isEntityProfileQuestion(input.question, input.contextPack))
|
|
824
1496
|
return undefined;
|
|
825
|
-
const
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
1497
|
+
const candidates = input.schemaContext
|
|
1498
|
+
.map((table) => profileTableCandidate(table, input.question, input.followUp, entityTexts))
|
|
1499
|
+
.filter((candidate) => Boolean(candidate))
|
|
1500
|
+
.sort((a, b) => b.score - a.score);
|
|
1501
|
+
const selected = candidates[0];
|
|
1502
|
+
if (!selected)
|
|
829
1503
|
return undefined;
|
|
830
|
-
const
|
|
1504
|
+
const where = selected.filterValue
|
|
1505
|
+
? [`WHERE ${sqlIdentifier(selected.entityColumn)} = ${sqlStringLiteral(selected.filterValue)}`]
|
|
1506
|
+
: [];
|
|
1507
|
+
const columns = selected.selectColumns.map((column) => ` ${sqlIdentifier(column)}`).join(',\n');
|
|
1508
|
+
const text = [
|
|
1509
|
+
`Prepared a review-required profile query for ${selected.filterValue ?? entityTexts[0]} from ${selected.table.relation}.`,
|
|
1510
|
+
selected.usedSampleValue
|
|
1511
|
+
? `The entity filter uses an inspected value match on ${selected.entityColumn}.`
|
|
1512
|
+
: `The entity filter uses the likely entity column ${selected.entityColumn} from inspected schema metadata.`,
|
|
1513
|
+
'This result is uncertified until reviewed and promoted.',
|
|
1514
|
+
].join(' ');
|
|
831
1515
|
return {
|
|
832
|
-
text
|
|
1516
|
+
text,
|
|
833
1517
|
sql: [
|
|
834
1518
|
'SELECT',
|
|
835
|
-
|
|
836
|
-
`
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
`JOIN ${sqlRelation(customers.relation)} AS c ON o.${sqlIdentifier(orderCustomerId)} = c.${sqlIdentifier(customerId)}`,
|
|
840
|
-
`GROUP BY c.${sqlIdentifier(customerName)}`,
|
|
841
|
-
'ORDER BY lifetime_spend DESC, orders DESC',
|
|
842
|
-
'LIMIT 10',
|
|
1519
|
+
columns,
|
|
1520
|
+
`FROM ${sqlRelation(selected.table.relation)}`,
|
|
1521
|
+
...where,
|
|
1522
|
+
'LIMIT 50',
|
|
843
1523
|
].join('\n'),
|
|
844
1524
|
viz: 'table',
|
|
845
1525
|
};
|
|
846
1526
|
}
|
|
1527
|
+
function profileTableCandidate(table, question, followUp, entityTexts) {
|
|
1528
|
+
if (table.columns.length === 0)
|
|
1529
|
+
return undefined;
|
|
1530
|
+
const matchedFilters = matchedEntityFiltersForQuestion(table, question, followUp);
|
|
1531
|
+
const sampled = matchedFilters.find((filter) => entityTexts.some((entity) => normalizeForEntityMatch(entity) === normalizeForEntityMatch(filter.value))) ?? matchedFilters[0];
|
|
1532
|
+
const entityColumn = sampled?.column ?? pickEntityProfileColumn(table, question);
|
|
1533
|
+
if (!entityColumn)
|
|
1534
|
+
return undefined;
|
|
1535
|
+
const filterValue = sampled?.value ?? entityTexts[0];
|
|
1536
|
+
if (!filterValue)
|
|
1537
|
+
return undefined;
|
|
1538
|
+
const selectColumns = orderedProfileColumns(table, entityColumn, question);
|
|
1539
|
+
if (selectColumns.length === 0)
|
|
1540
|
+
return undefined;
|
|
1541
|
+
const questionTokens = meaningfulTokens(question);
|
|
1542
|
+
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1543
|
+
const columnTokens = meaningfulTokens(table.columns.map((column) => column.name).join(' '));
|
|
1544
|
+
const overlap = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token)).length;
|
|
1545
|
+
const profileSignal = selectColumns.filter((column) => isProfileMeasureColumn(column, table)).length;
|
|
1546
|
+
const score = (sampled ? 80 : 42) +
|
|
1547
|
+
overlap * 6 +
|
|
1548
|
+
profileSignal * 3 +
|
|
1549
|
+
(/\b(player|customer|account|user|member|person|entity)\b/i.test(table.name) ? 8 : 0);
|
|
1550
|
+
return {
|
|
1551
|
+
table,
|
|
1552
|
+
entityColumn,
|
|
1553
|
+
filterValue,
|
|
1554
|
+
selectColumns,
|
|
1555
|
+
usedSampleValue: Boolean(sampled),
|
|
1556
|
+
score,
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
function isEntityProfileQuestion(question, contextPack) {
|
|
1560
|
+
const mode = contextPack?.questionPlan?.mode;
|
|
1561
|
+
if (mode === 'entity_profile')
|
|
1562
|
+
return true;
|
|
1563
|
+
return /\b(profile|overview|360|complete\s+(?:stats|statistics|view)|full\s+(?:stats|statistics|view)|all\s+(?:stats|statistics|metrics)|research|reserach)\b/i.test(question);
|
|
1564
|
+
}
|
|
1565
|
+
function entityMentionsForProfile(question, contextPack) {
|
|
1566
|
+
const values = [
|
|
1567
|
+
...(contextPack?.questionPlan?.entities ?? []).map((entity) => entity.text),
|
|
1568
|
+
...Array.from(question.matchAll(/\b(?:for|on|profile\s+for|research\s+on|reserach\s+on)\s+([A-Z][A-Za-z0-9&.'-]+(?:\s+[A-Z][A-Za-z0-9&.'-]+){0,5})/g))
|
|
1569
|
+
.map((match) => match[1] ?? ''),
|
|
1570
|
+
];
|
|
1571
|
+
if (isEntityProfileQuestion(question, contextPack)) {
|
|
1572
|
+
values.push(...Array.from(question.matchAll(/\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,5})\b/g)).map((match) => match[1] ?? ''));
|
|
1573
|
+
}
|
|
1574
|
+
return uniqueDrilldownStrings(values
|
|
1575
|
+
.map((value) => value.replace(/\b(profile|stats|statistics|details|summary)\b.*$/i, '').trim())
|
|
1576
|
+
.filter((value) => value.length > 1 && !/^(Can You|Could You|Tell Me|Show Me)$/i.test(value)))
|
|
1577
|
+
.slice(0, 6);
|
|
1578
|
+
}
|
|
1579
|
+
function pickEntityProfileColumn(table, question) {
|
|
1580
|
+
const lower = question.toLowerCase();
|
|
1581
|
+
const domainHints = [
|
|
1582
|
+
/\bplayers?\b/.test(lower) ? 'player' : '',
|
|
1583
|
+
/\bcustomers?\b/.test(lower) ? 'customer' : '',
|
|
1584
|
+
/\baccounts?\b/.test(lower) ? 'account' : '',
|
|
1585
|
+
/\busers?\b/.test(lower) ? 'user' : '',
|
|
1586
|
+
/\bmembers?\b/.test(lower) ? 'member' : '',
|
|
1587
|
+
/\bteams?\b/.test(lower) ? 'team' : '',
|
|
1588
|
+
/\bproducts?\b/.test(lower) ? 'product' : '',
|
|
1589
|
+
].filter(Boolean);
|
|
1590
|
+
const preferred = [
|
|
1591
|
+
...domainHints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
|
|
1592
|
+
'name',
|
|
1593
|
+
'full_name',
|
|
1594
|
+
'display_name',
|
|
1595
|
+
'title',
|
|
1596
|
+
];
|
|
1597
|
+
const direct = findSchemaColumn(table, preferred);
|
|
1598
|
+
if (direct)
|
|
1599
|
+
return direct;
|
|
1600
|
+
const scored = table.columns
|
|
1601
|
+
.map((column) => {
|
|
1602
|
+
const name = column.name.toLowerCase();
|
|
1603
|
+
let score = 0;
|
|
1604
|
+
if (name.endsWith('_name') || name === 'name')
|
|
1605
|
+
score += 30;
|
|
1606
|
+
if (/\b(name|title|email)\b/i.test(name.replace(/_/g, ' ')))
|
|
1607
|
+
score += 20;
|
|
1608
|
+
if (domainHints.some((hint) => name.includes(hint)))
|
|
1609
|
+
score += 16;
|
|
1610
|
+
if (/\b(id|key|code|date|time|amount|total|count|score|points|revenue)\b/i.test(name.replace(/_/g, ' ')))
|
|
1611
|
+
score -= 12;
|
|
1612
|
+
return { column: column.name, score };
|
|
1613
|
+
})
|
|
1614
|
+
.filter((item) => item.score > 0)
|
|
1615
|
+
.sort((a, b) => b.score - a.score);
|
|
1616
|
+
return scored[0]?.column;
|
|
1617
|
+
}
|
|
1618
|
+
function orderedProfileColumns(table, entityColumn, question) {
|
|
1619
|
+
const scored = table.columns
|
|
1620
|
+
.map((column, index) => ({
|
|
1621
|
+
column: column.name,
|
|
1622
|
+
score: profileColumnScore(column, question, index),
|
|
1623
|
+
index,
|
|
1624
|
+
}))
|
|
1625
|
+
.sort((a, b) => b.score - a.score || a.index - b.index);
|
|
1626
|
+
const selected = uniqueDrilldownStrings([
|
|
1627
|
+
entityColumn,
|
|
1628
|
+
...scored.map((item) => item.column),
|
|
1629
|
+
]).slice(0, 24);
|
|
1630
|
+
return selected.length > 0 ? selected : [entityColumn];
|
|
1631
|
+
}
|
|
1632
|
+
function profileColumnScore(column, question, index) {
|
|
1633
|
+
const name = column.name.toLowerCase();
|
|
1634
|
+
const text = name.replace(/_/g, ' ');
|
|
1635
|
+
const lowerQuestion = question.toLowerCase();
|
|
1636
|
+
let score = Math.max(0, 12 - index * 0.1);
|
|
1637
|
+
if (isProfileMeasureColumn(column.name, { relation: '', name: '', columns: [column] }))
|
|
1638
|
+
score += 35;
|
|
1639
|
+
if (/\b(name|title|team|season|year|date|type|category|segment|status)\b/i.test(text))
|
|
1640
|
+
score += 18;
|
|
1641
|
+
if (/\b(id|key|code)\b/i.test(text))
|
|
1642
|
+
score += 2;
|
|
1643
|
+
for (const token of meaningfulTokens(lowerQuestion)) {
|
|
1644
|
+
if (meaningfulTokens(column.name).has(token))
|
|
1645
|
+
score += 12;
|
|
1646
|
+
}
|
|
1647
|
+
if (column.description && meaningfulTokens(column.description).size > 0)
|
|
1648
|
+
score += 4;
|
|
1649
|
+
return score;
|
|
1650
|
+
}
|
|
1651
|
+
function isProfileMeasureColumn(columnName, table) {
|
|
1652
|
+
const column = table.columns.find((item) => namesEqualLoose(item.name, columnName));
|
|
1653
|
+
const name = columnName.toLowerCase();
|
|
1654
|
+
const text = name.replace(/_/g, ' ');
|
|
1655
|
+
return /\b(total|count|avg|average|sum|rate|score|points|pts|rebounds?|assists?|steals?|blocks?|turnovers?|minutes?|games?|wins?|losses?|revenue|amount|spend|orders?|quantity|margin|cost|value|stat)\b/i.test(text)
|
|
1656
|
+
|| /\b(INT|INTEGER|BIGINT|DECIMAL|DOUBLE|FLOAT|NUMBER|NUMERIC|REAL)\b/i.test(column?.type ?? '');
|
|
1657
|
+
}
|
|
847
1658
|
function buildMatchedEntityDrilldownProposal(input) {
|
|
848
1659
|
if (input.intent !== 'entity_drilldown' && input.followUp?.kind !== 'drilldown')
|
|
849
1660
|
return undefined;
|
|
@@ -896,15 +1707,21 @@ function buildMatchedEntityDrilldownProposal(input) {
|
|
|
896
1707
|
}
|
|
897
1708
|
function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
898
1709
|
const byRelation = new Map();
|
|
1710
|
+
const relationSelections = selectedRelationLookup(contextPack);
|
|
899
1711
|
for (const table of schemaContext) {
|
|
1712
|
+
const selection = relationSelectionFor(table.relation, relationSelections);
|
|
900
1713
|
byRelation.set(normalizeRelationKey(table.relation), {
|
|
901
1714
|
...table,
|
|
902
1715
|
columns: table.columns.map((column) => ({ ...column, sampleValues: column.sampleValues?.slice() })),
|
|
1716
|
+
selectionRank: selection?.rank ?? table.selectionRank,
|
|
1717
|
+
selectionScore: selection?.score ?? table.selectionScore,
|
|
1718
|
+
selectionReason: selection?.reason ?? table.selectionReason,
|
|
903
1719
|
});
|
|
904
1720
|
}
|
|
905
1721
|
for (const relation of contextPack?.allowedSqlContext?.relations ?? []) {
|
|
906
1722
|
const key = normalizeRelationKey(relation.relation);
|
|
907
1723
|
const existing = byRelation.get(key);
|
|
1724
|
+
const selection = relationSelectionFor(relation.relation, relationSelections);
|
|
908
1725
|
if (!existing) {
|
|
909
1726
|
byRelation.set(key, {
|
|
910
1727
|
relation: relation.relation,
|
|
@@ -916,9 +1733,15 @@ function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
|
916
1733
|
sampleValues: column.sampleValues?.slice(),
|
|
917
1734
|
})),
|
|
918
1735
|
source: relation.source,
|
|
1736
|
+
selectionRank: selection?.rank,
|
|
1737
|
+
selectionScore: selection?.score,
|
|
1738
|
+
selectionReason: selection?.reason,
|
|
919
1739
|
});
|
|
920
1740
|
continue;
|
|
921
1741
|
}
|
|
1742
|
+
existing.selectionRank = selection?.rank ?? existing.selectionRank;
|
|
1743
|
+
existing.selectionScore = selection?.score ?? existing.selectionScore;
|
|
1744
|
+
existing.selectionReason = selection?.reason ?? existing.selectionReason;
|
|
922
1745
|
const columns = new Map(existing.columns.map((column) => [column.name.toLowerCase(), column]));
|
|
923
1746
|
for (const column of relation.columns) {
|
|
924
1747
|
const existingColumn = columns.get(column.name.toLowerCase());
|
|
@@ -941,6 +1764,23 @@ function schemaContextWithAllowedSqlContext(schemaContext, contextPack) {
|
|
|
941
1764
|
}
|
|
942
1765
|
return Array.from(byRelation.values());
|
|
943
1766
|
}
|
|
1767
|
+
function selectedRelationLookup(contextPack) {
|
|
1768
|
+
const lookup = new Map();
|
|
1769
|
+
for (const relation of contextPack?.retrievalDiagnostics?.selectedRelations ?? []) {
|
|
1770
|
+
for (const key of relationLookupKeys(relation.relation)) {
|
|
1771
|
+
lookup.set(key, relation);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
return lookup;
|
|
1775
|
+
}
|
|
1776
|
+
function relationSelectionFor(relation, lookup) {
|
|
1777
|
+
for (const key of relationLookupKeys(relation)) {
|
|
1778
|
+
const selection = lookup.get(key);
|
|
1779
|
+
if (selection)
|
|
1780
|
+
return selection;
|
|
1781
|
+
}
|
|
1782
|
+
return undefined;
|
|
1783
|
+
}
|
|
944
1784
|
function pickDrilldownTable(schemaContext, question, followUp) {
|
|
945
1785
|
const scored = schemaContext
|
|
946
1786
|
.map((table) => ({
|
|
@@ -1113,6 +1953,18 @@ function namesEqualLoose(a, b) {
|
|
|
1113
1953
|
function normalizeRelationKey(relation) {
|
|
1114
1954
|
return relation.replace(/["`]/g, '').replace(/\s*\.\s*/g, '.').toLowerCase().trim();
|
|
1115
1955
|
}
|
|
1956
|
+
function relationLookupKeys(relation) {
|
|
1957
|
+
const normalized = normalizeRelationKey(relation);
|
|
1958
|
+
const parts = normalized.split('.').filter(Boolean);
|
|
1959
|
+
const keys = new Set();
|
|
1960
|
+
if (normalized)
|
|
1961
|
+
keys.add(normalized);
|
|
1962
|
+
if (parts.length >= 2)
|
|
1963
|
+
keys.add(parts.slice(-2).join('.'));
|
|
1964
|
+
if (parts.length >= 1)
|
|
1965
|
+
keys.add(parts[parts.length - 1]);
|
|
1966
|
+
return Array.from(keys);
|
|
1967
|
+
}
|
|
1116
1968
|
function normalizeForEntityMatch(value) {
|
|
1117
1969
|
return value.toLowerCase().replace(/[^a-z0-9.%+-]+/g, ' ').replace(/\s+/g, ' ').trim();
|
|
1118
1970
|
}
|
|
@@ -1142,6 +1994,9 @@ function buildContextPackAwareProposal(input) {
|
|
|
1142
1994
|
return undefined;
|
|
1143
1995
|
if (!input.contextPack)
|
|
1144
1996
|
return undefined;
|
|
1997
|
+
const profileProposal = buildContextPackProfileProposal(input.question, input.contextPack);
|
|
1998
|
+
if (profileProposal)
|
|
1999
|
+
return profileProposal;
|
|
1145
2000
|
const lower = input.question.toLowerCase();
|
|
1146
2001
|
if (!/\b(least|lowest|fewest|bottom|min(?:imum)?)\b/.test(lower))
|
|
1147
2002
|
return undefined;
|
|
@@ -1162,6 +2017,86 @@ function buildContextPackAwareProposal(input) {
|
|
|
1162
2017
|
}
|
|
1163
2018
|
return undefined;
|
|
1164
2019
|
}
|
|
2020
|
+
function buildContextPackProfileProposal(question, contextPack) {
|
|
2021
|
+
const entityTexts = entityMentionsForProfile(question, contextPack);
|
|
2022
|
+
if (entityTexts.length === 0 || !isEntityProfileQuestion(question, contextPack))
|
|
2023
|
+
return undefined;
|
|
2024
|
+
const sources = [...(contextPack.allowedSqlContext?.sourceBlockSql ?? [])]
|
|
2025
|
+
.sort((a, b) => {
|
|
2026
|
+
const preferred = contextPack.routeDecision.certifiedApplicability?.objectKey;
|
|
2027
|
+
if (preferred && a.objectKey === preferred)
|
|
2028
|
+
return -1;
|
|
2029
|
+
if (preferred && b.objectKey === preferred)
|
|
2030
|
+
return 1;
|
|
2031
|
+
return 0;
|
|
2032
|
+
});
|
|
2033
|
+
for (const source of sources) {
|
|
2034
|
+
const shape = extractSimpleSelectShape(source.sql);
|
|
2035
|
+
if (!shape)
|
|
2036
|
+
continue;
|
|
2037
|
+
const entityColumn = pickEntityColumnFromSelectExpressions(shape.selectExpressions, question);
|
|
2038
|
+
if (!entityColumn)
|
|
2039
|
+
continue;
|
|
2040
|
+
const selectExpressions = uniqueSqlSelectExpressions(shape.selectExpressions).slice(0, 24);
|
|
2041
|
+
if (selectExpressions.length === 0)
|
|
2042
|
+
continue;
|
|
2043
|
+
const orderColumn = selectExpressions
|
|
2044
|
+
.map(selectExpressionOutputName)
|
|
2045
|
+
.find((column) => column && /\b(season|year|date|month|week|game_date|created_at)\b/i.test(column));
|
|
2046
|
+
const sql = [
|
|
2047
|
+
'SELECT',
|
|
2048
|
+
selectExpressions.map((expression) => ` ${expression}`).join(',\n'),
|
|
2049
|
+
`FROM ${shape.relation}`,
|
|
2050
|
+
`WHERE ${sqlIdentifier(entityColumn)} = ${sqlStringLiteral(entityTexts[0])}`,
|
|
2051
|
+
orderColumn ? `ORDER BY ${sqlIdentifier(orderColumn)} DESC` : '',
|
|
2052
|
+
'LIMIT 50',
|
|
2053
|
+
].filter(Boolean).join('\n');
|
|
2054
|
+
return {
|
|
2055
|
+
text: `Prepared a review-required profile query for ${entityTexts[0]} by using certified block "${source.name}" as SQL-shape context. This result is uncertified until reviewed and promoted.`,
|
|
2056
|
+
sql,
|
|
2057
|
+
viz: 'table',
|
|
2058
|
+
};
|
|
2059
|
+
}
|
|
2060
|
+
return undefined;
|
|
2061
|
+
}
|
|
2062
|
+
function pickEntityColumnFromSelectExpressions(expressions, question) {
|
|
2063
|
+
const outputNames = expressions
|
|
2064
|
+
.map(selectExpressionOutputName)
|
|
2065
|
+
.filter((value) => Boolean(value));
|
|
2066
|
+
const lower = question.toLowerCase();
|
|
2067
|
+
const hints = [
|
|
2068
|
+
/\bplayers?\b/.test(lower) ? 'player' : '',
|
|
2069
|
+
/\bcustomers?\b/.test(lower) ? 'customer' : '',
|
|
2070
|
+
/\baccounts?\b/.test(lower) ? 'account' : '',
|
|
2071
|
+
/\busers?\b/.test(lower) ? 'user' : '',
|
|
2072
|
+
/\bteams?\b/.test(lower) ? 'team' : '',
|
|
2073
|
+
/\bproducts?\b/.test(lower) ? 'product' : '',
|
|
2074
|
+
].filter(Boolean);
|
|
2075
|
+
const preferred = [
|
|
2076
|
+
...hints.flatMap((hint) => [`${hint}_name`, `${hint}_full_name`, hint]),
|
|
2077
|
+
'name',
|
|
2078
|
+
'full_name',
|
|
2079
|
+
'display_name',
|
|
2080
|
+
];
|
|
2081
|
+
for (const wanted of preferred) {
|
|
2082
|
+
const match = outputNames.find((name) => namesEqualLoose(name, wanted));
|
|
2083
|
+
if (match)
|
|
2084
|
+
return match;
|
|
2085
|
+
}
|
|
2086
|
+
return outputNames.find((name) => /(^|_)(name|title|email)$/.test(name.toLowerCase()));
|
|
2087
|
+
}
|
|
2088
|
+
function uniqueSqlSelectExpressions(expressions) {
|
|
2089
|
+
const seen = new Set();
|
|
2090
|
+
const unique = [];
|
|
2091
|
+
for (const expression of expressions) {
|
|
2092
|
+
const normalized = expression.replace(/\s+/g, ' ').trim().toLowerCase();
|
|
2093
|
+
if (!normalized || seen.has(normalized))
|
|
2094
|
+
continue;
|
|
2095
|
+
seen.add(normalized);
|
|
2096
|
+
unique.push(expression.replace(/\s+/g, ' ').trim());
|
|
2097
|
+
}
|
|
2098
|
+
return unique;
|
|
2099
|
+
}
|
|
1165
2100
|
function invertRankingSql(sql) {
|
|
1166
2101
|
const withoutTrailingSemicolon = sql.replace(/;\s*$/, '').trim();
|
|
1167
2102
|
const inverted = withoutTrailingSemicolon.replace(/\border\s+by\s+([\s\S]*?)(\blimit\b|$)/i, (match, orderExpr, limitKeyword) => {
|
|
@@ -1579,7 +2514,10 @@ function isAdHocAnalysisQuestion(question) {
|
|
|
1579
2514
|
const lower = question.toLowerCase();
|
|
1580
2515
|
if (isBusinessDefinitionQuestion(question))
|
|
1581
2516
|
return false;
|
|
1582
|
-
|
|
2517
|
+
const asksCountBreakdown = /\b(how many|number of|count of|count)\b/i.test(lower)
|
|
2518
|
+
&& /\b(by\s+[a-z][\w\s-]{1,40}|per|for each|group(?:ed)? by|split|segment)\b/i.test(lower);
|
|
2519
|
+
return asksCountBreakdown
|
|
2520
|
+
|| /\b(break\s*down|breakdown|drill\s*(?:down|into)|slice|segment|split|compare|versus|vs\.?|trend|over time|top|bottom|best|worst|highest|lowest|least|fewest|minimum|min|smallest|rank|ranking|performed better|better performing|why|what drove|driver|drivers|top movers?|changed?|change|dropped?|drop|decreased?|decrease|declined?|decline|increased?|increase|anomal(?:y|ies)|exceptions?|root cause|contribut(?:e|ed|ion)|variance|delta|by\s+[a-z][\w\s-]{1,40})\b/i.test(lower)
|
|
1583
2521
|
|| /\b(show|list|find|give)\b.+\b(account|accounts|customer|customers|product|products|order|orders|region|location|month|week|day|user|users)\b/i.test(lower);
|
|
1584
2522
|
}
|
|
1585
2523
|
function isFilteredEntityQuestion(question) {
|
|
@@ -1616,7 +2554,9 @@ function buildAnalysisPlan(input) {
|
|
|
1616
2554
|
const tokens = meaningfulTokens(input.question);
|
|
1617
2555
|
const dimensions = inferDimensions(input.question, input.selectedNodes, input.schemaContext);
|
|
1618
2556
|
const measures = inferMeasures(input.question, input.selectedNodes, input.schemaContext);
|
|
1619
|
-
const
|
|
2557
|
+
const candidateJoins = buildCandidateJoinPaths(input.schemaContext);
|
|
2558
|
+
const candidateTables = [...input.schemaContext].sort((a, b) => (a.selectionRank ?? Number.MAX_SAFE_INTEGER) - (b.selectionRank ?? Number.MAX_SAFE_INTEGER) ||
|
|
2559
|
+
a.relation.localeCompare(b.relation)).slice(0, 8).map((table) => ({
|
|
1620
2560
|
relation: table.relation,
|
|
1621
2561
|
columns: table.columns.slice(0, 16).map((col) => col.name),
|
|
1622
2562
|
reason: tableReason(table, tokens),
|
|
@@ -1635,6 +2575,7 @@ function buildAnalysisPlan(input) {
|
|
|
1635
2575
|
measures,
|
|
1636
2576
|
dimensions,
|
|
1637
2577
|
candidateTables,
|
|
2578
|
+
candidateJoins,
|
|
1638
2579
|
trustedContext,
|
|
1639
2580
|
assumptions: input.assumptions ?? [],
|
|
1640
2581
|
sql: input.sql,
|
|
@@ -1694,6 +2635,11 @@ function inferMeasures(question, selectedNodes, schemaContext) {
|
|
|
1694
2635
|
return Array.from(measures).slice(0, 6);
|
|
1695
2636
|
}
|
|
1696
2637
|
function tableReason(table, questionTokens) {
|
|
2638
|
+
if (table.selectionReason) {
|
|
2639
|
+
return table.selectionRank
|
|
2640
|
+
? `metadata rank ${table.selectionRank}: ${table.selectionReason}`
|
|
2641
|
+
: table.selectionReason;
|
|
2642
|
+
}
|
|
1697
2643
|
const tableTokens = meaningfulTokens([table.relation, table.name, table.description ?? ''].join(' '));
|
|
1698
2644
|
const columnTokens = meaningfulTokens(table.columns.map((col) => col.name).join(' '));
|
|
1699
2645
|
const matches = [...questionTokens].filter((token) => tableTokens.has(token) || columnTokens.has(token));
|