@finos/legend-lego 2.0.217 → 2.0.218

package/lib/legend-ai/stores/LegendAIChatProcessors.js

@@ -17,11 +17,11 @@ import { assertErrorThrown, guaranteeNonNullable, isNonNullable, uuid, } from '@
 import { LegendAISelfHealKind, LegendAIQuestionIntent, LegendAIResponseOutcome, LegendAIThinkingStepStatus, LegendAIMessageRole, LegendAIErrorType, LegendAIExecutionTimeoutError, LegendAIServiceError, LegendAIUnsupportedEngineShapeError, TDSServiceSourceType, buildColumnDefsFromNames, LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID, LEGEND_AI_ALTERNATE_ROOT_ACTION_ID, LEGEND_AI_FEEDBACK_PROMPT, getTodayISO, } from '../LegendAITypes.js';
 import { LegendAIJudgeVerdict, } from '../LegendAI_LegendApplicationPlugin_Extension.js';
 import { extractElementNameFromPath, } from '@finos/legend-graph';
-import { buildEnrichedBusinessContext, buildModelContextEnrichmentText, buildModelCatalogText, buildDataQueryApproachText, findBestAlternateRoot, rankEntities, relaxExactStringFilters, extractFilteredColumns, buildProbedValueHints, splitIdentifierTokens, tokenizeText, } from '../LegendAIDocEnrichment.js';
+import { buildEnrichedBusinessContext, buildModelContextEnrichmentText, buildModelCatalogText, buildDataQueryApproachText, findBestAlternateRoot, rankEntities, relaxExactStringFilters, extractFilteredColumns, buildProbedValueHints, buildPriorSqlFailureHints, splitIdentifierTokens, tokenizeText, } from '../LegendAIDocEnrichment.js';
 import { isFuzzyMatch, preFilterServicesByRelevance, } from '../LegendAIServiceRetrieval.js';
-import { isNumericColumn, isStringColumn, } from '../components/LegendAIChatHelpers.js';
-import { HAS_LIMIT_PATTERN, AP_CALL_PATTERN, servicePId, pureRelationColumnRef, } from './LegendAISqlHelpers.js';
-import { buildCrossJoinZeroRowExplanation, buildJoinablePairSuggestions, } from './LegendAIJoinAnalysis.js';
+import { isNumericColumn, isStringColumn, isStringTypedColumn, } from '../components/LegendAIChatHelpers.js';
+import { buildColumnByNameIndex, accessPointName, HAS_LIMIT_PATTERN, accessPointCalls, servicePId, pureRelationColumnRef, extractJoinKeyColumns, sharedColumnNames, } from './LegendAISqlHelpers.js';
+import { buildCrossJoinZeroRowExplanation, buildJoinablePairSuggestions, detectDisjointJoinUniverses, resolveJoinedAccessPoints, buildDisjointJoinMessage, previewValues, } from './LegendAIJoinAnalysis.js';
 import { boundCrossAccessPointJoinDrivingSide, sanitizeJoinDuplicateColumns, sanitizeJoinOrderBy, sanitizeJoinSameKeyColumns, wrapBareJoinAccessPoints, } from './LegendAISqlJoinSanitizers.js';
 const MAX_ERROR_MESSAGE_LENGTH = 500;
 const MAX_DISPLAYED_ERROR_LENGTH = 4000;
@@ -32,6 +32,10 @@ const ANALYSIS_TIMEOUT_MS = 15_000;
 const ORCHESTRATOR_GENERATION_TIMEOUT_MS = 120_000;
 const DISTINCT_PROBE_ROW_LIMIT = 5000;
 const MAX_RECOVERED_FILTER_COLUMNS = 4;
+export const JOIN_OVERLAP_PROBE_LIMIT = 200;
+const JOIN_OVERLAP_CONCLUSIVE_MAX = JOIN_OVERLAP_PROBE_LIMIT - 1;
+const MAX_JOIN_OVERLAP_PROBE_KEYS = 4;
+const JOIN_OVERLAP_PROBE_TIMEOUT_MS = 15_000;
 const EXECUTION_TIMEOUT_MS = 300_000;
 const ANALYSIS_PREVIEW_ROW_LIMIT = 3;
 const ANALYSIS_PREVIEW_VALUE_LIMIT = 40;
@@ -42,11 +46,28 @@ const LITERAL_COL_PATTERN = /,\s*'[^']*'\s+AS\s+(?:"[^"]+"|[a-z]\w*)/gi;
 const SERVICE_CALL_PATTERN = /\bservice\s*\([^()]*\)/gi;
 const DEFAULT_SAFETY_LIMIT = 1000;
 const HAS_AGGREGATION_PATTERN = /\bGROUP\s+BY\b|\b(?:COUNT|SUM|AVG|MIN|MAX)\s*\(|\bSELECT\s+DISTINCT\b|\bHAVING\b|\bQUALIFY\b/i;
+const OUTER_JOIN_PATTERN = /\b(?:LEFT|RIGHT|FULL)\s+(?:OUTER\s+)?JOIN\b/i;
+const DISJUNCTIVE_PREDICATE_PATTERN = /\bOR\b/i;
+const SET_OPERATION_PATTERN = /\b(?:UNION|INTERSECT|EXCEPT)\b/i;
 const MAX_SERVICES_FOR_LLM_SELECTION = 30;
 const VALUE_GROUNDING_TIME_BUDGET_MS = 4000;
 const AP_SQL_EXECUTION_TIMEOUT_MS = 90_000;
 const AP_SQL_EXECUTION_TIMEOUT_MESSAGE = 'This query is scanning very large access-point feeds and is taking too long to return. Narrow it with a date or entity filter, join on the full key (including the date), or use a *_LATEST snapshot access point.';
 const ORCHESTRATOR_FALLBACK_LABEL = 'Try Legend AI Orchestrator';
+// The single orchestrator fallback action, so every dead-end path offers the
+// same button and threads any prior SQL failure the same way.
+export function buildOrchestratorFallbackAction(failure) {
+    return {
+        label: ORCHESTRATOR_FALLBACK_LABEL,
+        actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
+        ...(failure?.failedSql === undefined
+            ? {}
+            : { failedSql: failure.failedSql }),
+        ...(failure?.failedReason === undefined
+            ? {}
+            : { failedReason: failure.failedReason }),
+    };
+}
 const SQL_GENERATION_FAILURE_WITH_ORCHESTRATOR = 'SQL generation could not handle this query. You can try the Legend AI Orchestrator to generate a Pure query instead.';
 const SQL_GENERATION_FAILURE_NO_ORCHESTRATOR = 'SQL generation could not handle this query. Try rephrasing your question.';
 const SERVICE_PARAM_DATE_LIKE_PATTERNS = [
@@ -91,29 +112,32 @@ function deduplicateColumns(columns) {
         return count === 0 ? col : `${col}_${count + 1}`;
     });
 }
+export function createAssistantMessage() {
+    return {
+        id: uuid(),
+        role: LegendAIMessageRole.ASSISTANT,
+        thinkingSteps: [],
+        sql: null,
+        textAnswer: null,
+        dataContext: null,
+        gridData: null,
+        error: null,
+        errorType: null,
+        sqlGenTime: null,
+        execTime: null,
+        thinkingDuration: null,
+        isProcessing: true,
+        isExecuting: false,
+        suggestedQueries: [],
+        fallbackAction: null,
+        queriedAccessPointGroups: [],
+        queriedAccessPoints: [],
+    };
+}
 export function createMessagePair(text) {
     return [
         { id: uuid(), role: LegendAIMessageRole.USER, text },
-        {
-            id: uuid(),
-            role: LegendAIMessageRole.ASSISTANT,
-            thinkingSteps: [],
-            sql: null,
-            textAnswer: null,
-            dataContext: null,
-            gridData: null,
-            error: null,
-            errorType: null,
-            sqlGenTime: null,
-            execTime: null,
-            thinkingDuration: null,
-            isProcessing: true,
-            isExecuting: false,
-            suggestedQueries: [],
-            fallbackAction: null,
-            queriedAccessPointGroups: [],
-            queriedAccessPoints: [],
-        },
+        createAssistantMessage(),
     ];
 }
 export function updateLastAssistant(setMessages, updater) {
@@ -236,7 +260,7 @@ export function filterHistoryForAccessPoints(history, currentAccessPoints) {
 function orderedAccessPointsFromSql(sql) {
     const ordered = [];
     const seen = new Set();
-    for (const match of sql.matchAll(AP_CALL_PATTERN)) {
+    for (const match of accessPointCalls(sql)) {
         const path = match.groups?.pId ?? '';
         const dotIdx = path.lastIndexOf('.');
         const apName = dotIdx === -1 ? path : path.slice(dotIdx + 1);
@@ -247,6 +271,8 @@ function orderedAccessPointsFromSql(sql) {
     }
     return ordered;
 }
+const MAX_HISTORY_ANSWER_LENGTH = 1500;
+const MAX_CONVERSATION_HISTORY_TURNS = 10;
 function buildTurnFromAssistant(userText, asstMsg) {
     if (asstMsg.sql) {
         const turn = {
@@ -274,7 +300,7 @@ function buildTurnFromAssistant(userText, asstMsg) {
     if (asstMsg.textAnswer) {
         return {
             question: userText,
-            sql: asstMsg.textAnswer,
+            sql: asstMsg.textAnswer.slice(0, MAX_HISTORY_ANSWER_LENGTH),
             intent: LegendAIQuestionIntent.METADATA,
         };
     }
@@ -309,6 +335,10 @@ export function classifyResponseOutcome(message) {
     }
     return LegendAIResponseOutcome.NO_ANSWER;
 }
+/**
+ * Pairs each question with the assistant turn that answered it, keeping only
+ * the most recent turns so prompts do not grow with conversation length.
+ */
 export function buildConversationHistory(messages) {
     const history = [];
     let i = 0;
@@ -327,7 +357,7 @@ export function buildConversationHistory(messages) {
             i += 1;
         }
     }
-    return history;
+    return history.slice(-MAX_CONVERSATION_HISTORY_TURNS);
 }
 function formatServiceParams(services) {
     return services.flatMap((s) => {
@@ -690,18 +720,103 @@ export function attachMetadataOverview(setMessages, metadataOverview) {
         };
     });
 }
+const PRODUCT_OVERVIEW_PATTERNS = [
+    /\bwhat\s+does\b.*\b(?:offer|provide|contain|have|include|cover)\b/,
+    /\b(?:describe|summarize|overview|summary)\b/,
+    /\bwhat\s+(?:access\s*points?|services?|data|datasets?)\b/,
+    /\bwhat\s+can\s+(?:i|we|you)\b/,
+    /\btell\s+me\s+(?:about|more)\b/,
+];
+const LISTING_VERB_PATTERN = /\b(?:list|show|what\s+are)\b/;
+const LISTING_NOUN_PATTERN = /\b(?:access\s*points?|services?|endpoints?|datasets?|tables?)\b/;
+export function isProductOverviewQuestion(question) {
+    const q = question.toLowerCase();
+    return (PRODUCT_OVERVIEW_PATTERNS.some((pattern) => pattern.test(q)) ||
+        (LISTING_VERB_PATTERN.test(q) && LISTING_NOUN_PATTERN.test(q)));
+}
+const CATALOG_MIN_SERVICES = 10;
+const CATALOG_MAX_ENTRIES = 200;
+// Buckets services under their access point group, groups in title order.
+function groupServicesByAccessPointGroup(services) {
+    const byGroup = new Map();
+    for (const svc of services) {
+        const group = svc.accessPointGroupTitle ?? '';
+        const existing = byGroup.get(group);
+        if (existing) {
+            existing.push(svc);
+        }
+        else {
+            byGroup.set(group, [svc]);
+        }
+    }
+    return Array.from(byGroup.entries()).sort((a, b) => a[0].localeCompare(b[0]));
+}
+// Renders each group as a heading followed by its services, up to the cap.
+function renderCatalogEntries(groups) {
+    const lines = [];
+    let rendered = 0;
+    for (const [group, entries] of groups) {
+        if (rendered >= CATALOG_MAX_ENTRIES) {
+            break;
+        }
+        if (group) {
+            lines.push(`\n### ${group}`);
+        }
+        for (const svc of [...entries].sort((a, b) => a.title.localeCompare(b.title))) {
+            if (rendered >= CATALOG_MAX_ENTRIES) {
+                break;
+            }
+            const count = svc.columns.length;
+            lines.push(`- ${svc.title} (${count} column${count === 1 ? '' : 's'})`);
+            rendered += 1;
+        }
+    }
+    return { lines, rendered };
+}
+// Deterministic, grouped listing of a product's access points/services, so an
+// overview answer on a large product is never cut off by the LLM's token budget.
+export function buildAccessPointCatalog(services) {
+    const noun = services.some((s) => s.sourceType === TDSServiceSourceType.ACCESS_POINT)
+        ? 'access points'
+        : 'services';
+    const { lines, rendered } = renderCatalogEntries(groupServicesByAccessPointGroup(services));
+    const omitted = services.length - rendered;
+    if (omitted > 0) {
+        lines.push(`\n- ...and ${omitted} more (ask about a specific group to see them).`);
+    }
+    const heading = omitted > 0
+        ? `${rendered} of ${services.length}`
+        : `All ${services.length}`;
+    return [`## ${heading} ${noun}`, ...lines].join('\n');
+}
+const MAX_METADATA_PROMPT_SERVICES = 30;
+/**
+ * Ranks the access points against the question so a product with hundreds of
+ * them contributes a bounded, relevant slice of the metadata prompt.
+ */
+function selectMetadataPromptServices(question, services) {
+    if (!services || services.length <= MAX_METADATA_PROMPT_SERVICES) {
+        return services;
+    }
+    return preFilterServicesByRelevance(question, services, MAX_METADATA_PROMPT_SERVICES);
+}
 export async function handleMetadataQuestion(question, metadata, context, startTime, hasQueryableServices, services, modelContextEnrichment) {
     const { config, plugin, history, setMessages } = context;
     addThinkingStep(setMessages, 'Answering from product metadata...');
-    const metadataPromptText = plugin.buildMetadataPrompt(question, metadata, history, services, modelContextEnrichment);
+    const metadataPromptText = plugin.buildMetadataPrompt(question, metadata, history, selectMetadataPromptServices(question, services), modelContextEnrichment);
     const rawAnswer = await plugin.callLLM(metadataPromptText, config);
     const { answer, suggestedQueries: parsedSuggestions } = parseSuggestedQueries(rawAnswer);
     const suggestedQueries = hasQueryableServices === false && !config.orchestratorUrl
         ? []
         : parsedSuggestions;
+    const textAnswer = services !== undefined &&
+        services.length >= CATALOG_MIN_SERVICES &&
+        isProductOverviewQuestion(question)
+        ? `${answer}\n\n${buildAccessPointCatalog(services)}`
+        : answer;
     completeThinkingSteps(setMessages);
     updateLastAssistant(setMessages, () => ({
-        textAnswer: answer,
+        textAnswer,
         suggestedQueries,
         isProcessing: false,
         thinkingDuration: elapsedSeconds(startTime),
@@ -1205,7 +1320,7 @@ async function reportOrchestratorExecutionResult(params, context) {
         }));
     }
 }
-export async function processQuestionViaOrchestrator(question, dataProductCoordinates, metadata, context, pureExecutionContext, preResolvedEntities, modelContext) {
+export async function processQuestionViaOrchestrator(question, dataProductCoordinates, metadata, context, pureExecutionContext, preResolvedEntities, modelContext, priorFailure) {
     const { config, plugin, setMessages } = context;
     const startTime = Date.now();
     try {
@@ -1214,9 +1329,26 @@ export async function processQuestionViaOrchestrator(question, dataProductCoordi
             addThinkingStep(setMessages, `Found ${resolvedEntities.relatedEntities.length} related entities`);
         }
         addThinkingStep(setMessages, 'Generating Legend query via orchestrator...');
-        const enrichedContext = modelContext
+        const baseContext = modelContext
             ? buildEnrichedBusinessContext(question, resolvedEntities.rootEntity, resolvedEntities.relatedEntities, modelContext)
             : undefined;
+        const priorFailureHints = priorFailure
+            ? buildPriorSqlFailureHints(priorFailure.failedSql, priorFailure.failedReason)
+            : [];
+        const enrichedContext = priorFailureHints.length > 0
+            ? {
+                naturalLanguageQuery: question,
+                ...baseContext,
+                businessContextMatch: {
+                    ...baseContext?.businessContextMatch,
+                    additionalNlModelContext: [
+                        ...(baseContext?.businessContextMatch
+                            ?.additionalNlModelContext ?? []),
+                        ...priorFailureHints,
+                    ],
+                },
+            }
+            : baseContext;
         const orchestratorResponse = await withTimeout(plugin.generateQueryViaOrchestrator({
             user_question: question,
             semantic_search_resolution_details: {
@@ -1516,9 +1648,16 @@ export function ensureSafeLimit(sql, limit = DEFAULT_SAFETY_LIMIT) {
     }
     return `${sql.trimEnd()}\nLIMIT ${limit}`;
 }
+/**
+ * The one place generated SQL text is rewritten, working around engine
+ * transpiler limits on joins. Every step no-ops on a shape it cannot fix.
+ */
 function prepareSafeSql(sql, services) {
-    const joined = sanitizeLiteralColumns(sanitizeJoinSameKeyColumns(sanitizeJoinOrderBy(sanitizeJoinDuplicateColumns(sql, services)), services));
-    const wrapped = wrapBareJoinAccessPoints(joined, services);
+    const deduped = sanitizeJoinDuplicateColumns(sql, services);
+    const ordered = sanitizeJoinOrderBy(deduped);
+    const keyed = sanitizeJoinSameKeyColumns(ordered, services);
+    const literals = sanitizeLiteralColumns(keyed);
+    const wrapped = wrapBareJoinAccessPoints(literals, services);
     const bounded = boundCrossAccessPointJoinDrivingSide(wrapped);
     const safeSql = ensureSafeLimit(ensureDateParameters(bounded, services));
     const unsupported = detectUnsupportedEnginePattern(safeSql);
@@ -1670,10 +1809,9 @@ function handleSqlGenerationFailure(setMessages, startTime, hasOrchestratorFallb
         updateLastAssistant(setMessages, () => ({
             textAnswer: orchestratorMessage,
             suggestedQueries: suggestions,
-            fallbackAction: {
-                label: ORCHESTRATOR_FALLBACK_LABEL,
-                actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
-            },
+            fallbackAction: buildOrchestratorFallbackAction({
+                failedReason: orchestratorMessage,
+            }),
             isProcessing: false,
             thinkingDuration: elapsedSeconds(startTime),
         }));
@@ -1688,7 +1826,7 @@ function handleSqlGenerationFailure(setMessages, startTime, hasOrchestratorFallb
     }
 }
 async function reportQueryResults(report, metadata, context, startTime, hasOrchestratorFallback) {
-    const { currentSql, sqlResult, question, services, allAccessPoints } = report;
+    const { currentSql, sqlResult, question, services, allAccessPoints, zeroRowExplanation, } = report;
     const { config, plugin, setMessages } = context;
     if (sqlResult.rows.length > 0) {
         const columns = deduplicateColumns(sqlResult.columns);
@@ -1723,13 +1861,16 @@ async function reportQueryResults(report, metadata, context, startTime, hasOrche
         }
     }
     else {
-        addThinkingStep(setMessages, 'Query returned 0 rows after correction attempts.');
-        const joinExplanation = buildCrossJoinZeroRowExplanation(currentSql, services);
+        addThinkingStep(setMessages, zeroRowExplanation === undefined
+            ? 'Query returned 0 rows after correction attempts.'
+            : 'No overlapping join keys — the query would return 0 rows.');
+        const joinExplanation = zeroRowExplanation ??
+            buildCrossJoinZeroRowExplanation(currentSql, services);
         const joinSuggestions = joinExplanation === undefined
             ? []
             : buildJoinablePairSuggestions(services, allAccessPoints ?? services);
         const llmAnalysis = await withTimeout(plugin
-            .buildNoResultsFallback(question, currentSql, metadata, config, joinExplanation)
+            .buildNoResultsFallback(question, currentSql, metadata, config, joinExplanation, services)
             .catch((error) => {
             assertErrorThrown(error);
             return undefined;
@@ -1738,18 +1879,17 @@ async function reportQueryResults(report, metadata, context, startTime, hasOrche
         const suggestions = joinSuggestions.length > 0
             ? joinSuggestions
             : (llmAnalysis?.suggestedQueries ?? []);
+        const zeroRowReason = llmAnalysis?.summary ?? joinExplanation ?? buildZeroRowMessage(services);
         const fallback = hasOrchestratorFallback
             ? {
-                fallbackAction: {
-                    label: ORCHESTRATOR_FALLBACK_LABEL,
-                    actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
-                },
+                fallbackAction: buildOrchestratorFallbackAction({
+                    failedSql: currentSql,
+                    failedReason: zeroRowReason,
+                }),
             }
             : {};
         updateLastAssistant(setMessages, () => ({
-            textAnswer: llmAnalysis?.summary ??
-                joinExplanation ??
-                buildZeroRowMessage(services),
+            textAnswer: zeroRowReason,
             ...(suggestions.length > 0 ? { suggestedQueries: suggestions } : {}),
             ...fallback,
             isProcessing: false,
@@ -1864,11 +2004,11 @@ export function applyMultiTurnBias(services, history) {
     }
     // Sort so that previously-used APs come first, preserving relative order
     const biased = [...services].sort((a, b) => {
-        const aUsed = previousAPs.has(a.pattern.replace(/^\//, '').toLowerCase()) ||
+        const aUsed = previousAPs.has(accessPointName(a).toLowerCase()) ||
             previousAPs.has(a.title.toLowerCase())
             ? 1
             : 0;
-        const bUsed = previousAPs.has(b.pattern.replace(/^\//, '').toLowerCase()) ||
+        const bUsed = previousAPs.has(accessPointName(b).toLowerCase()) ||
             previousAPs.has(b.title.toLowerCase())
             ? 1
             : 0;
@@ -2128,43 +2268,37 @@ function buildAccessPointColumnIndex(services) {
     }
     return index;
 }
-// Recovers a zero-row query by re-probing each WHERE-filter column's real distinct
-// values and rewriting equality/IN literals that don't match (casing/spelling).
-async function attemptDistinctValueRecovery(currentSql, selectedServices, dataProductCoordinates, context) {
+// Builds a memoized distinct-value prober for a query's columns: access-point
+// columns resolve via a relation query, service columns via SELECT DISTINCT.
+export function createColumnValueProbe(services, dataProductCoordinates, fromClause, isAccessPointProbe, context, options) {
     const { plugin, config, setMessages } = context;
-    const isAccessPointProbe = dataProductCoordinates !== undefined &&
-        selectedServices.some((s) => s.sourceType === TDSServiceSourceType.ACCESS_POINT);
+    const rowLimit = options?.rowLimit ?? DISTINCT_PROBE_ROW_LIMIT;
+    const maxColumns = options?.maxColumns ?? MAX_RECOVERED_FILTER_COLUMNS;
     const accessPointByColumn = isAccessPointProbe
-        ? buildAccessPointColumnIndex(selectedServices)
+        ? buildAccessPointColumnIndex(services)
         : new Map();
-    const fromClause = isAccessPointProbe
-        ? undefined
-        : extractFromClause(currentSql);
-    if (!isAccessPointProbe && fromClause === undefined) {
-        return undefined;
-    }
     const probeAccessPointColumn = async (column) => {
         const accessPoint = accessPointByColumn.get(column.toLowerCase());
         const pId = accessPoint ? servicePId(accessPoint) : undefined;
         if (pId === undefined) {
             return { columns: [], rows: [] };
         }
-        const relationQuery = `#P{${pId}}#->select(~[${pureRelationColumnRef(column)}])->distinct()->take(${DISTINCT_PROBE_ROW_LIMIT})`;
+        const relationQuery = `#P{${pId}}#->select(~[${pureRelationColumnRef(column)}])->distinct()->take(${rowLimit})`;
         return plugin.executeLakehouseRelationQuery(relationQuery, guaranteeNonNullable(dataProductCoordinates), config);
     };
     const probedValues = new Map();
-    const probeColumnValues = async (column) => {
+    return async (column) => {
         if (probedValues.has(column)) {
             return probedValues.get(column);
         }
-        if (probedValues.size >= MAX_RECOVERED_FILTER_COLUMNS) {
+        if (probedValues.size >= maxColumns) {
             return undefined;
         }
         let values;
         try {
             const probe = isAccessPointProbe
                 ? await probeAccessPointColumn(column)
-                : await executeSqlForServices(`SELECT DISTINCT "${column}" ${fromClause} LIMIT ${DISTINCT_PROBE_ROW_LIMIT}`, selectedServices, dataProductCoordinates, plugin, config);
+                : await executeSqlForServices(`SELECT DISTINCT "${column}" ${fromClause} LIMIT ${rowLimit}`, services, dataProductCoordinates, plugin, config);
             values = probe.rows
                 .map((row) => row[column])
                 .filter(isNonNullable)
@@ -2172,12 +2306,92 @@ async function attemptDistinctValueRecovery(currentSql, selectedServices, dataPr
         }
         catch (probeError) {
             assertErrorThrown(probeError);
-            addThinkingStep(setMessages, `Could not probe values for "${column}"`);
+            if (options?.silent !== true) {
+                addThinkingStep(setMessages, `Could not probe values for "${column}"`);
+            }
             values = undefined;
         }
         probedValues.set(column, values);
         return values;
     };
+}
+// An empty inner join yields no rows, but an outer join, an aggregate, a
+// disjunctive predicate or a set operation can still return some.
+export function canSkipExecutionOnEmptyJoin(sql) {
+    return (!OUTER_JOIN_PATTERN.test(sql) &&
+        !HAS_AGGREGATION_PATTERN.test(sql) &&
+        !DISJUNCTIVE_PREDICATE_PATTERN.test(sql) &&
+        !SET_OPERATION_PATTERN.test(sql));
+}
+/**
+ * Reports the join key whose values do not overlap, so a two access point join
+ * can be explained instead of run. Text keys only, because dates and numbers
+ * serialize differently on either side and would read as disjoint when they
+ * are not, and only for exactly two access points, because a key belonging to
+ * another pair would otherwise be tested against the wrong relations.
+ */
+export async function probeJoinOverlap(sql, services, joinKeys, dataProductCoordinates, context) {
+    if (dataProductCoordinates === undefined) {
+        return undefined;
+    }
+    const involved = resolveJoinedAccessPoints(sql, services);
+    const apA = involved[0];
+    const apB = involved[1];
+    if (involved.length !== 2 || apA === undefined || apB === undefined) {
+        return undefined;
+    }
+    const sharedKeys = sharedColumnNames(apA, apB)
+        .filter((name) => joinKeys.has(name.toLowerCase()))
+        .slice(0, MAX_JOIN_OVERLAP_PROBE_KEYS);
+    if (sharedKeys.length === 0) {
+        return undefined;
+    }
+    const columnsA = buildColumnByNameIndex(apA.columns);
+    const columnsB = buildColumnByNameIndex(apB.columns);
+    const probeOptions = {
+        rowLimit: JOIN_OVERLAP_PROBE_LIMIT,
+        maxColumns: MAX_JOIN_OVERLAP_PROBE_KEYS,
+        silent: true,
+    };
+    const probeA = createColumnValueProbe([apA], dataProductCoordinates, undefined, true, context, probeOptions);
+    const probeB = createColumnValueProbe([apB], dataProductCoordinates, undefined, true, context, probeOptions);
+    const fullyEnumerated = (values) => values !== undefined &&
+        values.length > 0 &&
+        values.length < JOIN_OVERLAP_CONCLUSIVE_MAX;
+    for (const key of sharedKeys) {
+        const columnA = columnsA.get(key.toLowerCase());
+        const columnB = columnsB.get(key.toLowerCase());
+        if (columnA === undefined || columnB === undefined) {
+            continue;
+        }
+        const nameB = columnB.name;
+        if (!isStringTypedColumn(columnA) || !isStringTypedColumn(columnB)) {
+            continue;
+        }
+        const [valuesA, valuesB] = await Promise.all([probeA(key), probeB(nameB)]);
+        if (!fullyEnumerated(valuesA) || !fullyEnumerated(valuesB)) {
+            continue;
+        }
+        const setB = new Set(valuesB.map((v) => v.toLowerCase()));
+        if (!valuesA.some((v) => setB.has(v.toLowerCase()))) {
+            return buildDisjointJoinMessage(apA.title, apB.title, key, previewValues(valuesA), previewValues(valuesB));
+        }
+    }
+    return undefined;
+}
+// Recovers a zero-row query by re-probing each WHERE-filter column's real distinct
+// values and rewriting equality/IN literals that don't match (casing/spelling).
+async function attemptDistinctValueRecovery(currentSql, selectedServices, dataProductCoordinates, context) {
+    const { plugin, config, setMessages } = context;
+    const isAccessPointProbe = dataProductCoordinates !== undefined &&
+        selectedServices.some((s) => s.sourceType === TDSServiceSourceType.ACCESS_POINT);
+    const fromClause = isAccessPointProbe
+        ? undefined
+        : extractFromClause(currentSql);
+    if (!isAccessPointProbe && fromClause === undefined) {
+        return undefined;
+    }
+    const probeColumnValues = createColumnValueProbe(selectedServices, dataProductCoordinates, fromClause, isAccessPointProbe, context);
     await Promise.all(collectWhereFilterColumns(currentSql)
         .slice(0, MAX_RECOVERED_FILTER_COLUMNS)
         .map((column) => probeColumnValues(column)));
@@ -2343,14 +2557,14 @@ async function processAccessPointQuery(question, accessPoints, metadata, context
     const { config, plugin, setMessages } = context;
     addThinkingStep(setMessages, 'Found relevant access points to query');
     const selectedAPs = await selectBestServices(question, accessPoints, context);
-    const currentAccessPoints = new Set(selectedAPs.map((ap) => ap.pattern.replace(/^\//, '')));
+    const currentAccessPoints = new Set(selectedAPs.map((ap) => accessPointName(ap)));
     const scopedContext = {
         ...context,
         history: filterHistoryForAccessPoints(context.history, currentAccessPoints),
     };
     try {
         addThinkingStep(setMessages, 'Grounding filter values from real data...');
-        await withTimeout(plugin.enrichAccessPointSampleValues(selectedAPs, dataProductCoordinates, config), VALUE_GROUNDING_TIME_BUDGET_MS);
+        await withTimeout(plugin.enrichAccessPointSampleValues(selectedAPs, dataProductCoordinates, config, question), VALUE_GROUNDING_TIME_BUDGET_MS);
     }
     catch (error) {
         assertErrorThrown(error);
@@ -2368,9 +2582,34 @@ async function processAccessPointQuery(question, accessPoints, metadata, context
     updateLastAssistant(setMessages, () => ({
         sql: finalSql,
         sqlGenTime: sqlGenTimeValue,
-        isExecuting: true,
         queriedAccessPoints: orderedAccessPointsFromSql(finalSql),
     }));
+    const joinKeys = canSkipExecutionOnEmptyJoin(finalSql)
+        ? extractJoinKeyColumns(finalSql)
+        : new Set();
+    let disjointJoin = joinKeys.size > 0
+        ? detectDisjointJoinUniverses(finalSql, selectedAPs, {
+            requireComplete: true,
+            restrictToColumns: joinKeys,
+        })
+        : undefined;
+    if (disjointJoin === undefined && joinKeys.size > 0) {
+        addThinkingStep(setMessages, 'Probing join-key overlap...');
+        disjointJoin = await withTimeout(probeJoinOverlap(finalSql, selectedAPs, joinKeys, dataProductCoordinates, scopedContext), JOIN_OVERLAP_PROBE_TIMEOUT_MS);
+    }
+    if (disjointJoin !== undefined) {
+        addThinkingStep(setMessages, 'Join keys do not overlap — skipping execution');
+        await reportQueryResults({
+            currentSql: finalSql,
+            sqlResult: { columns: [], rows: [] },
+            question,
+            services: selectedAPs,
+            allAccessPoints: accessPoints,
+            zeroRowExplanation: disjointJoin,
+        }, metadata, scopedContext, startTime, false);
+        return;
+    }
+    updateLastAssistant(setMessages, () => ({ isExecuting: true }));
     const execStartTime = Date.now();
     try {
         let execSql = finalSql;
@@ -2428,13 +2667,22 @@ async function processDataQuery(question, services, coordinates, metadata, conte
     const missingParams = detectMissingServiceParams(judgedSql, selectedServices);
     if (missingParams.length > 0) {
         const sqlGenTimeValue = elapsedSeconds(startTime, 2);
+        const missingParamsReason = `Missing required parameter${missingParams.length > 1 ? 's' : ''}: ${missingParams.map((p) => p.name).join(', ')}`;
         completeThinkingSteps(setMessages);
-        addThinkingStep(setMessages, `Missing required parameter${missingParams.length > 1 ? 's' : ''}: ${missingParams.map((p) => p.name).join(', ')}`);
+        addThinkingStep(setMessages, missingParamsReason);
         completeThinkingSteps(setMessages);
         updateLastAssistant(setMessages, () => ({
             sql: judgedSql,
             sqlGenTime: sqlGenTimeValue,
             textAnswer: buildMissingParamsWarning(missingParams),
+            ...(hasOrchestratorFallback
+                ? {
+                    fallbackAction: buildOrchestratorFallbackAction({
+                        failedSql: judgedSql,
+                        failedReason: missingParamsReason,
+                    }),
+                }
+                : {}),
             isProcessing: false,
             isExecuting: false,
             thinkingDuration: elapsedSeconds(startTime),
@@ -2450,9 +2698,10 @@ async function processDataQuery(question, services, coordinates, metadata, conte
     }));
     const execOutcome = await executeSqlWithRetries(judgedSql, question, selectedServices, coordinates, dataProductCoordinates, context);
     if (execOutcome.error) {
-        const execErrorType = classifyError(new Error(execOutcome.error));
-        addThinkingStep(setMessages, `Execution failed: ${execOutcome.error.slice(0, MAX_THINKING_ERROR_PREVIEW_LENGTH)}`);
-        finishWithThinkingError(setMessages, buildExecutionErrorMessage(execOutcome.error, selectedServices), startTime, execErrorType === LegendAIErrorType.GENERAL
+        const execError = execOutcome.error;
+        const execErrorType = classifyError(new Error(execError));
+        addThinkingStep(setMessages, `Execution failed: ${execError.slice(0, MAX_THINKING_ERROR_PREVIEW_LENGTH)}`);
+        finishWithThinkingError(setMessages, buildExecutionErrorMessage(execError, selectedServices), startTime, execErrorType === LegendAIErrorType.GENERAL
             ? LegendAIErrorType.EXECUTION
             : execErrorType);
         updateLastAssistant(setMessages, () => ({
@@ -2460,10 +2709,10 @@ async function processDataQuery(question, services, coordinates, metadata, conte
             suggestedQueries: buildFallbackSuggestions(selectedServices),
             ...(hasOrchestratorFallback
                 ? {
-                    fallbackAction: {
-                        label: ORCHESTRATOR_FALLBACK_LABEL,
-                        actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
-                    },
+                    fallbackAction: buildOrchestratorFallbackAction({
+                        failedSql: execOutcome.sql,
+                        failedReason: execError,
+                    }),
                 }
                 : {}),
         }));
@@ -2549,10 +2798,7 @@ export async function processQuestion(question, services, coordinates, metadata,
             await handleMetadataQuestion(question, metadata, context, startTime, false, services, getMetadataEnrichment());
             if (config.orchestratorUrl && dataProductCoordinates) {
                 updateLastAssistant(setMessages, () => ({
-                    fallbackAction: {
-                        label: ORCHESTRATOR_FALLBACK_LABEL,
-                        actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
-                    },
+                    fallbackAction: buildOrchestratorFallbackAction(),
                 }));
             }
             return;
@@ -2625,10 +2871,7 @@ export async function processQuestionWithIntent(question, intent, services, coor
         const startTime = Date.now();
         await handleMetadataQuestion(question, metadata, context, startTime, false, services, getMetadataEnrichment());
         updateLastAssistant(setMessages, () => ({
-            fallbackAction: {
-                label: ORCHESTRATOR_FALLBACK_LABEL,
-                actionId: LEGEND_AI_ORCHESTRATOR_FALLBACK_ACTION_ID,
-            },
+            fallbackAction: buildOrchestratorFallbackAction(),
         }));
         return;
     }