@duckcodeailabs/dql-agent 1.6.6 → 1.6.8

package/dist/metadata/catalog.js

@@ -97,33 +97,34 @@ export async function buildLocalContextPack(projectRoot, request) {
     const catalog = openMetadataCatalog(projectRoot);
     try {
         const mode = request.mode ?? 'question';
+        const followUp = normalizeFollowUpContext(request.followUp);
         const runtimeSnapshot = request.runtimeSchemaSnapshot ?? catalog.latestRuntimeSchemaSnapshot();
         const runtimeObjects = runtimeSnapshot ? runtimeSchemaObjects(runtimeSnapshot) : [];
         const selectedObjects = selectedContextObjects(request.selectedContext);
-        const selectedReferencedObjects = catalog
-            .getObjectsByKeys(selectedContextReferenceKeys(request.selectedContext, selectedObjects))
-            .map((object) => ({ ...object, score: Math.max(object.score ?? 0, 20) }));
+        const followUpObjects = followUpContextObjects(followUp);
+        const followUpSourceObjects = catalog.getObjectsByKeys(followUpSourceObjectKeys(followUp));
+        const searchQuery = buildFollowUpSearchQuery(request.question, followUp);
         const searchRows = catalog.searchObjects({
-            query: request.question,
+            query: searchQuery,
             objectTypes: request.objectTypes,
             limit: Math.max(request.limit ?? 80, 20),
         });
         const exact = request.focusObjectKey ? catalog.getObject(request.focusObjectKey) : null;
         const ranked = rankMetadataObjects({
             rows: mergeObjects(exact
-                ? [exact, ...selectedReferencedObjects, ...searchRows, ...runtimeObjects, ...selectedObjects]
-                : [...selectedReferencedObjects, ...searchRows, ...runtimeObjects, ...selectedObjects]),
-            question: request.question,
+                ? [exact, ...followUpSourceObjects, ...followUpObjects, ...searchRows, ...runtimeObjects, ...selectedObjects]
+                : [...followUpSourceObjects, ...followUpObjects, ...searchRows, ...runtimeObjects, ...selectedObjects]),
+            question: searchQuery,
             limit: request.limit ?? 80,
         });
-        const selected = ranked.selected;
-        const focusObjectKey = request.focusObjectKey ?? selectedReferencedObjects[0]?.objectKey ?? selected[0]?.objectKey ?? null;
+        const selected = mergeObjects([...followUpSourceObjects, ...followUpObjects, ...ranked.selected]);
+        const focusObjectKey = request.focusObjectKey ?? selected[0]?.objectKey ?? null;
         const edgeWalk = catalog.edgesForKeys(selected.map((row) => row.objectKey), 3);
         const edgeObjectKeys = Array.from(new Set(edgeWalk.flatMap((edge) => [edge.fromKey, edge.toKey])));
         const graphObjects = catalog.getObjectsByKeys(edgeObjectKeys);
         const objects = rankMetadataObjects({
-            rows: mergeObjects([...selectedReferencedObjects, ...selected, ...graphObjects, ...runtimeObjects, ...selectedObjects]),
-            question: request.question,
+            rows: mergeObjects([...followUpSourceObjects, ...followUpObjects, ...selected, ...graphObjects, ...runtimeObjects, ...selectedObjects]),
+            question: searchQuery,
             limit: request.limit ?? 120,
         }).selected;
         const objectKeys = objects.map((row) => row.objectKey);
@@ -137,7 +138,7 @@ export async function buildLocalContextPack(projectRoot, request) {
         const evidenceRoles = buildEvidenceRoles(objects, queryRuns);
         const reranked = rankMetadataObjects({
             rows: mergeObjects([...searchRows, ...objects]),
-            question: request.question,
+            question: searchQuery,
             limit: request.limit ?? 120,
         });
         const conflicts = buildCandidateConflicts(reranked.ranked);
@@ -152,6 +153,7 @@ export async function buildLocalContextPack(projectRoot, request) {
         const payload = {
             id: '',
             question: request.question,
+            followUp: followUp ?? undefined,
             focusObjectKey,
             mode,
             trustLabel,
@@ -682,6 +684,7 @@ function objectFromKGNode(node) {
         businessRules: node.businessRules ?? [],
         caveats: node.caveats ?? [],
         llmContext: node.llmContext,
+        referencedBy: node.referencedBy ?? [],
     };
     return {
         objectKey: objectKeyFromKGNode(node),
@@ -838,6 +841,7 @@ function addManifestBlockDetails(manifest, objects) {
                 chartType: block.chartType,
                 blockType: block.blockType,
                 tests: block.tests,
+                draftMetadata: block.draftMetadata,
             }),
         }));
     }
@@ -959,10 +963,12 @@ function addProjectDiagnostics(manifest, semanticLayer, diagnostics) {
 function planContextPackRoute(input) {
     const intent = input.request.intent ?? classifyMetadataIntent(input.request.question, input.request.followUp);
     const exact = findExactCertifiedObject(input.request.question, intent, input.objects);
+    const exactExampleMatch = exact ? hasExactExampleQuestion(input.request.question, exact) : false;
     const missingContext = buildMissingContext(input.request, intent, input.objects, input.allowedSqlContext);
     const selectedEvidence = input.evidenceRoles.slice(0, 16);
     if (exact && (intent === 'exact_certified_lookup'
         || intent === 'definition_lookup'
+        || exactExampleMatch
         || (intent === 'ad_hoc_ranking' && objectNameInQuestion(input.request.question, exact)))) {
         return {
             route: 'certified',
@@ -1075,6 +1081,8 @@ function classifyMetadataIntent(question, followUp) {
         return 'ad_hoc_ranking';
     if (/\b(block|certified|saved|existing|approved|governed)\b/.test(lower))
         return 'exact_certified_lookup';
+    if (isDirectKpiValueQuestion(question))
+        return 'exact_certified_lookup';
     if (/\b(show|list|find|which|who|how many|how much|metric|kpi|dashboard|performance|revenue|sales|points|goals|orders|customers|users)\b/.test(lower))
         return 'ad_hoc_ranking';
     return 'clarify';
@@ -1098,6 +1106,11 @@ function findExactCertifiedObject(question, intent, objects) {
         hasCompatibleMetadataRankingDirection(question, object));
     if (namedExact)
         return namedExact;
+    const exampleExact = candidates.find((object) => object.objectType === 'dql_block' &&
+        hasExactExampleQuestion(question, object) &&
+        hasCompatibleMetadataRankingDirection(question, object));
+    if (exampleExact)
+        return exampleExact;
     if (isGeneratedMetadataIntent(intent))
         return undefined;
     return candidates.find((object) => object.objectType === 'dql_block' &&
@@ -1116,6 +1129,15 @@ function objectNameInQuestion(question, object) {
     const fullName = normalizeSearchText(object.fullName ?? '');
     return Boolean(name && q.includes(name)) || Boolean(fullName && q.includes(fullName));
 }
+function hasExactExampleQuestion(question, object) {
+    const q = normalizeSearchText(question);
+    if (!q)
+        return false;
+    const examples = Array.isArray(object.payload?.examples) ? object.payload.examples : [];
+    return examples.some((example) => example &&
+        typeof example === 'object' &&
+        normalizeSearchText(String(example.question ?? '')) === q);
+}
 function hasMeaningfulObjectOverlap(question, object) {
     const terms = new Set(tokenize(question));
     if (terms.size === 0)
@@ -1132,6 +1154,17 @@ function hasMeaningfulObjectOverlap(question, object) {
 function looksLikeDifferentGrainQuestion(question) {
     return /\b(for|where|only|specific|single|individual|named|called|by|break\s*down|breakdown|drill|compare|versus|vs\.?|segment|least|lowest|fewest|bottom|why|changed?|driver|anomal|exception)\b/i.test(question);
 }
+function isDirectKpiValueQuestion(question) {
+    const lower = question.toLowerCase();
+    if (/\b(by|break\s*down|breakdown|drill|compare|versus|vs\.?|segment|cohort|top|bottom|best|worst|highest|lowest|least|fewest|rank|ranking|most|why|changed?|driver|anomal|exception)\b/.test(lower)) {
+        return false;
+    }
+    if (isEntityQuestion(question))
+        return false;
+    const asksForValue = /\b(what\s+(?:is|was|were|are)|how\s+(?:much|many)|show|report|calculate|give\s+me|tell\s+me)\b/.test(lower);
+    const metricLanguage = /\b(revenue|sales|arr|mrr|bookings|orders|customers|users|churn|retention|conversion|rate|count|total|points|goals|kpi|metric)\b/.test(lower);
+    return asksForValue && metricLanguage;
+}
 function hasCompatibleMetadataRankingDirection(question, object) {
     const questionDirection = rankingDirection(question);
     if (!questionDirection)
@@ -1231,7 +1264,7 @@ function isTimeLikeColumn(name) {
     return /\b(date|time|day|week|month|quarter|year|season|period|created_at|updated_at)\b/i.test(name);
 }
 function selectedRows(value) {
-    const root = normalizeSelectedContextRoot(value);
+    const root = value && typeof value === 'object' ? value : null;
     const selected = root?.selectedBlock && typeof root.selectedBlock === 'object' ? root.selectedBlock : root;
     const candidates = [
         selected?.resultSample,
@@ -1313,6 +1346,7 @@ function evidenceReasonForObject(object) {
 }
 function buildAllowedSqlContext(objects, edges) {
     const byRelation = new Map();
+    const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
     const addRelation = (relation) => {
         const key = normalizeRelationKey(relation.relation);
         if (!key)
@@ -1330,9 +1364,15 @@ function buildAllowedSqlContext(objects, edges) {
         });
     };
     for (const object of objects) {
+        if (object.objectType === 'warehouse_table' && !warehouseTableHasTrustedReference(object, objectsByKey)) {
+            continue;
+        }
         const relation = metadataRelationFromObject(object);
         if (relation)
             addRelation(relation);
+        if (object.objectType === 'dql_block' && !isCertifiedMetadataObject(object)) {
+            continue;
+        }
         for (const table of metadataStringArray(object.payload?.tableDependencies)) {
             addRelation({
                 relation: table,
@@ -1352,7 +1392,6 @@ function buildAllowedSqlContext(objects, edges) {
             });
         }
     }
-    const objectsByKey = new Map(objects.map((object) => [object.objectKey, object]));
     for (const edge of edges) {
         if (edge.edgeType !== 'maps_to_dbt_model' && edge.edgeType !== 'uses_dbt_model')
             continue;
@@ -1366,7 +1405,10 @@ function buildAllowedSqlContext(objects, edges) {
     return {
         relations: Array.from(byRelation.values()).sort((a, b) => a.relation.localeCompare(b.relation)),
         sourceBlockSql: objects
-            .filter((object) => object.objectType === 'dql_block' && typeof object.payload?.sql === 'string' && object.payload.sql.trim())
+            .filter((object) => object.objectType === 'dql_block' &&
+            isCertifiedMetadataObject(object) &&
+            typeof object.payload?.sql === 'string' &&
+            object.payload.sql.trim())
             .slice(0, 8)
             .map((object) => ({
             objectKey: object.objectKey,
@@ -1376,6 +1418,18 @@ function buildAllowedSqlContext(objects, edges) {
         })),
     };
 }
+function warehouseTableHasTrustedReference(object, objectsByKey) {
+    const refs = metadataStringArray(object.payload?.referencedBy);
+    if (refs.length === 0)
+        return true;
+    if (refs.some((ref) => !ref.startsWith('block:')))
+        return true;
+    return refs.some((ref) => {
+        const blockName = ref.slice('block:'.length);
+        const block = objectsByKey.get(`dql:block:${blockName}`);
+        return block ? isCertifiedMetadataObject(block) : false;
+    });
+}
 function metadataRelationFromObject(object) {
     if (object.objectType === 'dbt_column' || object.objectType === 'runtime_column') {
         const relation = metadataPayloadString(object, 'relation');
@@ -1444,7 +1498,7 @@ function runtimeSchemaObjects(snapshot) {
     });
 }
 function selectedContextObjects(value) {
-    const root = normalizeSelectedContextRoot(value);
+    const root = value && typeof value === 'object' ? value : null;
     if (!root)
         return [];
     const selected = root.selectedBlock && typeof root.selectedBlock === 'object' ? root.selectedBlock : root;
@@ -1468,60 +1522,65 @@ function selectedContextObjects(value) {
             }),
         }];
 }
-function selectedContextReferenceKeys(value, selectedObjects) {
-    const root = normalizeSelectedContextRoot(value);
-    const keys = new Set();
-    const addBlock = (blockId) => {
-        if (typeof blockId !== 'string')
-            return;
-        const name = blockId.replace(/^block:/i, '').trim();
-        if (name)
-            keys.add(`dql:block:${name}`);
-    };
-    if (root) {
-        const selected = root.selectedBlock && typeof root.selectedBlock === 'object' ? root.selectedBlock : root;
-        addBlock(selected.blockId);
-        addBlock(root.sourceBlockId);
-        addBlock(root.blockId);
-    }
-    for (const object of selectedObjects) {
-        addBlock(object.payload?.blockId);
-    }
-    return Array.from(keys);
-}
-function normalizeSelectedContextRoot(value) {
-    if (!value || typeof value !== 'object' || Array.isArray(value))
+function normalizeFollowUpContext(value) {
+    if (!value || typeof value !== 'object')
+        return null;
+    const record = value;
+    const kind = record.kind === 'drilldown' ? 'drilldown' : record.kind === 'generic' ? 'generic' : null;
+    if (!kind)
         return null;
-    const original = value;
-    const fromSql = typeof original.sql === 'string' ? parseJsonObject(original.sql) : null;
-    const root = fromSql ?? original;
-    const nested = root.context && typeof root.context === 'object' && !Array.isArray(root.context)
-        ? root.context
-        : null;
-    if (!nested)
-        return root;
     return {
-        ...root,
-        ...nested,
-        requestTitle: root.title,
-        requestIntent: root.intent,
-        requestMode: root.mode,
-        context: nested,
+        kind,
+        sourceBlockName: stringValue(record.sourceBlockName),
+        sourceQuestion: stringValue(record.sourceQuestion),
+        sourceAnswer: stringValue(record.sourceAnswer),
+        filters: metadataStringArray(record.filters),
+        dimensions: metadataStringArray(record.dimensions),
     };
 }
-function parseJsonObject(raw) {
-    const trimmed = raw.trim();
-    if (!trimmed || (!trimmed.startsWith('{') && !trimmed.startsWith('[')))
-        return null;
-    try {
-        const parsed = JSON.parse(trimmed);
-        return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
-            ? parsed
-            : null;
-    }
-    catch {
-        return null;
-    }
+function followUpSourceObjectKeys(followUp) {
+    if (!followUp?.sourceBlockName)
+        return [];
+    return [`dql:block:${followUp.sourceBlockName}`];
+}
+function followUpContextObjects(followUp) {
+    if (!followUp)
+        return [];
+    const text = [
+        followUp.kind,
+        followUp.sourceBlockName ?? '',
+        followUp.sourceQuestion ?? '',
+        followUp.sourceAnswer ?? '',
+        ...(followUp.filters ?? []),
+        ...(followUp.dimensions ?? []),
+    ].join(' ');
+    return [{
+            objectKey: `selected:followup:${sha256(stableStringify(followUp)).slice(0, 16)}`,
+            objectType: 'selected_context',
+            name: followUp.kind === 'drilldown' ? 'Follow-up drilldown request' : 'Follow-up request',
+            description: text.trim() || undefined,
+            status: 'transient_context',
+            sourceSystem: 'agent follow-up context',
+            payload: compactObject({
+                kind: followUp.kind,
+                sourceBlockName: followUp.sourceBlockName,
+                sourceQuestion: followUp.sourceQuestion,
+                sourceAnswer: followUp.sourceAnswer,
+                filters: followUp.filters,
+                dimensions: followUp.dimensions,
+            }),
+        }];
+}
+function buildFollowUpSearchQuery(question, followUp) {
+    if (!followUp)
+        return question;
+    return [
+        question,
+        followUp.sourceBlockName ?? '',
+        followUp.sourceQuestion ?? '',
+        ...(followUp.filters ?? []),
+        ...(followUp.dimensions ?? []),
+    ].filter(Boolean).join(' ');
 }
 function normalizeRuntimeSchemaTables(tables) {
     const byRelation = new Map();
@@ -1633,7 +1692,6 @@ function mergeObject(a, b) {
     return {
         ...a,
         ...b,
-        score: Math.max(a.score ?? 0, b.score ?? 0) || undefined,
         description: b.description || a.description,
         payload: compactObject({ ...(a.payload ?? {}), ...(b.payload ?? {}) }),
     };