@finos/legend-lego 2.0.210 → 2.0.212

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/lib/index.css +2 -2
  2. package/lib/index.css.map +1 -1
  3. package/lib/legend-ai/LegendAIDocEnrichment.d.ts +12 -12
  4. package/lib/legend-ai/LegendAIDocEnrichment.d.ts.map +1 -1
  5. package/lib/legend-ai/LegendAIDocEnrichment.js +317 -86
  6. package/lib/legend-ai/LegendAIDocEnrichment.js.map +1 -1
  7. package/lib/legend-ai/LegendAIHostIntegration.d.ts +31 -0
  8. package/lib/legend-ai/LegendAIHostIntegration.d.ts.map +1 -0
  9. package/lib/legend-ai/LegendAIHostIntegration.js +50 -0
  10. package/lib/legend-ai/LegendAIHostIntegration.js.map +1 -0
  11. package/lib/legend-ai/LegendAIServiceRetrieval.d.ts.map +1 -1
  12. package/lib/legend-ai/LegendAIServiceRetrieval.js +34 -35
  13. package/lib/legend-ai/LegendAIServiceRetrieval.js.map +1 -1
  14. package/lib/legend-ai/LegendAITypes.d.ts +85 -7
  15. package/lib/legend-ai/LegendAITypes.d.ts.map +1 -1
  16. package/lib/legend-ai/LegendAITypes.js +99 -1
  17. package/lib/legend-ai/LegendAITypes.js.map +1 -1
  18. package/lib/legend-ai/LegendAI_LegendApplicationPlugin_Extension.d.ts +36 -3
  19. package/lib/legend-ai/LegendAI_LegendApplicationPlugin_Extension.d.ts.map +1 -1
  20. package/lib/legend-ai/LegendAI_LegendApplicationPlugin_Extension.js +29 -0
  21. package/lib/legend-ai/LegendAI_LegendApplicationPlugin_Extension.js.map +1 -1
  22. package/lib/legend-ai/__test-utils__/LegendAITestUtils.d.ts.map +1 -1
  23. package/lib/legend-ai/__test-utils__/LegendAITestUtils.js +7 -0
  24. package/lib/legend-ai/__test-utils__/LegendAITestUtils.js.map +1 -1
  25. package/lib/legend-ai/components/LegendAICharts.d.ts.map +1 -1
  26. package/lib/legend-ai/components/LegendAICharts.js +6 -10
  27. package/lib/legend-ai/components/LegendAICharts.js.map +1 -1
  28. package/lib/legend-ai/components/LegendAIChat.d.ts.map +1 -1
  29. package/lib/legend-ai/components/LegendAIChat.js +263 -41
  30. package/lib/legend-ai/components/LegendAIChat.js.map +1 -1
  31. package/lib/legend-ai/components/LegendAIChatInput.d.ts +1 -1
  32. package/lib/legend-ai/components/LegendAIChatInput.d.ts.map +1 -1
  33. package/lib/legend-ai/components/LegendAIChatInput.js +2 -1
  34. package/lib/legend-ai/components/LegendAIChatInput.js.map +1 -1
  35. package/lib/legend-ai/index.d.ts +5 -2
  36. package/lib/legend-ai/index.d.ts.map +1 -1
  37. package/lib/legend-ai/index.js +5 -2
  38. package/lib/legend-ai/index.js.map +1 -1
  39. package/lib/legend-ai/stores/LegendAIChatProcessors.d.ts +9 -3
  40. package/lib/legend-ai/stores/LegendAIChatProcessors.d.ts.map +1 -1
  41. package/lib/legend-ai/stores/LegendAIChatProcessors.js +794 -220
  42. package/lib/legend-ai/stores/LegendAIChatProcessors.js.map +1 -1
  43. package/lib/legend-ai/stores/LegendAIChatState.d.ts.map +1 -1
  44. package/lib/legend-ai/stores/LegendAIChatState.js +47 -21
  45. package/lib/legend-ai/stores/LegendAIChatState.js.map +1 -1
  46. package/lib/legend-ai/stores/LegendAIJoinAnalysis.d.ts +29 -0
  47. package/lib/legend-ai/stores/LegendAIJoinAnalysis.d.ts.map +1 -0
  48. package/lib/legend-ai/stores/LegendAIJoinAnalysis.js +124 -0
  49. package/lib/legend-ai/stores/LegendAIJoinAnalysis.js.map +1 -0
  50. package/lib/legend-ai/stores/LegendAISqlHelpers.d.ts +25 -0
  51. package/lib/legend-ai/stores/LegendAISqlHelpers.d.ts.map +1 -0
  52. package/lib/legend-ai/stores/LegendAISqlHelpers.js +64 -0
  53. package/lib/legend-ai/stores/LegendAISqlHelpers.js.map +1 -0
  54. package/lib/legend-ai/stores/LegendAISqlJoinSanitizers.d.ts +22 -0
  55. package/lib/legend-ai/stores/LegendAISqlJoinSanitizers.d.ts.map +1 -0
  56. package/lib/legend-ai/stores/LegendAISqlJoinSanitizers.js +411 -0
  57. package/lib/legend-ai/stores/LegendAISqlJoinSanitizers.js.map +1 -0
  58. package/package.json +6 -6
  59. package/src/legend-ai/LegendAIDocEnrichment.ts +435 -97
  60. package/src/legend-ai/LegendAIHostIntegration.tsx +80 -0
  61. package/src/legend-ai/LegendAIServiceRetrieval.ts +50 -35
  62. package/src/legend-ai/LegendAITypes.ts +169 -5
  63. package/src/legend-ai/LegendAI_LegendApplicationPlugin_Extension.ts +73 -1
  64. package/src/legend-ai/__test-utils__/LegendAITestUtils.ts +10 -0
  65. package/src/legend-ai/components/LegendAICharts.tsx +30 -33
  66. package/src/legend-ai/components/LegendAIChat.tsx +555 -91
  67. package/src/legend-ai/components/LegendAIChatInput.tsx +6 -2
  68. package/src/legend-ai/index.ts +22 -3
  69. package/src/legend-ai/stores/LegendAIChatProcessors.ts +1286 -322
  70. package/src/legend-ai/stores/LegendAIChatState.ts +73 -28
  71. package/src/legend-ai/stores/LegendAIJoinAnalysis.ts +165 -0
  72. package/src/legend-ai/stores/LegendAISqlHelpers.ts +82 -0
  73. package/src/legend-ai/stores/LegendAISqlJoinSanitizers.ts +515 -0
  74. package/tsconfig.json +4 -0
@@ -14,7 +14,9 @@
14
14
  * limitations under the License.
15
15
  */
16
16
  import { extractElementNameFromPath, PRIMITIVE_TYPE, } from '@finos/legend-graph';
17
+ import { isNonNullable } from '@finos/legend-shared';
17
18
  import { ClassDocumentationEntry, AssociationDocumentationEntry, EnumerationDocumentationEntry, PropertyDocumentationEntry, } from '../model-documentation/index.js';
19
+ import { sharedColumnNames } from './stores/LegendAISqlHelpers.js';
18
20
  /**
19
21
  * Builds a lookup map from lowercase property name to its documentation entry.
20
22
  * Used to enrich TDS columns with business descriptions and nullability
@@ -100,12 +102,6 @@ function buildAssociationAdjacency(uniqueAssocs) {
100
102
  }
101
103
  return adjacency;
102
104
  }
103
- function findSharedColumns(svcA, svcB) {
104
- const colsA = new Set(svcA.columns.map((c) => c.name.toLowerCase()));
105
- return svcB.columns
106
- .filter((c) => colsA.has(c.name.toLowerCase()))
107
- .map((c) => c.name);
108
- }
109
105
  function findViaConnectionFromClass(clsA, classNamesB, adjacency) {
110
106
  const edgesA = adjacency.get(clsA.toLowerCase());
111
107
  if (!edgesA) {
@@ -138,7 +134,7 @@ function findViaRelationship(svcA, svcB, adjacency) {
138
134
  return {
139
135
  leftService: svcA.title,
140
136
  rightService: svcB.title,
141
- joinColumns: findSharedColumns(svcA, svcB),
137
+ joinColumns: sharedColumnNames(svcB, svcA),
142
138
  viaEntity: via.viaEntity,
143
139
  leftCardinality: via.leftMult,
144
140
  rightCardinality: via.rightMult,
@@ -161,7 +157,7 @@ function findDirectRelationship(svcA, svcB, adjacency) {
161
157
  return {
162
158
  leftService: svcA.title,
163
159
  rightService: svcB.title,
164
- joinColumns: findSharedColumns(svcA, svcB),
160
+ joinColumns: sharedColumnNames(svcB, svcA),
165
161
  };
166
162
  }
167
163
  }
@@ -394,39 +390,6 @@ export function extractLambdaPreFilters(rawLambdaBody) {
394
390
  extractIsNotNullPostFilters(rawLambdaBody, results);
395
391
  return results;
396
392
  }
397
- /**
398
- * Formats pre-filter constraints into a human-readable summary for LLM context.
399
- * This generates a concise description of what's hardcoded in the service lambda
400
- * so the AI avoids generating contradictory WHERE clauses.
401
- */
402
- export function formatPreFiltersForContext(preFilters) {
403
- if (preFilters.length === 0) {
404
- return '';
405
- }
406
- const parts = [];
407
- for (const pf of preFilters) {
408
- const shortProp = pf.property.includes('.')
409
- ? (pf.property.split('.').pop() ?? pf.property)
410
- : pf.property;
411
- switch (pf.operator) {
412
- case 'equal':
413
- parts.push(`${shortProp} = '${String(pf.value)}'`);
414
- break;
415
- case 'isEmpty':
416
- parts.push(`${shortProp} IS NULL (always)`);
417
- break;
418
- case 'isNotEmpty':
419
- parts.push(`${shortProp} IS NOT NULL (always)`);
420
- break;
421
- case 'isNotNull':
422
- parts.push(`${shortProp} IS NOT NULL (post-filter)`);
423
- break;
424
- default:
425
- break;
426
- }
427
- }
428
- return `Pre-applied filters: ${parts.join('; ')}`;
429
- }
430
393
  // ────────────────────────────────────────────────────────────────────────────
431
394
  // Model context extraction from DataSpace elementDocs
432
395
  // ────────────────────────────────────────────────────────────────────────────
@@ -438,13 +401,14 @@ export function formatPreFiltersForContext(preFilters) {
438
401
  *
439
402
  * Only applicable to DataSpaces (not DataProducts, which lack elementDocs).
440
403
  */
404
+ const UNKNOWN_TYPE = 'Unknown';
441
405
  function collectClassEntities(classMap) {
442
406
  return Array.from(classMap.values()).map((cls) => {
443
407
  const properties = cls.properties.map((prop) => {
444
408
  const upperBound = prop.multiplicity?.upperBound;
445
409
  return {
446
410
  name: prop.name,
447
- type: prop.type ?? 'Unknown',
411
+ type: prop.type ?? UNKNOWN_TYPE,
448
412
  isCollection: upperBound === undefined || upperBound > 1,
449
413
  isOptional: (prop.multiplicity?.lowerBound ?? 0) === 0,
450
414
  };
@@ -689,10 +653,47 @@ function buildCrossClassNlHints(question, rootEntity, rootEntityObj, modelContex
689
653
  },
690
654
  ];
691
655
  }
656
+ const MAX_VALID_VALUES_PER_PROPERTY = 30;
657
+ function buildValidValueNlHints(rootEntity, relatedEntities, modelContext, entityMap) {
658
+ const valuesByEnum = new Map();
659
+ for (const enumeration of modelContext.enumerations ?? []) {
660
+ valuesByEnum.set(enumeration.path, enumeration.values);
661
+ valuesByEnum.set(enumeration.name, enumeration.values);
662
+ }
663
+ if (valuesByEnum.size === 0) {
664
+ return [];
665
+ }
666
+ const entries = [];
667
+ for (const path of [rootEntity, ...relatedEntities]) {
668
+ const entity = entityMap.get(path);
669
+ if (!entity) {
670
+ continue;
671
+ }
672
+ for (const prop of entity.properties) {
673
+ const values = valuesByEnum.get(prop.type);
674
+ if (values && values.length > 0) {
675
+ const shown = values.slice(0, MAX_VALID_VALUES_PER_PROPERTY).join(', ');
676
+ entries.push({
677
+ id: `valid_values:${entity.name}.${prop.name}`,
678
+ description: `Valid values for '${prop.name}' on ${entity.name}: ${shown}. Filter using these exact values (match case-insensitively).`,
679
+ category: 'valid_values',
680
+ });
681
+ }
682
+ }
683
+ }
684
+ return entries;
685
+ }
692
686
  function buildNlContextEntries(question, rootEntity, rootEntityObj, relatedEntities, modelContext, entityMap) {
693
687
  const entries = [
694
688
  ...buildRootEntityNlHints(rootEntity, rootEntityObj, modelContext),
695
689
  ];
690
+ if (rootEntityObj) {
691
+ entries.unshift({
692
+ id: 'filter_guidance',
693
+ description: 'Filter guidance: match text/descriptive concepts on descriptive columns (e.g. description) using case-insensitive contains, not exact equals — and do NOT invent identifier/mnemonic codes (e.g. name, haverId, ticker) unless you were given their actual values. When several descriptive terms apply combine them with OR (never AND two contains on the same column). For country/geography prefer a coded column such as isoCode when one exists.',
694
+ category: 'filter_guidance',
695
+ });
696
+ }
696
697
  if (modelContext.dataspaceDescription) {
697
698
  const shortDesc = modelContext.dataspaceDescription
698
699
  .split('\n')
@@ -718,7 +719,7 @@ function buildNlContextEntries(question, rootEntity, rootEntityObj, relatedEntit
718
719
  });
719
720
  }
720
721
  }
721
- entries.push(...buildAssociationNlHints(rootEntity, relatedEntities, modelContext), ...buildExecutableNlHints(rootEntity, rootEntityObj, modelContext));
722
+ entries.push(...buildAssociationNlHints(rootEntity, relatedEntities, modelContext), ...buildExecutableNlHints(rootEntity, rootEntityObj, modelContext), ...buildValidValueNlHints(rootEntity, relatedEntities, modelContext, entityMap));
722
723
  if (rootEntityObj) {
723
724
  entries.push(...buildCrossClassNlHints(question, rootEntity, rootEntityObj, modelContext));
724
725
  }
@@ -846,19 +847,11 @@ export function buildSemanticPropertyIndex(modelContext) {
846
847
  const index = new Map();
847
848
  for (const entity of modelContext.entities) {
848
849
  for (const prop of entity.properties) {
849
- const nameTokens = prop.name
850
- .replaceAll(/(?<lower>[a-z])(?<upper>[A-Z])/g, '$<lower> $<upper>')
851
- .toLowerCase()
852
- .split(/[\s_]+/)
853
- .filter((t) => t.length > 2);
850
+ const nameTokens = splitIdentifierTokens(prop.name).filter((token) => token.length >= DEFAULT_MIN_TOKEN_LENGTH);
854
851
  addTokensToIndex(index, nameTokens, entity.path);
855
852
  }
856
853
  if (entity.description) {
857
- const descTokens = entity.description
858
- .toLowerCase()
859
- .split(/[\s,.;:!?'"()\-/]+/)
860
- .filter((t) => t.length > 2);
861
- addTokensToIndex(index, descTokens, entity.path);
854
+ addTokensToIndex(index, tokenizeText(entity.description), entity.path);
862
855
  }
863
856
  }
864
857
  return index;
@@ -866,6 +859,7 @@ export function buildSemanticPropertyIndex(modelContext) {
866
859
  // ────────────────────────────────────────────────────────────────────────────
867
860
  // Deterministic entity resolution (no LLM required)
868
861
  // ────────────────────────────────────────────────────────────────────────────
862
+ const MAX_RESOLVED_RELATED_ENTITIES = 5;
869
863
  /**
870
864
  * Resolves root and related entities using keyword matching against the model
871
865
  * context. This works without any LLM call and is used as the final fallback
@@ -896,10 +890,31 @@ function buildExecutableByRootIndex(modelContext) {
896
890
  words = [];
897
891
  index.set(exec.rootEntityPath, words);
898
892
  }
899
- words.push(...`${exec.title} ${exec.description ?? ''}`
900
- .toLowerCase()
901
- .split(/[\s,.;:!?'"()\-/]+/)
902
- .filter((t) => t.length > 2));
893
+ words.push(...tokenizeText(`${exec.title} ${exec.description ?? ''}`));
894
+ for (const param of exec.requiredParameters ?? []) {
895
+ words.push(...tokenizeText(param.name));
896
+ }
897
+ }
898
+ return index;
899
+ }
900
+ function buildEnumValueIndex(modelContext) {
901
+ const valueTokensByEnum = new Map();
902
+ for (const enumeration of modelContext.enumerations ?? []) {
903
+ const valueTokens = enumeration.values.flatMap((value) => tokenizeText(value));
904
+ valueTokensByEnum.set(enumeration.path, valueTokens);
905
+ valueTokensByEnum.set(enumeration.name, valueTokens);
906
+ }
907
+ const index = new Map();
908
+ if (valueTokensByEnum.size === 0) {
909
+ return index;
910
+ }
911
+ for (const entity of modelContext.entities) {
912
+ for (const prop of entity.properties) {
913
+ const valueTokens = valueTokensByEnum.get(prop.type);
914
+ if (valueTokens) {
915
+ addTokensToIndex(index, valueTokens, entity.path);
916
+ }
917
+ }
903
918
  }
904
919
  return index;
905
920
  }
@@ -916,7 +931,7 @@ function scoreEntityByProperties(entity, tokens) {
916
931
  }
917
932
  return score;
918
933
  }
919
- function scoreEntity(entity, tokens, semanticIndex, executableByRoot) {
934
+ function scoreEntity(entity, tokens, semanticIndex, executableByRoot, enumValueIndex) {
920
935
  const nameLower = entity.name.toLowerCase();
921
936
  let score = tokens.filter((t) => nameLower === t || nameLower.includes(t)).length * 5;
922
937
  score += scoreEntityByProperties(entity, tokens);
@@ -937,8 +952,22 @@ function scoreEntity(entity, tokens, semanticIndex, executableByRoot) {
937
952
  .length * 4;
938
953
  }
939
954
  score += tokens.filter((t) => semanticIndex.get(t)?.has(entity.path)).length;
955
+ score +=
956
+ tokens.filter((t) => enumValueIndex.get(t)?.has(entity.path)).length * 2;
940
957
  return score;
941
958
  }
959
+ export function rankEntities(question, modelContext) {
960
+ const executableByRoot = buildExecutableByRootIndex(modelContext);
961
+ const semanticIndex = buildSemanticPropertyIndex(modelContext);
962
+ const enumValueIndex = buildEnumValueIndex(modelContext);
963
+ const tokens = tokenizeText(question);
964
+ return modelContext.entities
965
+ .map((entity) => ({
966
+ entity,
967
+ score: scoreEntity(entity, tokens, semanticIndex, executableByRoot, enumValueIndex),
968
+ }))
969
+ .sort((a, b) => b.score - a.score);
970
+ }
942
971
  export function resolveEntitiesDeterministic(question, modelContext) {
943
972
  if (modelContext.entities.length === 0) {
944
973
  return undefined;
@@ -950,21 +979,10 @@ export function resolveEntitiesDeterministic(question, modelContext) {
950
979
  }
951
980
  }
952
981
  const queryableEntities = modelContext.entities.filter((e) => e.isQueryable);
953
- const executableByRoot = buildExecutableByRootIndex(modelContext);
954
- const semanticIndex = buildSemanticPropertyIndex(modelContext);
955
- const tokens = question
956
- .toLowerCase()
957
- .split(/[\s,.;:!?'"()\-/]+/)
958
- .filter((t) => t.length > 2);
959
- if (tokens.length === 0) {
982
+ if (tokenizeText(question).length === 0) {
960
983
  return pickFallbackEntity(queryableEntities, modelContext.entities);
961
984
  }
962
- const scored = modelContext.entities
963
- .map((entity) => ({
964
- entity,
965
- score: scoreEntity(entity, tokens, semanticIndex, executableByRoot),
966
- }))
967
- .sort((a, b) => b.score - a.score);
985
+ const scored = rankEntities(question, modelContext);
968
986
  const rootCandidates = queryableEntities.length > 0
969
987
  ? scored.filter((s) => s.entity.isQueryable)
970
988
  : scored;
@@ -983,15 +1001,101 @@ export function resolveEntitiesDeterministic(question, modelContext) {
983
1001
  }
984
1002
  }
985
1003
  for (const s of scored.slice(1)) {
986
- if (s.score > 0 && relatedSet.size < 5) {
1004
+ if (s.score > 0 && relatedSet.size < MAX_RESOLVED_RELATED_ENTITIES) {
987
1005
  relatedSet.add(s.entity.path);
988
1006
  }
989
1007
  }
990
1008
  return {
991
1009
  rootEntity: rootPath,
992
- relatedEntities: Array.from(relatedSet).slice(0, 5),
1010
+ relatedEntities: Array.from(relatedSet).slice(0, MAX_RESOLVED_RELATED_ENTITIES),
993
1011
  };
994
1012
  }
1013
+ const MAX_APPROACH_COLUMNS = 6;
1014
+ const MAX_APPROACH_DESCRIPTION_LENGTH = 200;
1015
+ const MAX_DESCRIPTION_PREVIEW_LENGTH = 300;
1016
+ export function buildDataQueryApproachText(question, modelContext, rootEntityPath) {
1017
+ const rootPath = rootEntityPath ??
1018
+ resolveEntitiesDeterministic(question, modelContext)?.rootEntity;
1019
+ if (!rootPath) {
1020
+ return undefined;
1021
+ }
1022
+ const root = modelContext.entities.find((entity) => entity.path === rootPath);
1023
+ if (!root) {
1024
+ return undefined;
1025
+ }
1026
+ const service = modelContext.executables?.find((exec) => exec.rootEntityPath === root.path)?.title;
1027
+ const columns = root.properties
1028
+ .filter((prop) => !prop.type.includes('::'))
1029
+ .map((prop) => prop.name);
1030
+ const relatedPaths = new Set();
1031
+ for (const assoc of modelContext.associations) {
1032
+ if (assoc.leftEntity === root.path) {
1033
+ relatedPaths.add(assoc.rightEntity);
1034
+ }
1035
+ else if (assoc.rightEntity === root.path) {
1036
+ relatedPaths.add(assoc.leftEntity);
1037
+ }
1038
+ }
1039
+ const related = Array.from(relatedPaths).map((path) => extractElementNameFromPath(path));
1040
+ const sentences = [];
1041
+ const via = service ? ` via the **${service}** service` : '';
1042
+ const description = root.description
1043
+ ? ` — ${root.description.slice(0, MAX_APPROACH_DESCRIPTION_LENGTH).trim()}`
1044
+ : '';
1045
+ sentences.push(`To answer this, I'll query **${root.name}**${via}${description}.`);
1046
+ if (columns.length > 0) {
1047
+ const shown = columns
1048
+ .slice(0, MAX_APPROACH_COLUMNS)
1049
+ .map((name) => `\`${name}\``)
1050
+ .join(', ');
1051
+ const more = columns.length > MAX_APPROACH_COLUMNS
1052
+ ? ` (and ${columns.length - MAX_APPROACH_COLUMNS} more)`
1053
+ : '';
1054
+ sentences.push(`It exposes fields such as ${shown}${more}.`);
1055
+ }
1056
+ if (related.length > 0) {
1057
+ const list = related.map((name) => `**${name}**`).join(', ');
1058
+ sentences.push(`Where the question needs more, results can be enriched by joining related ${related.length > 1 ? 'entities' : 'entity'} ${list} through the model's associations.`);
1059
+ }
1060
+ return sentences.join(' ');
1061
+ }
1062
+ const EXACT_STRING_FILTER_PATTERN = /(?<lhs>\$\w+(?:\.\w+)+(?:->toOne\(\))?)\s*==\s*'(?<literal>[^']*)'/g;
1063
+ export function relaxExactStringFilters(pureQuery) {
1064
+ return pureQuery.replace(EXACT_STRING_FILTER_PATTERN, (_match, lhs, literal) => `${lhs}->toLower()->contains('${literal.toLowerCase()}')`);
1065
+ }
1066
+ const STRING_COMPARISON_COLUMN_PATTERN = /\$\w+(?:\.\w+)*\.(?<col>\w+)(?:->toOne\(\))?\s*(?:==|!=|<=|>=|<|>)\s*'/g;
1067
+ const CONTAINS_COLUMN_PATTERN = /\$\w+(?:\.\w+)*\.(?<col>\w+)(?:->to\w+\(\))*->(?:contains|startsWith|endsWith)\(/g;
1068
+ export function extractFilteredColumns(pureQuery) {
1069
+ const columns = new Set();
1070
+ for (const pattern of [
1071
+ STRING_COMPARISON_COLUMN_PATTERN,
1072
+ CONTAINS_COLUMN_PATTERN,
1073
+ ]) {
1074
+ for (const match of pureQuery.matchAll(pattern)) {
1075
+ const col = match.groups?.col;
1076
+ if (col) {
1077
+ columns.add(col);
1078
+ }
1079
+ }
1080
+ }
1081
+ return Array.from(columns);
1082
+ }
1083
+ const MAX_PROBED_VALUES_PER_COLUMN = 30;
1084
+ export function buildProbedValueHints(probed) {
1085
+ const entries = [];
1086
+ for (const [column, values] of probed) {
1087
+ if (values.length === 0) {
1088
+ continue;
1089
+ }
1090
+ const shown = values.slice(0, MAX_PROBED_VALUES_PER_COLUMN).join(', ');
1091
+ entries.push({
1092
+ id: `probed_values:${column}`,
1093
+ description: `Actual values present in column '${column}': ${shown}. Filter using one of these exact values (case-insensitively).`,
1094
+ category: 'probed_values',
1095
+ });
1096
+ }
1097
+ return entries;
1098
+ }
995
1099
  // ────────────────────────────────────────────────────────────────────────────
996
1100
  // Model context enrichment for SQL / metadata prompts
997
1101
  // ────────────────────────────────────────────────────────────────────────────
@@ -1000,6 +1104,9 @@ const MAX_ENRICHMENT_PROPS_PER_ENTITY = 15;
1000
1104
  const MAX_ENRICHMENT_ENUMS = 10;
1001
1105
  const MAX_ENRICHMENT_ENUM_VALUES = 20;
1002
1106
  const MAX_ENRICHMENT_ASSOCIATIONS = 10;
1107
+ const MAX_ENRICHMENT_EXECUTABLES = 25;
1108
+ const MAX_ENRICHMENT_JOIN_SERVICES = 25;
1109
+ const MIN_SERVICE_ENTITY_COLUMN_OVERLAP = 2;
1003
1110
  /**
1004
1111
  * Maps each TDS service to the model entity whose properties best match
1005
1112
  * the service's column names. Uses a simple overlap-scoring approach:
@@ -1011,7 +1118,6 @@ const MAX_ENRICHMENT_ASSOCIATIONS = 10;
1011
1118
  */
1012
1119
  function inferServiceEntityMapping(services, modelContext) {
1013
1120
  const result = new Map();
1014
- const MIN_OVERLAP = 2; // require at least 2 matching columns
1015
1121
  const entityPropSets = modelContext.entities.map((entity) => ({
1016
1122
  path: entity.path,
1017
1123
  props: new Set(entity.properties.map((p) => p.name.toLowerCase())),
@@ -1027,7 +1133,7 @@ function inferServiceEntityMapping(services, modelContext) {
1027
1133
  bestEntity = path;
1028
1134
  }
1029
1135
  }
1030
- if (bestScore >= MIN_OVERLAP && bestEntity) {
1136
+ if (bestScore >= MIN_SERVICE_ENTITY_COLUMN_OVERLAP && bestEntity) {
1031
1137
  result.set(svc.title, bestEntity);
1032
1138
  }
1033
1139
  }
@@ -1068,7 +1174,7 @@ function buildEntitySection(modelContext) {
1068
1174
  const tagStr = tags.length > 0 ? ` [${tags.join(', ')}]` : '';
1069
1175
  lines.push(`\n### ${entity.name}${tagStr}`);
1070
1176
  if (entity.description) {
1071
- lines.push(entity.description.slice(0, 300));
1177
+ lines.push(entity.description.slice(0, MAX_DESCRIPTION_PREVIEW_LENGTH));
1072
1178
  }
1073
1179
  const propLines = entity.properties
1074
1180
  .slice(0, MAX_ENRICHMENT_PROPS_PER_ENTITY)
@@ -1113,7 +1219,7 @@ function buildExecutableSection(modelContext) {
1113
1219
  return undefined;
1114
1220
  }
1115
1221
  const lines = ['## Available Executables'];
1116
- for (const exec of modelContext.executables) {
1222
+ for (const exec of modelContext.executables.slice(0, MAX_ENRICHMENT_EXECUTABLES)) {
1117
1223
  lines.push(`\n- "${exec.title}" → ${extractElementNameFromPath(exec.rootEntityPath)}`);
1118
1224
  if (exec.description) {
1119
1225
  lines.push(` ${exec.description.slice(0, 200)}`);
@@ -1125,6 +1231,9 @@ function buildExecutableSection(modelContext) {
1125
1231
  lines.push(` Required parameters: ${paramList}`);
1126
1232
  }
1127
1233
  }
1234
+ if (modelContext.executables.length > MAX_ENRICHMENT_EXECUTABLES) {
1235
+ lines.push(`\n(${modelContext.executables.length - MAX_ENRICHMENT_EXECUTABLES} additional executables omitted)`);
1236
+ }
1128
1237
  return lines.join('\n');
1129
1238
  }
1130
1239
  function buildPropToEnumPathMap(modelContext, enumPathMap) {
@@ -1238,13 +1347,9 @@ function buildServicePairHint(svcA, entityA, svcB, entityB, modelContext, servic
1238
1347
  }
1239
1348
  const leftName = extractElementNameFromPath(assoc.leftEntity);
1240
1349
  const rightName = extractElementNameFromPath(assoc.rightEntity);
1241
- const colsA = new Set(services
1242
- .find((s) => s.title === svcA)
1243
- ?.columns.map((c) => c.name.toLowerCase()) ?? []);
1244
- const svcBCols = services.find((s) => s.title === svcB)?.columns ?? [];
1245
- const shared = svcBCols
1246
- .filter((c) => colsA.has(c.name.toLowerCase()))
1247
- .map((c) => c.name);
1350
+ const serviceA = services.find((s) => s.title === svcA);
1351
+ const serviceB = services.find((s) => s.title === svcB);
1352
+ const shared = serviceA && serviceB ? sharedColumnNames(serviceB, serviceA) : [];
1248
1353
  const joinKeyHint = shared.length > 0
1249
1354
  ? `JOIN ON shared columns: ${shared.join(', ')}`
1250
1355
  : `No shared columns by name — match on equivalent business keys (e.g. ${leftName} ID ↔ ${rightName} foreign key)`;
@@ -1281,16 +1386,21 @@ function buildServiceJoinSection(modelContext, services) {
1281
1386
  if (serviceEntityMap.size === 0) {
1282
1387
  return undefined;
1283
1388
  }
1389
+ const allEntries = Array.from(serviceEntityMap);
1390
+ const cappedEntries = allEntries.slice(0, MAX_ENRICHMENT_JOIN_SERVICES);
1284
1391
  const joinLines = [
1285
1392
  '## Model-Aware Service JOIN Guide',
1286
1393
  'Each service below is mapped to the data model entity whose properties best match its columns.',
1287
1394
  'Use the entity relationships to determine correct JOIN keys and semantics.',
1288
1395
  '',
1289
1396
  ];
1290
- for (const [svcTitle, entityPath] of serviceEntityMap) {
1397
+ for (const [svcTitle, entityPath] of cappedEntries) {
1291
1398
  joinLines.push(`- **${svcTitle}** → entity ${extractElementNameFromPath(entityPath)}`);
1292
1399
  }
1293
- const pairHints = buildServicePairHints(serviceEntityMap, modelContext, services);
1400
+ if (allEntries.length > MAX_ENRICHMENT_JOIN_SERVICES) {
1401
+ joinLines.push(`\n(${allEntries.length - MAX_ENRICHMENT_JOIN_SERVICES} additional services omitted)`);
1402
+ }
1403
+ const pairHints = buildServicePairHints(new Map(cappedEntries), modelContext, services);
1294
1404
  if (pairHints.length > 0) {
1295
1405
  joinLines.push('', '### Inter-Service Relationships', ...pairHints);
1296
1406
  }
@@ -1313,6 +1423,127 @@ export function buildModelContextEnrichmentText(modelContext, services) {
1313
1423
  return `# DATA MODEL CONTEXT
1314
1424
  The following describes the underlying data model — entities, properties, enumerations, and relationships. Use this to understand column semantics, valid filter values, and how entities relate to each other.
1315
1425
 
1426
+ ${sections.join('\n\n')}`;
1427
+ }
1428
+ const MAX_CATALOG_ENTITIES = 40;
1429
+ const MAX_CATALOG_PROPS_PER_ENTITY = 50;
1430
+ const MAX_CATALOG_ENUMS = 30;
1431
+ const MAX_CATALOG_ENUM_VALUES = 50;
1432
+ const MAX_CATALOG_ASSOCIATIONS = 40;
1433
+ const MAX_CATALOG_EXECUTABLES = 30;
1434
+ /** Formats a property's multiplicity marker from its cardinality flags. */
1435
+ function formatCatalogMultiplicity(property) {
1436
+ if (property.isCollection) {
1437
+ return '[*]';
1438
+ }
1439
+ return property.isOptional ? '[0..1]' : '[1]';
1440
+ }
1441
+ /** Builds the entities-and-columns section of the model catalog. */
1442
+ function buildCatalogEntitiesSection(modelContext) {
1443
+ if (modelContext.entities.length === 0) {
1444
+ return undefined;
1445
+ }
1446
+ const scoreOf = (e) => (e.isQueryable ? 10 : 0) + (e.isRootMapped ? 3 : 0);
1447
+ const sorted = [...modelContext.entities].sort((a, b) => scoreOf(b) - scoreOf(a));
1448
+ const lines = ['## Entities and Columns'];
1449
+ for (const entity of sorted.slice(0, MAX_CATALOG_ENTITIES)) {
1450
+ const tags = [];
1451
+ if (entity.isQueryable) {
1452
+ tags.push('queryable');
1453
+ }
1454
+ if (entity.isRootMapped) {
1455
+ tags.push('root-mapped');
1456
+ }
1457
+ const tagStr = tags.length > 0 ? ` [${tags.join(', ')}]` : '';
1458
+ lines.push(`\n### ${entity.name}${tagStr}`);
1459
+ if (entity.description) {
1460
+ lines.push(entity.description);
1461
+ }
1462
+ const propLines = entity.properties
1463
+ .slice(0, MAX_CATALOG_PROPS_PER_ENTITY)
1464
+ .map((p) => `- ${p.name}: ${p.type} ${formatCatalogMultiplicity(p)}`);
1465
+ if (propLines.length > 0) {
1466
+ lines.push('Columns:', ...propLines);
1467
+ }
1468
+ if (entity.properties.length > MAX_CATALOG_PROPS_PER_ENTITY) {
1469
+ lines.push(`- (${entity.properties.length - MAX_CATALOG_PROPS_PER_ENTITY} more columns)`);
1470
+ }
1471
+ }
1472
+ if (modelContext.entities.length > MAX_CATALOG_ENTITIES) {
1473
+ lines.push(`\n(${modelContext.entities.length - MAX_CATALOG_ENTITIES} additional entities omitted — ask about a specific one for its columns)`);
1474
+ }
1475
+ return lines.join('\n');
1476
+ }
1477
+ /** Builds the relationships section of the model catalog. */
1478
+ function buildCatalogRelationshipsSection(modelContext) {
1479
+ if (modelContext.associations.length === 0) {
1480
+ return undefined;
1481
+ }
1482
+ const lines = ['## Relationships'];
1483
+ for (const assoc of modelContext.associations.slice(0, MAX_CATALOG_ASSOCIATIONS)) {
1484
+ lines.push(`- ${assoc.leftEntity}.${assoc.leftProperty} <-> ${assoc.rightEntity}.${assoc.rightProperty}`);
1485
+ }
1486
+ if (modelContext.associations.length > MAX_CATALOG_ASSOCIATIONS) {
1487
+ lines.push(`- (${modelContext.associations.length - MAX_CATALOG_ASSOCIATIONS} more relationships)`);
1488
+ }
1489
+ return lines.join('\n');
1490
+ }
1491
+ /** Builds the enumerations (valid values) section of the model catalog. */
1492
+ function buildCatalogEnumerationsSection(modelContext) {
1493
+ const enumerations = modelContext.enumerations ?? [];
1494
+ if (enumerations.length === 0) {
1495
+ return undefined;
1496
+ }
1497
+ const lines = ['## Enumerations (valid values)'];
1498
+ for (const en of enumerations.slice(0, MAX_CATALOG_ENUMS)) {
1499
+ const shown = en.values.slice(0, MAX_CATALOG_ENUM_VALUES).join(', ');
1500
+ const more = en.values.length > MAX_CATALOG_ENUM_VALUES
1501
+ ? `, ... (${en.values.length} total)`
1502
+ : '';
1503
+ lines.push(`- ${en.name}: ${shown}${more}`);
1504
+ }
1505
+ return lines.join('\n');
1506
+ }
1507
+ /** Builds the available-executables section of the model catalog. */
1508
+ function buildCatalogExecutablesSection(modelContext) {
1509
+ const executables = modelContext.executables ?? [];
1510
+ if (executables.length === 0) {
1511
+ return undefined;
1512
+ }
1513
+ const lines = ['## How to query (available executables)'];
1514
+ for (const exec of executables.slice(0, MAX_CATALOG_EXECUTABLES)) {
1515
+ const root = exec.rootEntityPath
1516
+ ? ` → ${extractElementNameFromPath(exec.rootEntityPath)}`
1517
+ : '';
1518
+ lines.push(`\n- **${exec.title}**${root}`);
1519
+ if (exec.description) {
1520
+ lines.push(` ${exec.description.slice(0, MAX_DESCRIPTION_PREVIEW_LENGTH)}`);
1521
+ }
1522
+ if (exec.requiredParameters && exec.requiredParameters.length > 0) {
1523
+ const paramList = exec.requiredParameters
1524
+ .map((p) => `${p.name} (${p.type})`)
1525
+ .join(', ');
1526
+ lines.push(` Required parameters: ${paramList}`);
1527
+ }
1528
+ }
1529
+ if (executables.length > MAX_CATALOG_EXECUTABLES) {
1530
+ lines.push(`\n(${executables.length - MAX_CATALOG_EXECUTABLES} more)`);
1531
+ }
1532
+ return lines.join('\n');
1533
+ }
1534
+ export function buildModelCatalogText(modelContext) {
1535
+ const sections = [
1536
+ buildCatalogEntitiesSection(modelContext),
1537
+ buildCatalogRelationshipsSection(modelContext),
1538
+ buildCatalogEnumerationsSection(modelContext),
1539
+ buildCatalogExecutablesSection(modelContext),
1540
+ ].filter(isNonNullable);
1541
+ if (sections.length === 0) {
1542
+ return undefined;
1543
+ }
1544
+ return `# MODEL REFERENCE
1545
+ A reference for this data product's model. Use it to explain the entities and their columns, how entities relate, valid enumeration values, and how to query the data. Give a thorough, well-structured answer.
1546
+
1316
1547
  ${sections.join('\n\n')}`;
1317
1548
  }
1318
1549
  //# sourceMappingURL=LegendAIDocEnrichment.js.map