@memberjunction/core-entities-server 5.16.0 → 5.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/dist/custom/MJQueryEntityServer.server.d.ts +12 -91
  2. package/dist/custom/MJQueryEntityServer.server.d.ts.map +1 -1
  3. package/dist/custom/MJQueryEntityServer.server.js +69 -1097
  4. package/dist/custom/MJQueryEntityServer.server.js.map +1 -1
  5. package/dist/custom/query-extraction/dialect.d.ts +23 -0
  6. package/dist/custom/query-extraction/dialect.d.ts.map +1 -0
  7. package/dist/custom/query-extraction/dialect.js +45 -0
  8. package/dist/custom/query-extraction/dialect.js.map +1 -0
  9. package/dist/custom/query-extraction/enrich.d.ts +51 -0
  10. package/dist/custom/query-extraction/enrich.d.ts.map +1 -0
  11. package/dist/custom/query-extraction/enrich.js +180 -0
  12. package/dist/custom/query-extraction/enrich.js.map +1 -0
  13. package/dist/custom/query-extraction/index.d.ts +9 -0
  14. package/dist/custom/query-extraction/index.d.ts.map +1 -0
  15. package/dist/custom/query-extraction/index.js +15 -0
  16. package/dist/custom/query-extraction/index.js.map +1 -0
  17. package/dist/custom/query-extraction/parse.d.ts +7 -0
  18. package/dist/custom/query-extraction/parse.d.ts.map +1 -0
  19. package/dist/custom/query-extraction/parse.js +13 -0
  20. package/dist/custom/query-extraction/parse.js.map +1 -0
  21. package/dist/custom/query-extraction/pipeline.d.ts +24 -0
  22. package/dist/custom/query-extraction/pipeline.d.ts.map +1 -0
  23. package/dist/custom/query-extraction/pipeline.js +101 -0
  24. package/dist/custom/query-extraction/pipeline.js.map +1 -0
  25. package/dist/custom/query-extraction/resolve.d.ts +110 -0
  26. package/dist/custom/query-extraction/resolve.d.ts.map +1 -0
  27. package/dist/custom/query-extraction/resolve.js +677 -0
  28. package/dist/custom/query-extraction/resolve.js.map +1 -0
  29. package/dist/custom/query-extraction/sync.d.ts +30 -0
  30. package/dist/custom/query-extraction/sync.d.ts.map +1 -0
  31. package/dist/custom/query-extraction/sync.js +456 -0
  32. package/dist/custom/query-extraction/sync.js.map +1 -0
  33. package/dist/custom/query-extraction/types.d.ts +114 -0
  34. package/dist/custom/query-extraction/types.d.ts.map +1 -0
  35. package/dist/custom/query-extraction/types.js +2 -0
  36. package/dist/custom/query-extraction/types.js.map +1 -0
  37. package/package.json +19 -19
@@ -4,16 +4,11 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
4
4
  else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
5
  return c > 3 && r && Object.defineProperty(target, key, r), r;
6
6
  };
7
- import { BaseEntity, LogError, Metadata, TypeScriptTypeFromSQLType } from "@memberjunction/core";
8
- import { QueryCompositionEngine } from "@memberjunction/generic-database-provider";
7
+ import { BaseEntity, LogError, Metadata, } from "@memberjunction/core";
9
8
  import { MJQueryEntity } from "@memberjunction/core-entities";
10
9
  import { RegisterClass, MJGlobal, UUIDsEqual } from "@memberjunction/global";
11
- import { AIEngine } from "@memberjunction/aiengine";
12
- import { AIPromptRunner } from "@memberjunction/ai-prompts";
13
- import { AIPromptParams } from "@memberjunction/ai-core-plus";
14
- import { removeNPrefix, convertIdentifiers, convertCommonFunctions, convertDateFunctions, convertCharIndex, convertStuff, convertIIF, convertConvertFunction, convertCastTypes, convertTopToLimit, convertStringConcat, removeCollate, } from "@memberjunction/sql-converter";
15
10
  import { EmbedTextLocalHelper } from "./util.js";
16
- import { SQLParser } from "@memberjunction/sql-parser";
11
+ import { RunExtractionPipeline, CleanupQueryData, ConvertTSQLToPostgreSQL, } from "./query-extraction/index.js";
17
12
  let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
18
13
  constructor() {
19
14
  super(...arguments);
@@ -34,10 +29,9 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
34
29
  get QueryPermissions() {
35
30
  return this._queryPermissions;
36
31
  }
32
+ // ─── Embedding Methods ───────────────────────────────────────────────────────
37
33
  /**
38
34
  * Simple proxy to local helper method for embeddings. Needed for BaseEntity sub-classes that want to use embeddings built into BaseEntity
39
- * @param textToEmbed
40
- * @returns
41
35
  */
42
36
  async EmbedTextLocal(textToEmbed) {
43
37
  return EmbedTextLocalHelper(this, textToEmbed);
@@ -64,51 +58,38 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
64
58
  this.EmbeddingModelID = result.modelID;
65
59
  }
66
60
  }
61
+ // ─── Save / Delete Overrides ─────────────────────────────────────────────────
67
62
  async Save(options) {
68
63
  try {
69
- // Check if this is a new record or if SQL/Description has changed
70
64
  const sqlField = this.GetFieldByName('SQL');
71
65
  const nameField = this.GetFieldByName('Name');
72
66
  const descriptionField = this.GetFieldByName('Description');
73
67
  const userQuestionField = this.GetFieldByName('UserQuestion');
74
68
  const shouldExtractData = !this.IsSaved || sqlField.Dirty;
75
69
  const shouldGenerateEmbedding = !this.IsSaved || nameField.Dirty || descriptionField.Dirty || userQuestionField.Dirty;
76
- // Generate embedding from composite text (Name + UserQuestion + Description) for better semantic search
70
+ // Generate embedding from composite text for better semantic search
77
71
  if (shouldGenerateEmbedding) {
78
72
  await this.GenerateCompositeEmbedding();
79
73
  }
80
74
  else if (!this.Description || this.Description.trim().length === 0) {
81
- // Clear embedding if description is empty
82
75
  this.EmbeddingVector = null;
83
76
  this.EmbeddingModelID = null;
84
77
  }
85
- // Auto-set Reusable=true for queries in Golden-Queries/ categories.
86
- // Golden Queries are composable building blocks, so they should always be reusable.
87
- if (this.CategoryID) {
88
- const categoryPath = this.buildCategoryPathFromID(this.CategoryID);
89
- if (categoryPath.toLowerCase().startsWith('golden-queries/') || categoryPath.toLowerCase() === 'golden-queries') {
90
- this.Reusable = true;
91
- }
92
- }
93
- // Save the query first without AI processing (no transaction needed for basic save)
78
+ // Save the query first without AI processing
94
79
  const saveResult = await super.Save(options);
95
80
  if (!saveResult) {
96
81
  return false;
97
82
  }
98
83
  // Extract and sync parameters AFTER saving, outside of any transaction
99
- // This prevents connection pool exhaustion from long-running AI operations
100
84
  if (shouldExtractData && this.SQL && this.SQL.trim().length > 0) {
101
85
  await this.extractAndSyncDataAsync();
102
- // Auto-convert SQL to other dialects if configured
103
86
  await this.autoConvertDialectsAsync();
104
- await this.RefreshRelatedMetadata(true); // sync the related metadata so this entity is correct
87
+ await this.RefreshRelatedMetadata(true);
105
88
  }
106
89
  else if (!this.SQL || this.SQL.trim().length === 0) {
107
- // If SQL is empty, ensure UsesTemplate is false and remove all related data
108
- // This can also happen asynchronously since it's cleanup work
109
90
  this.UsesTemplate = false;
110
91
  await this.cleanupEmptyQueryAsync();
111
- await this.RefreshRelatedMetadata(true); // sync the related metadata so this entity is correct
92
+ await this.RefreshRelatedMetadata(true);
112
93
  }
113
94
  return true;
114
95
  }
@@ -120,14 +101,11 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
120
101
  }
121
102
  async Delete(options) {
122
103
  try {
123
- // Perform the actual delete operation
124
104
  const deleteResult = await super.Delete(options);
125
105
  if (!deleteResult) {
126
106
  return false;
127
107
  }
128
- // CRITICAL: Refresh metadata cache after deletion to prevent stale query references
129
- // This ensures that MJAPI and any other services using cached metadata
130
- // immediately see that this query no longer exists
108
+ // Refresh metadata cache after deletion to prevent stale query references
131
109
  await this.RefreshRelatedMetadata(true);
132
110
  return true;
133
111
  }
@@ -137,11 +115,12 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
137
115
  return false;
138
116
  }
139
117
  }
118
+ // ─── Pipeline Delegation ─────────────────────────────────────────────────────
140
119
  /**
141
120
  * Builds a category path string from a CategoryID by walking up the category hierarchy.
142
- * Returns a path like "Golden-Queries/Sales" or "Golden-Queries".
121
+ * Returns a slash-delimited path like "Ground-Truth-Queries/Sales" or "Ground-Truth-Queries".
143
122
  */
144
- buildCategoryPathFromID(categoryID) {
123
+ BuildCategoryPathFromID(categoryID) {
145
124
  const categories = Metadata.Provider.QueryCategories;
146
125
  const segments = [];
147
126
  let currentID = categoryID;
@@ -155,14 +134,29 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
155
134
  return segments.join('/');
156
135
  }
157
136
  /**
158
- * Asynchronous version of extractAndSyncData that runs outside the main save operation
159
- * to prevent connection pool exhaustion
137
+ * Builds the QuerySyncContext from this entity instance for use by the extraction pipeline.
138
+ */
139
+ buildSyncContext() {
140
+ return {
141
+ queryID: this.ID,
142
+ queryName: this.Name,
143
+ sql: this.SQL,
144
+ isSaved: this.IsSaved,
145
+ contextUser: this.ContextCurrentUser,
146
+ metadataProvider: this.ProviderToUse,
147
+ runViewProvider: this.RunViewProviderToUse,
148
+ };
149
+ }
150
+ /**
151
+ * Runs the extraction pipeline and saves the UsesTemplate flag.
152
+ * On error, sets UsesTemplate=false and attempts to save so the query remains usable.
160
153
  */
161
154
  async extractAndSyncDataAsync() {
162
155
  try {
163
- await this.extractAndSyncData();
164
- // Save the query again to update the UsesTemplate flag and any changes from AI processing
165
- // This is a separate, fast operation that doesn't involve AI
156
+ const ctx = this.buildSyncContext();
157
+ const pipelineResult = await RunExtractionPipeline(ctx);
158
+ this.UsesTemplate = pipelineResult.usesTemplate;
159
+ // Save the query again to persist UsesTemplate and any changes from AI processing
166
160
  const updateResult = await super.Save();
167
161
  if (!updateResult) {
168
162
  LogError('Failed to save query after AI processing completed');
@@ -170,8 +164,6 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
170
164
  }
171
165
  catch (e) {
172
166
  LogError('Error in async AI processing:', e);
173
- // Set UsesTemplate to false on error and save
174
- // This ensures the query is still usable even if AI extraction fails
175
167
  this.UsesTemplate = false;
176
168
  try {
177
169
  await super.Save();
@@ -182,17 +174,14 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
182
174
  }
183
175
  }
184
176
  /**
185
- * Asynchronous cleanup for empty queries
177
+ * Cleans up all extraction data when SQL is empty, then saves the updated UsesTemplate flag.
186
178
  */
187
179
  async cleanupEmptyQueryAsync() {
188
180
  try {
189
- await Promise.all([
190
- this.removeAllQueryParameters(),
191
- this.removeAllQueryFields(),
192
- this.removeAllQueryEntities(),
193
- this.removeAllQuerySQLRecordsAsync()
194
- ]);
195
- // Save the updated UsesTemplate flag
181
+ const ctx = this.buildSyncContext();
182
+ await CleanupQueryData(ctx);
183
+ // Also remove dialect records (not managed by the pipeline)
184
+ await this.removeAllQuerySQLRecordsAsync();
196
185
  const updateResult = await super.Save();
197
186
  if (!updateResult) {
198
187
  LogError('Failed to save query after cleanup');
@@ -202,1081 +191,66 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
202
191
  LogError('Error in async cleanup:', e);
203
192
  }
204
193
  }
205
- async extractAndSyncData() {
206
- try {
207
- if (!this.SQL) {
208
- this.UsesTemplate = false;
209
- return;
210
- }
211
- // ── Deterministic analysis via SQLParser (source of truth) ──
212
- const analysis = SQLParser.Analyze(this.SQL);
213
- const deterministicParams = SQLParser.ExtractParameterInfo(this.SQL);
214
- const hasTemplateParameters = analysis.hasTemplateExpressions && deterministicParams.length > 0;
215
- // Extract entity metadata deterministically (for LLM context and entity sync)
216
- const entityMetadata = await this.extractEntityMetadataFromSQL();
217
- // ── LLM enrichment (descriptions and sample values only) ──
218
- const llmResult = await this.runLLMEnrichment(entityMetadata);
219
- // ── Merge: deterministic structure + LLM descriptions ──
220
- const syncPromises = [];
221
- // Parameters: SQLParser provides name, type, isRequired, defaultValue.
222
- // LLM provides description and sampleValue as enrichment.
223
- if (hasTemplateParameters) {
224
- const enrichedParams = this.mergeParametersWithLLM(deterministicParams, llmResult);
225
- syncPromises.push(this.syncQueryParameters(enrichedParams));
226
- }
227
- else {
228
- syncPromises.push(this.removeAllQueryParameters());
229
- }
230
- // Fields: deterministic for SELECT *, LLM for explicit column lists
231
- const selectStarFields = this.buildFieldsForSelectStar();
232
- if (selectStarFields) {
233
- syncPromises.push(this.syncQueryFields(selectStarFields));
234
- }
235
- else if (llmResult?.selectClause && Array.isArray(llmResult.selectClause) && llmResult.selectClause.length > 0) {
236
- syncPromises.push(this.syncQueryFields(llmResult.selectClause));
237
- }
238
- // Entities: deterministic SQL parsing (more reliable than LLM)
239
- if (entityMetadata.length > 0) {
240
- syncPromises.push(this.syncQueryEntities(entityMetadata));
241
- }
242
- // Dependencies: deterministic composition token parsing (no LLM)
243
- syncPromises.push(this.extractAndSyncQueryDependencies());
244
- await Promise.all(syncPromises);
245
- this.UsesTemplate = hasTemplateParameters;
246
- }
247
- catch (e) {
248
- LogError(`Query "${this.Name}" extraction error:`, e);
249
- this.UsesTemplate = false;
250
- }
251
- }
252
- /**
253
- * Runs the LLM enrichment prompt to get descriptions and sample values.
254
- * Returns null if the LLM is unavailable or fails (non-fatal).
255
- */
256
- async runLLMEnrichment(entityMetadata) {
257
- try {
258
- await AIEngine.Instance.Config(false, this.ContextCurrentUser, this.ProviderToUse);
259
- const aiPrompt = AIEngine.Instance.Prompts.find(p => p.Name === 'SQL Query Parameter Extraction' &&
260
- p.Category === 'MJ: System');
261
- if (!aiPrompt) {
262
- console.warn('AI prompt for SQL Query Parameter Extraction not found. Descriptions will use heuristics.');
263
- return null;
264
- }
265
- const promptRunner = new AIPromptRunner();
266
- const params = new AIPromptParams();
267
- params.prompt = aiPrompt;
268
- params.data = { templateText: this.SQL, entities: entityMetadata };
269
- params.contextUser = this.ContextCurrentUser;
270
- const result = await promptRunner.ExecutePrompt(params);
271
- if (!result.success || !result.result) {
272
- console.warn(`Query "${this.Name}" - LLM enrichment failed, using heuristic descriptions.`);
273
- return null;
274
- }
275
- return result.result;
276
- }
277
- catch (e) {
278
- console.warn(`Query "${this.Name}" - LLM enrichment error, using heuristic descriptions:`, e);
279
- return null;
280
- }
281
- }
282
- /**
283
- * Merges deterministic parameter extraction (SQLParser) with LLM enrichment.
284
- *
285
- * SQLParser is the authority for: name, type, isRequired, defaultValue
286
- * LLM provides: description, sampleValue (with heuristic fallbacks if LLM unavailable)
287
- */
288
- mergeParametersWithLLM(deterministicParams, llmResult) {
289
- const llmParams = llmResult?.parameters ?? [];
290
- return deterministicParams.map(dp => {
291
- // Find matching LLM parameter by name (case-insensitive)
292
- const llmMatch = llmParams.find(lp => lp.name.toLowerCase() === dp.name.toLowerCase());
293
- return {
294
- name: dp.name,
295
- type: dp.type === 'unknown' ? (llmMatch?.type ?? 'string') : dp.type,
296
- isRequired: dp.isRequired,
297
- description: llmMatch?.description ?? this.generateParameterDescription(dp),
298
- usage: llmMatch?.usage ?? dp.usageLocations,
299
- defaultValue: dp.defaultValue !== null ? String(dp.defaultValue) : (llmMatch?.defaultValue ?? null),
300
- sampleValue: llmMatch?.sampleValue ?? this.generateSampleValue(dp),
301
- };
302
- });
303
- }
304
- /**
305
- * Generates a deterministic description for a parameter when LLM is unavailable.
306
- * Uses the parameter name, type, and filter information to create a meaningful description.
307
- */
308
- generateParameterDescription(param) {
309
- const typeDescriptions = {
310
- 'string': 'text value',
311
- 'number': 'numeric value',
312
- 'date': 'date value',
313
- 'boolean': 'true/false flag',
314
- 'array': 'list of values',
315
- };
316
- const typeDesc = typeDescriptions[param.type] || 'value';
317
- const requiredDesc = param.isRequired ? 'Required' : 'Optional';
318
- const humanName = param.name.replace(/([A-Z])/g, ' $1').trim();
319
- const defaultDesc = param.defaultValue !== null ? ` (default: ${param.defaultValue})` : '';
320
- return `${requiredDesc} ${typeDesc} for ${humanName}${defaultDesc}`;
321
- }
322
- /**
323
- * Generates a deterministic sample value based on the parameter type.
324
- */
325
- generateSampleValue(param) {
326
- if (param.defaultValue !== null)
327
- return String(param.defaultValue);
328
- switch (param.type) {
329
- case 'string': return 'Example';
330
- case 'number': return '10';
331
- case 'date': return '2024-01-01';
332
- case 'boolean': return 'true';
333
- case 'array': return 'Value1,Value2';
334
- default: return null;
335
- }
336
- }
337
- /**
338
- * Extracts entity metadata from the SQL to provide context to the LLM for parameter type inference.
339
- * Uses SQLParser for robust SQL parsing with MJ template support.
340
- */
341
- async extractEntityMetadataFromSQL() {
342
- const md = this.ProviderToUse;
343
- const results = [];
344
- if (!this.SQL)
345
- return results;
346
- try {
347
- const tableRefs = SQLParser.ExtractTableRefs(this.SQL);
348
- const columnRefs = SQLParser.ExtractColumnRefs(this.SQL);
349
- for (const tableRef of tableRefs) {
350
- const matchingEntity = md.Entities.find(e => (e.BaseView.toLowerCase() === tableRef.TableName.toLowerCase() ||
351
- e.BaseTable.toLowerCase() === tableRef.TableName.toLowerCase()) &&
352
- e.SchemaName.toLowerCase() === tableRef.SchemaName.toLowerCase());
353
- if (matchingEntity) {
354
- const relevantFields = matchingEntity.Fields
355
- .filter(f => {
356
- const fieldLower = f.Name.toLowerCase();
357
- for (const colRef of columnRefs) {
358
- const colLower = colRef.ColumnName.toLowerCase();
359
- if (colLower === fieldLower)
360
- return true;
361
- if (colRef.TableQualifier) {
362
- const qualLower = colRef.TableQualifier.toLowerCase();
363
- if ((qualLower === tableRef.Alias.toLowerCase() ||
364
- qualLower === tableRef.TableName.toLowerCase()) &&
365
- colLower === fieldLower) {
366
- return true;
367
- }
368
- }
369
- }
370
- return f.IsPrimaryKey;
371
- })
372
- .slice(0, 20)
373
- .map(f => ({
374
- name: f.Name,
375
- type: f.Type,
376
- isPrimaryKey: f.IsPrimaryKey
377
- }));
378
- if (relevantFields.length > 0) {
379
- results.push({
380
- name: matchingEntity.Name,
381
- schemaName: matchingEntity.SchemaName,
382
- baseView: matchingEntity.BaseView,
383
- fields: relevantFields
384
- });
385
- }
386
- }
387
- }
388
- return results;
389
- }
390
- catch (error) {
391
- console.warn(`Error in extractEntityMetadataFromSQL: ${error}`);
392
- return results;
393
- }
394
- }
395
- async syncQueryParameters(extractedParams) {
396
- // Filter out any parameter named "query" that came from {{query:"..."}} composition tokens.
397
- // The AI extraction prompt should skip these, but this is a safety net.
398
- const compositionTokenRegex = /\{\{query:"[^"]+"\}\}/;
399
- const filteredParams = extractedParams.filter(p => {
400
- if (p.name === 'query' && this.SQL && compositionTokenRegex.test(this.SQL)) {
401
- return false;
402
- }
403
- return true;
404
- });
405
- extractedParams = filteredParams;
406
- // Use the entity's provider instead of creating new Metadata instance
407
- // Use same casting pattern as RefreshRelatedMetadata method
408
- const md = this.ProviderToUse;
409
- try {
410
- // Get existing query parameters
411
- const rv = this.RunViewProviderToUse;
412
- const existingParams = [];
413
- if (this.IsSaved) {
414
- const existingParamsResult = await rv.RunView({
415
- EntityName: 'MJ: Query Parameters',
416
- ExtraFilter: `QueryID='${this.ID}'`,
417
- ResultType: 'entity_object'
418
- }, this.ContextCurrentUser);
419
- if (!existingParamsResult.Success) {
420
- throw new Error(`Failed to load existing query parameters: ${existingParamsResult.ErrorMessage}`);
421
- }
422
- existingParams.push(...existingParamsResult.Results || []);
423
- }
424
- // Convert extracted param names to lowercase for comparison
425
- const extractedParamNames = extractedParams.map(p => p.name.toLowerCase());
426
- // Find parameters to add, update, or remove
427
- const paramsToAdd = extractedParams.filter(p => !existingParams.some(ep => ep.Name.toLowerCase() === p.name.toLowerCase()));
428
- const paramsToUpdate = existingParams.filter(ep => extractedParams.some(p => p.name.toLowerCase() === ep.Name.toLowerCase()));
429
- const paramsToRemove = existingParams.filter(ep => !extractedParamNames.includes(ep.Name.toLowerCase()));
430
- // Prepare all save/delete operations
431
- const promises = [];
432
- // Add new parameters
433
- for (const param of paramsToAdd) {
434
- const newParam = await md.GetEntityObject('MJ: Query Parameters', this.ContextCurrentUser);
435
- newParam.QueryID = this.ID;
436
- newParam.Name = param.name;
437
- // Normalize type to lowercase for case-insensitive matching
438
- const normalizedType = param.type?.toLowerCase();
439
- switch (normalizedType) {
440
- case "array":
441
- case "boolean":
442
- case "string":
443
- case "date":
444
- case "number":
445
- newParam.Type = normalizedType;
446
- break;
447
- case "object":
448
- // Object type is supported in Nunjucks/RunQuery but not yet in database schema
449
- // Store as string for now - the actual validation happens at runtime
450
- console.log(`Query "${this.Name}" - Parameter "${param.name}" is type "object", storing as "string" (runtime will handle object validation)`);
451
- newParam.Type = 'string';
452
- break;
453
- default:
454
- console.warn(`Query "${this.Name}" - Unknown parameter type "${param.type}" for parameter "${param.name}", defaulting to "string"`);
455
- newParam.Type = 'string';
456
- }
457
- newParam.IsRequired = param.isRequired;
458
- newParam.DefaultValue = param.defaultValue;
459
- newParam.Description = param.description;
460
- newParam.SampleValue = param.sampleValue;
461
- newParam.DetectionMethod = 'AI'; // Structure from SQLParser AST, descriptions enriched by LLM
462
- promises.push(newParam.Save());
463
- }
464
- // Update existing parameters if properties changed
465
- for (const existingParam of paramsToUpdate) {
466
- const extractedParam = extractedParams.find(p => p.name.toLowerCase() === existingParam.Name.toLowerCase());
467
- if (extractedParam) {
468
- let hasChanges = false;
469
- // Normalize type to lowercase for case-insensitive comparison
470
- const normalizedType = extractedParam.type?.toLowerCase();
471
- const validTypes = ['array', 'boolean', 'string', 'date', 'number'];
472
- const isValidType = validTypes.includes(normalizedType);
473
- let targetType;
474
- if (isValidType) {
475
- targetType = normalizedType;
476
- }
477
- else if (normalizedType === 'object') {
478
- // Object type is supported in Nunjucks/RunQuery but not yet in database schema
479
- console.log(`Query "${this.Name}" - Parameter "${extractedParam.name}" is type "object", storing as "string" (runtime will handle object validation)`);
480
- targetType = 'string';
481
- }
482
- else {
483
- console.warn(`Query "${this.Name}" - Unknown parameter type "${extractedParam.type}" for parameter "${extractedParam.name}", defaulting to "string"`);
484
- targetType = 'string';
485
- }
486
- // Check each property for changes
487
- if (existingParam.Type !== targetType) {
488
- existingParam.Type = targetType;
489
- hasChanges = true;
490
- }
491
- if (existingParam.IsRequired !== extractedParam.isRequired) {
492
- existingParam.IsRequired = extractedParam.isRequired;
493
- hasChanges = true;
494
- }
495
- if (existingParam.DefaultValue !== extractedParam.defaultValue) {
496
- existingParam.DefaultValue = extractedParam.defaultValue;
497
- hasChanges = true;
498
- }
499
- if (existingParam.Description !== extractedParam.description) {
500
- existingParam.Description = extractedParam.description;
501
- hasChanges = true;
502
- }
503
- if (existingParam.SampleValue !== extractedParam.sampleValue) {
504
- existingParam.SampleValue = extractedParam.sampleValue;
505
- hasChanges = true;
506
- }
507
- if (existingParam.DetectionMethod !== 'AI') {
508
- existingParam.DetectionMethod = 'AI';
509
- hasChanges = true;
510
- }
511
- if (hasChanges) {
512
- promises.push(existingParam.Save());
513
- }
514
- }
515
- }
516
- // Remove parameters that are no longer in the SQL
517
- for (const paramToRemove of paramsToRemove) {
518
- promises.push(paramToRemove.Delete());
519
- }
520
- // Execute all operations in parallel
521
- if (promises.length > 0) {
522
- await Promise.all(promises);
523
- }
524
- }
525
- catch (e) {
526
- LogError(`Query "${this.Name}" - Failed to sync parameters:`, e);
527
- throw e;
528
- }
529
- }
530
- async removeAllQueryParameters() {
531
- try {
532
- if (this.IsSaved) {
533
- // Get all existing query parameters
534
- const rv = this.RunViewProviderToUse;
535
- const existingParamsResult = await rv.RunView({
536
- EntityName: 'MJ: Query Parameters',
537
- ExtraFilter: `QueryID='${this.ID}'`,
538
- ResultType: 'entity_object'
539
- }, this.ContextCurrentUser);
540
- if (!existingParamsResult.Success) {
541
- throw new Error(`Failed to load existing query parameters: ${existingParamsResult.ErrorMessage}`);
542
- }
543
- const existingParams = existingParamsResult.Results || [];
544
- // Delete all existing parameters
545
- const deletePromises = existingParams.map(param => param.Delete());
546
- if (deletePromises.length > 0) {
547
- await Promise.all(deletePromises);
548
- }
549
- }
550
- }
551
- catch (e) {
552
- LogError('Failed to remove query parameters:', e);
553
- throw e; // Re-throw since we're in a transaction
554
- }
555
- }
556
- /**
557
- * Detects if the SQL uses SELECT * and if so, builds the complete field list deterministically
558
- * from entity metadata without relying on the AI (which can't enumerate * columns).
559
- * Returns null if the SQL doesn't use SELECT * or if entities can't be resolved.
560
- */
561
- buildFieldsForSelectStar() {
562
- if (!this.SQL)
563
- return null;
564
- // Check if SELECT clause contains * (possibly with table alias like "a.*")
565
- const selectStarRegex = /\bSELECT\s+(?:(?:TOP\s+\d+|DISTINCT)\s+)?(\*|[\w.]+\.\*)\s+FROM\b/i;
566
- if (!selectStarRegex.test(this.SQL))
567
- return null;
568
- const md = this.ProviderToUse;
569
- try {
570
- const tableRefs = SQLParser.ExtractTableRefs(this.SQL);
571
- const expandedFields = [];
572
- for (const tableRef of tableRefs) {
573
- const matchingEntity = md.Entities.find(e => (e.BaseView.toLowerCase() === tableRef.TableName.toLowerCase() ||
574
- e.BaseTable.toLowerCase() === tableRef.TableName.toLowerCase()) &&
575
- e.SchemaName.toLowerCase() === tableRef.SchemaName.toLowerCase());
576
- if (matchingEntity) {
577
- for (const entityField of matchingEntity.Fields) {
578
- if (!expandedFields.some(f => f.name === entityField.Name)) {
579
- expandedFields.push({
580
- name: entityField.Name,
581
- description: entityField.Description || `${entityField.Name} field from ${matchingEntity.Name}`,
582
- type: TypeScriptTypeFromSQLType(entityField.Type).toLowerCase(),
583
- optional: false,
584
- sourceEntity: matchingEntity.Name,
585
- sourceFieldName: entityField.Name,
586
- isComputed: false,
587
- isSummary: false
588
- });
589
- }
590
- }
591
- }
592
- }
593
- // Also check for {{query:"..."}} composition tokens in FROM clause
594
- if (expandedFields.length === 0) {
595
- const compositionRefs = SQLParser.ExtractCompositionRefs(this.SQL);
596
- for (const ref of compositionRefs) {
597
- const referencedQuery = Metadata.Provider.Queries.find(q => q.Name.toLowerCase() === ref.queryName.toLowerCase());
598
- if (referencedQuery && referencedQuery.Fields.length > 0) {
599
- for (const qField of referencedQuery.Fields) {
600
- if (!expandedFields.some(f => f.name === qField.Name)) {
601
- expandedFields.push({
602
- name: qField.Name,
603
- description: qField.Description || `${qField.Name} from query "${referencedQuery.Name}"`,
604
- type: TypeScriptTypeFromSQLType(qField.SQLBaseType).toLowerCase(),
605
- optional: false,
606
- sourceEntity: qField.SourceEntityID ? md.Entities.find(e => UUIDsEqual(e.ID, qField.SourceEntityID))?.Name || null : null,
607
- sourceFieldName: qField.SourceFieldName || null,
608
- isComputed: qField.IsComputed || false,
609
- isSummary: qField.IsSummary || false
610
- });
611
- }
612
- }
613
- }
614
- }
615
- }
616
- return expandedFields.length > 0 ? expandedFields : null;
617
- }
618
- catch (error) {
619
- console.warn(`Query "${this.Name}" - Error in buildFieldsForSelectStar:`, error);
620
- return null;
621
- }
622
- }
623
- /**
624
- * Finds an entity by name, checking both Name (which may have "MJ: " prefix) and DisplayName.
625
- * The AI often returns entity names without the "MJ: " prefix, so we need to match both.
626
- */
627
- findEntityByName(md, name) {
628
- const lower = name.toLowerCase();
629
- return md.Entities.find(e => e.Name.toLowerCase() === lower || (e.DisplayName && e.DisplayName.toLowerCase() === lower));
630
- }
631
- /**
632
- * Expands wildcard (*) entries in the extracted fields list.
633
- * When AI returns a field with name="*" or sourceFieldName="*", we expand it into
634
- * individual field entries by looking up the source from metadata.
635
- * If the AI didn't provide a sourceEntity, we parse the SQL's FROM clause to find it.
636
- */
637
- expandWildcardFields(extractedFields, md) {
638
- // Check if any field is a wildcard that needs expansion
639
- const hasWildcard = extractedFields.some(f => f.name === '*' || f.sourceFieldName === '*');
640
- if (!hasWildcard)
641
- return extractedFields;
642
- const expandedFields = [];
643
- for (const field of extractedFields) {
644
- const isWildcard = field.name === '*' || field.sourceFieldName === '*';
645
- if (!isWildcard) {
646
- expandedFields.push(field);
647
- continue;
648
- }
649
- // If we have a sourceEntity, try to resolve from it
650
- if (field.sourceEntity) {
651
- const sourceEntityInfo = this.findEntityByName(md, field.sourceEntity);
652
- if (sourceEntityInfo) {
653
- for (const entityField of sourceEntityInfo.Fields) {
654
- expandedFields.push({
655
- name: entityField.Name,
656
- description: entityField.Description || `${entityField.Name} field from ${sourceEntityInfo.Name}`,
657
- type: TypeScriptTypeFromSQLType(entityField.Type).toLowerCase(),
658
- optional: field.optional,
659
- sourceEntity: sourceEntityInfo.Name,
660
- sourceFieldName: entityField.Name,
661
- isComputed: false,
662
- isSummary: false
663
- });
664
- }
665
- continue;
666
- }
667
- // sourceEntity didn't match an entity — try as a query name
668
- const composedQuery = Metadata.Provider.Queries.find(q => q.Name.toLowerCase() === field.sourceEntity.toLowerCase());
669
- if (composedQuery && composedQuery.Fields.length > 0) {
670
- for (const qField of composedQuery.Fields) {
671
- expandedFields.push({
672
- name: qField.Name,
673
- description: qField.Description || `${qField.Name} from query "${composedQuery.Name}"`,
674
- type: TypeScriptTypeFromSQLType(qField.SQLBaseType).toLowerCase(),
675
- optional: field.optional,
676
- sourceEntity: qField.SourceEntityID ? md.Entities.find(e => UUIDsEqual(e.ID, qField.SourceEntityID))?.Name || null : null,
677
- sourceFieldName: qField.SourceFieldName || null,
678
- isComputed: qField.IsComputed || false,
679
- isSummary: qField.IsSummary || false
680
- });
681
- }
682
- continue;
683
- }
684
- expandedFields.push(field);
685
- continue;
686
- }
687
- // No sourceEntity and no match — keep the raw entry as-is
688
- expandedFields.push(field);
689
- }
690
- return expandedFields;
691
- }
692
- async syncQueryFields(extractedFields) {
693
- // Use the entity's provider instead of creating new Metadata instance
694
- // Use same casting pattern as RefreshRelatedMetadata method
695
- const md = this.ProviderToUse;
696
- // Expand any wildcard (*) entries before processing
697
- const fieldsToSync = this.expandWildcardFields(extractedFields, md);
698
- try {
699
- const existingFields = [];
700
- if (this.IsSaved) {
701
- // Get existing query fields
702
- const rv = this.RunViewProviderToUse;
703
- const existingFieldsResult = await rv.RunView({
704
- EntityName: 'MJ: Query Fields',
705
- ExtraFilter: `QueryID='${this.ID}'`,
706
- ResultType: 'entity_object'
707
- }, this.ContextCurrentUser);
708
- if (!existingFieldsResult.Success) {
709
- throw new Error(`Failed to load existing query fields: ${existingFieldsResult.ErrorMessage}`);
710
- }
711
- existingFields.push(...existingFieldsResult.Results || []);
712
- }
713
- // Convert field names to lowercase for comparison (using expanded fieldsToSync)
714
- const fieldNamesToSync = fieldsToSync.map(f => f.name.toLowerCase());
715
- // Find fields to add, update, or remove
716
- const fieldsToAdd = fieldsToSync.filter(f => !existingFields.some(ef => ef.Name.toLowerCase() === f.name.toLowerCase()));
717
- const fieldsToUpdate = existingFields.filter(ef => fieldsToSync.some(f => f.name.toLowerCase() === ef.Name.toLowerCase()));
718
- const fieldsToRemove = existingFields.filter(ef => !fieldNamesToSync.includes(ef.Name.toLowerCase()));
719
- // Prepare all save/delete operations
720
- const promises = [];
721
- // Add new fields
722
- for (let i = 0; i < fieldsToAdd.length; i++) {
723
- const field = fieldsToAdd[i];
724
- const newField = await md.GetEntityObject('MJ: Query Fields', this.ContextCurrentUser);
725
- newField.QueryID = this.ID;
726
- newField.Name = field.name;
727
- newField.Description = field.description;
728
- newField.Sequence = i + 1;
729
- // Map type to SQL types
730
- switch (field.type) {
731
- case 'number':
732
- newField.SQLBaseType = 'decimal';
733
- newField.SQLFullType = 'decimal(18,2)';
734
- break;
735
- case 'date':
736
- newField.SQLBaseType = 'datetime';
737
- newField.SQLFullType = 'datetime';
738
- break;
739
- case 'boolean':
740
- newField.SQLBaseType = 'bit';
741
- newField.SQLFullType = 'bit';
742
- break;
743
- default:
744
- newField.SQLBaseType = 'nvarchar';
745
- newField.SQLFullType = 'nvarchar(MAX)';
746
- }
747
- // Set computed/summary flags
748
- newField.IsComputed = field.isComputed || field.dynamicName || false;
749
- newField.IsSummary = field.isSummary || false;
750
- if (field.computationDescription) {
751
- newField.ComputationDescription = field.computationDescription;
752
- }
753
- // Set source entity tracking
754
- if (field.sourceEntity) {
755
- // Look up entity ID from entity name (check both Name and DisplayName for "MJ: " prefix)
756
- const sourceEntityInfo = this.findEntityByName(md, field.sourceEntity);
757
- if (sourceEntityInfo) {
758
- newField.SourceEntityID = sourceEntityInfo.ID;
759
- }
760
- }
761
- newField.SourceFieldName = field.sourceFieldName || (field.dynamicName ? field.name : null);
762
- promises.push(newField.Save());
763
- }
764
- // Update existing fields if properties changed
765
- for (const existingField of fieldsToUpdate) {
766
- const extractedField = fieldsToSync.find(f => f.name.toLowerCase() === existingField.Name.toLowerCase());
767
- if (extractedField) {
768
- let hasChanges = false;
769
- if (existingField.Description !== extractedField.description) {
770
- existingField.Description = extractedField.description;
771
- hasChanges = true;
772
- }
773
- const newIsComputed = extractedField.isComputed || extractedField.dynamicName || false;
774
- if (existingField.IsComputed !== newIsComputed) {
775
- existingField.IsComputed = newIsComputed;
776
- hasChanges = true;
777
- }
778
- const newIsSummary = extractedField.isSummary || false;
779
- if (existingField.IsSummary !== newIsSummary) {
780
- existingField.IsSummary = newIsSummary;
781
- hasChanges = true;
782
- }
783
- if (extractedField.computationDescription && existingField.ComputationDescription !== extractedField.computationDescription) {
784
- existingField.ComputationDescription = extractedField.computationDescription;
785
- hasChanges = true;
786
- }
787
- // Update source entity tracking
788
- if (extractedField.sourceEntity) {
789
- const sourceEntityInfo = this.findEntityByName(md, extractedField.sourceEntity);
790
- if (sourceEntityInfo && !UUIDsEqual(existingField.SourceEntityID, sourceEntityInfo.ID)) {
791
- existingField.SourceEntityID = sourceEntityInfo.ID;
792
- hasChanges = true;
793
- }
794
- }
795
- else if (existingField.SourceEntityID != null) {
796
- existingField.SourceEntityID = null;
797
- hasChanges = true;
798
- }
799
- const newSourceFieldName = extractedField.sourceFieldName || (extractedField.dynamicName ? extractedField.name : null);
800
- if (existingField.SourceFieldName !== newSourceFieldName) {
801
- existingField.SourceFieldName = newSourceFieldName;
802
- hasChanges = true;
803
- }
804
- if (existingField.Sequence !== fieldsToSync.indexOf(extractedField) + 1) {
805
- existingField.Sequence = fieldsToSync.indexOf(extractedField) + 1;
806
- hasChanges = true;
807
- }
808
- if (hasChanges) {
809
- promises.push(existingField.Save());
810
- }
811
- }
812
- }
813
- // Remove fields that are no longer in the SQL
814
- for (const fieldToRemove of fieldsToRemove) {
815
- promises.push(fieldToRemove.Delete());
816
- }
817
- // Execute all operations in parallel
818
- if (promises.length > 0) {
819
- await Promise.all(promises);
820
- }
821
- }
822
- catch (e) {
823
- LogError(`Query "${this.Name}" - Failed to sync fields:`, e);
824
- throw e;
825
- }
826
- }
827
- async syncQueryEntities(extractedEntities) {
828
- // Use the entity's provider instead of creating new Metadata instance
829
- // Use same casting pattern as RefreshRelatedMetadata method
830
- const md = this.ProviderToUse;
831
- try {
832
- // Get existing query entities
833
- const existingEntities = [];
834
- if (this.IsSaved) {
835
- const rv = this.RunViewProviderToUse;
836
- const existingEntitiesResult = await rv.RunView({
837
- EntityName: 'MJ: Query Entities',
838
- ExtraFilter: `QueryID='${this.ID}'`,
839
- ResultType: 'entity_object'
840
- }, this.ContextCurrentUser);
841
- if (!existingEntitiesResult.Success) {
842
- throw new Error(`Failed to load existing query entities: ${existingEntitiesResult.ErrorMessage}`);
843
- }
844
- existingEntities.push(...existingEntitiesResult.Results || []);
845
- }
846
- // Look up MJ entity IDs for the extracted entities using pre-loaded metadata
847
- // Since extractEntityMetadataFromSQL already matched entities, we just need to find them by name
848
- const entityMappings = extractedEntities.map(extracted => {
849
- // Find matching entity in metadata by name (already matched during extraction)
850
- const matchingEntity = md.Entities.find(e => e.Name === extracted.name &&
851
- e.SchemaName.toLowerCase() === extracted.schemaName.toLowerCase());
852
- if (matchingEntity) {
853
- return {
854
- extracted,
855
- entityID: matchingEntity.ID,
856
- entityName: matchingEntity.Name
857
- };
858
- }
859
- return null;
860
- }).filter(m => m !== null);
861
- // Find entities to add or remove
862
- const entitiesToAdd = entityMappings.filter(mapping => !existingEntities.some(ee => UUIDsEqual(ee.EntityID, mapping.entityID)));
863
- const entitiesToRemove = existingEntities.filter(ee => !entityMappings.some(mapping => UUIDsEqual(mapping.entityID, ee.EntityID)));
864
- // Prepare all save/delete operations
865
- const promises = [];
866
- // Add new query entity relationships
867
- for (const mapping of entitiesToAdd) {
868
- if (mapping) {
869
- const newEntity = await md.GetEntityObject('MJ: Query Entities', this.ContextCurrentUser);
870
- newEntity.QueryID = this.ID;
871
- newEntity.EntityID = mapping.entityID;
872
- newEntity.DetectionMethod = 'AI'; // Using 'AI' as it's the closest match to automated detection
873
- newEntity.AutoDetectConfidenceScore = 1.0; // 100% confidence since we're using deterministic SQL parsing
874
- promises.push(newEntity.Save());
875
- }
876
- }
877
- // Remove entities that are no longer in the SQL
878
- for (const entityToRemove of entitiesToRemove) {
879
- promises.push(entityToRemove.Delete());
880
- }
881
- // Execute all operations in parallel
882
- if (promises.length > 0) {
883
- await Promise.all(promises);
884
- }
885
- }
886
- catch (e) {
887
- LogError(`Query "${this.Name}" - Failed to sync entities:`, e);
888
- throw e;
889
- }
890
- }
891
- async removeAllQueryFields() {
892
- try {
893
- if (!this.IsSaved)
894
- return; // Nothing to remove if not saved
895
- const rv = this.RunViewProviderToUse;
896
- const existingFieldsResult = await rv.RunView({
897
- EntityName: 'MJ: Query Fields',
898
- ExtraFilter: `QueryID='${this.ID}'`,
899
- ResultType: 'entity_object'
900
- }, this.ContextCurrentUser);
901
- if (!existingFieldsResult.Success) {
902
- throw new Error(`Failed to load existing query fields: ${existingFieldsResult.ErrorMessage}`);
903
- }
904
- const existingFields = existingFieldsResult.Results || [];
905
- const deletePromises = existingFields.map(field => field.Delete());
906
- if (deletePromises.length > 0) {
907
- await Promise.all(deletePromises);
908
- }
909
- }
910
- catch (e) {
911
- LogError('Failed to remove query fields:', e);
912
- throw e;
913
- }
914
- }
915
- async removeAllQueryEntities() {
916
- try {
917
- if (!this.IsSaved)
918
- return; // Nothing to remove if not saved
919
- const rv = this.RunViewProviderToUse;
920
- const existingEntitiesResult = await rv.RunView({
921
- EntityName: 'MJ: Query Entities',
922
- ExtraFilter: `QueryID='${this.ID}'`,
923
- ResultType: 'entity_object'
924
- }, this.ContextCurrentUser);
925
- if (!existingEntitiesResult.Success) {
926
- throw new Error(`Failed to load existing query entities: ${existingEntitiesResult.ErrorMessage}`);
927
- }
928
- const existingEntities = existingEntitiesResult.Results || [];
929
- const deletePromises = existingEntities.map(entity => entity.Delete());
930
- if (deletePromises.length > 0) {
931
- await Promise.all(deletePromises);
932
- }
933
- }
934
- catch (e) {
935
- LogError('Failed to remove query entities:', e);
936
- throw e;
937
- }
938
- }
939
- /**
940
- * Deterministically extracts {{query:"..."}} composition references from this query's SQL
941
- * and syncs the QueryDependency records accordingly. No LLM needed — uses regex parsing.
942
- */
943
- async extractAndSyncQueryDependencies() {
944
- const compositionEngine = new QueryCompositionEngine();
945
- const sql = this.SQL;
946
- if (!sql || !compositionEngine.HasCompositionTokens(sql)) {
947
- // No composition tokens — remove any stale dependency records
948
- await this.removeAllQueryDependencies();
949
- return;
950
- }
951
- // Parse all {{query:"..."}} tokens
952
- const tokens = compositionEngine.ParseCompositionTokens(sql);
953
- if (tokens.length === 0) {
954
- await this.removeAllQueryDependencies();
955
- return;
956
- }
957
- // Resolve each token to a QueryInfo to get its ID
958
- const extractedDeps = this.resolveTokensToQueryDependencies(tokens);
959
- // Check for cycles before syncing
960
- const cycleError = await this.detectDependencyCycles(extractedDeps);
961
- if (cycleError) {
962
- LogError(`Query "${this.Name}" creates circular dependency: ${cycleError}. Dependencies will not be synced.`);
963
- return;
964
- }
965
- await this.syncQueryDependencies(extractedDeps);
966
- }
967
- /**
968
- * Resolves parsed composition tokens to dependency records with QueryIDs.
969
- */
970
- resolveTokensToQueryDependencies(tokens) {
971
- const allQueries = Metadata.Provider.Queries;
972
- const deps = [];
973
- for (const token of tokens) {
974
- const queryName = token.QueryName.toLowerCase();
975
- // Try category-qualified lookup first
976
- let matchedQuery;
977
- if (token.CategorySegments.length > 0) {
978
- const expectedPath = `/${token.CategorySegments.join('/')}/`;
979
- matchedQuery = allQueries.find(q => q.Name.toLowerCase() === queryName &&
980
- q.CategoryPath.toLowerCase() === expectedPath.toLowerCase());
981
- }
982
- // Fall back to name-only
983
- if (!matchedQuery) {
984
- const nameMatches = allQueries.filter(q => q.Name.toLowerCase() === queryName);
985
- if (nameMatches.length === 1) {
986
- matchedQuery = nameMatches[0];
987
- }
988
- }
989
- if (!matchedQuery) {
990
- LogError(`Query "${this.Name}": Referenced query not found: "${token.FullPath}". Dependency will not be tracked.`);
991
- continue;
992
- }
993
- // Validate the referenced query is marked Reusable
994
- if (!matchedQuery.Reusable) {
995
- throw new Error(`Query "${this.Name}" references "${token.FullPath}" via composition syntax, ` +
996
- `but "${matchedQuery.Name}" is not marked as Reusable. ` +
997
- `Set Reusable=true on "${matchedQuery.Name}" before using it in {{query:"..."}} references.`);
998
- }
999
- // Extract alias from SQL context (look for the token followed by whitespace + identifier)
1000
- const alias = this.extractAliasFromSQL(token.FullToken);
1001
- // Build parameter mapping
1002
- let parameterMapping = null;
1003
- if (token.Parameters.length > 0) {
1004
- parameterMapping = {};
1005
- for (const param of token.Parameters) {
1006
- if (param.StaticValue !== null) {
1007
- parameterMapping[param.Name] = param.StaticValue;
1008
- }
1009
- else if (param.PassThroughName !== null) {
1010
- parameterMapping[param.Name] = `@${param.PassThroughName}`;
1011
- }
1012
- }
1013
- }
1014
- deps.push({
1015
- dependsOnQueryID: matchedQuery.ID,
1016
- referencePath: token.FullPath,
1017
- alias,
1018
- parameterMapping
1019
- });
1020
- }
1021
- return deps;
1022
- }
1023
- /**
1024
- * Extracts the SQL alias following a {{query:"..."}} token from this query's SQL.
1025
- */
1026
- extractAliasFromSQL(fullToken) {
1027
- if (!this.SQL)
1028
- return null;
1029
- const tokenIndex = this.SQL.indexOf(fullToken);
1030
- if (tokenIndex < 0)
1031
- return null;
1032
- const afterToken = this.SQL.substring(tokenIndex + fullToken.length);
1033
- const aliasMatch = /^\s+(\w+)/i.exec(afterToken);
1034
- // Make sure it's not a SQL keyword
1035
- if (aliasMatch) {
1036
- const keywords = new Set(['ON', 'WHERE', 'AND', 'OR', 'JOIN', 'LEFT', 'RIGHT', 'INNER', 'OUTER', 'CROSS', 'FULL', 'GROUP', 'ORDER', 'HAVING', 'UNION', 'EXCEPT', 'INTERSECT', 'LIMIT', 'OFFSET', 'FETCH']);
1037
- if (!keywords.has(aliasMatch[1].toUpperCase())) {
1038
- return aliasMatch[1];
1039
- }
1040
- }
1041
- return null;
1042
- }
1043
- /**
1044
- * Detects circular dependencies by walking the dependency graph.
1045
- * Returns a descriptive error string if a cycle is found, or null if clean.
1046
- */
1047
- async detectDependencyCycles(proposedDeps) {
1048
- const allDependencies = Metadata.Provider.QueryDependencies;
1049
- const visited = new Set();
1050
- const stack = new Set();
1051
- const hasCycle = (queryID, path) => {
1052
- if (stack.has(queryID)) {
1053
- const queryName = Metadata.Provider.Queries.find(q => UUIDsEqual(q.ID, queryID))?.Name || queryID;
1054
- return [...path, queryName].join(' → ');
1055
- }
1056
- if (visited.has(queryID))
1057
- return null;
1058
- visited.add(queryID);
1059
- stack.add(queryID);
1060
- const queryName = Metadata.Provider.Queries.find(q => UUIDsEqual(q.ID, queryID))?.Name || queryID;
1061
- path.push(queryName);
1062
- // Get existing dependencies for this query
1063
- const deps = allDependencies.filter(d => d.QueryID === queryID);
1064
- for (const dep of deps) {
1065
- const result = hasCycle(dep.DependsOnQueryID, [...path]);
1066
- if (result)
1067
- return result;
1068
- }
1069
- stack.delete(queryID);
1070
- return null;
1071
- };
1072
- // Check each proposed dependency
1073
- for (const dep of proposedDeps) {
1074
- // Simulate: dep.dependsOnQueryID → (its dependencies) → ... → back to this.ID?
1075
- const visited2 = new Set();
1076
- const stack2 = new Set([this.ID]);
1077
- const checkFromDep = (queryID, path) => {
1078
- if (UUIDsEqual(queryID, this.ID)) {
1079
- return [...path, this.Name].join(' → ');
1080
- }
1081
- if (visited2.has(queryID))
1082
- return null;
1083
- visited2.add(queryID);
1084
- const queryName = Metadata.Provider.Queries.find(q => UUIDsEqual(q.ID, queryID))?.Name || queryID;
1085
- path.push(queryName);
1086
- const deps = allDependencies.filter(d => d.QueryID === queryID);
1087
- for (const d of deps) {
1088
- const result = checkFromDep(d.DependsOnQueryID, [...path]);
1089
- if (result)
1090
- return result;
1091
- }
1092
- return null;
1093
- };
1094
- const result = checkFromDep(dep.dependsOnQueryID, [this.Name]);
1095
- if (result)
1096
- return result;
1097
- }
1098
- return null;
1099
- }
1100
- /**
1101
- * Syncs extracted query dependencies against existing QueryDependency records.
1102
- * Follows the same add/update/delete pattern as syncQueryParameters/Fields/Entities.
1103
- */
1104
- async syncQueryDependencies(extractedDeps) {
1105
- const md = new Metadata();
1106
- try {
1107
- const existingDeps = [];
1108
- if (this.IsSaved) {
1109
- const rv = this.RunViewProviderToUse;
1110
- const existingResult = await rv.RunView({
1111
- EntityName: 'MJ: Query Dependencies',
1112
- ExtraFilter: `QueryID='${this.ID}'`,
1113
- ResultType: 'entity_object'
1114
- }, this.ContextCurrentUser);
1115
- if (!existingResult.Success) {
1116
- throw new Error(`Failed to load existing query dependencies: ${existingResult.ErrorMessage}`);
1117
- }
1118
- existingDeps.push(...(existingResult.Results || []));
1119
- }
1120
- // Compare: match on DependsOnQueryID + ReferencePath for uniqueness
1121
- const depsToAdd = extractedDeps.filter(d => !existingDeps.some(ed => UUIDsEqual(ed.DependsOnQueryID, d.dependsOnQueryID) &&
1122
- ed.ReferencePath === d.referencePath));
1123
- const depsToUpdate = existingDeps.filter(ed => extractedDeps.some(d => UUIDsEqual(ed.DependsOnQueryID, d.dependsOnQueryID) &&
1124
- ed.ReferencePath === d.referencePath));
1125
- const depsToRemove = existingDeps.filter(ed => !extractedDeps.some(d => UUIDsEqual(ed.DependsOnQueryID, d.dependsOnQueryID) &&
1126
- ed.ReferencePath === d.referencePath));
1127
- const promises = [];
1128
- // Add new dependencies
1129
- for (const dep of depsToAdd) {
1130
- const newDep = await md.GetEntityObject('MJ: Query Dependencies', this.ContextCurrentUser);
1131
- newDep.QueryID = this.ID;
1132
- newDep.DependsOnQueryID = dep.dependsOnQueryID;
1133
- newDep.ReferencePath = dep.referencePath;
1134
- newDep.Alias = dep.alias;
1135
- newDep.ParameterMapping = dep.parameterMapping ? JSON.stringify(dep.parameterMapping) : null;
1136
- newDep.DetectionMethod = 'Auto';
1137
- promises.push(newDep.Save());
1138
- }
1139
- // Update existing dependencies (check for changes)
1140
- for (const existingDep of depsToUpdate) {
1141
- const extractedDep = extractedDeps.find(d => UUIDsEqual(existingDep.DependsOnQueryID, d.dependsOnQueryID) &&
1142
- existingDep.ReferencePath === d.referencePath);
1143
- if (extractedDep) {
1144
- let hasChanges = false;
1145
- if (existingDep.Alias !== extractedDep.alias) {
1146
- existingDep.Alias = extractedDep.alias;
1147
- hasChanges = true;
1148
- }
1149
- const newMapping = extractedDep.parameterMapping ? JSON.stringify(extractedDep.parameterMapping) : null;
1150
- if (existingDep.ParameterMapping !== newMapping) {
1151
- existingDep.ParameterMapping = newMapping;
1152
- hasChanges = true;
1153
- }
1154
- if (hasChanges) {
1155
- promises.push(existingDep.Save());
1156
- }
1157
- }
1158
- }
1159
- // Remove stale dependencies
1160
- for (const depToRemove of depsToRemove) {
1161
- promises.push(depToRemove.Delete());
1162
- }
1163
- if (promises.length > 0) {
1164
- await Promise.all(promises);
1165
- }
1166
- }
1167
- catch (e) {
1168
- LogError(`Query "${this.Name}" - Failed to sync dependencies:`, e);
1169
- throw e;
1170
- }
1171
- }
1172
- /**
1173
- * Removes all QueryDependency records for this query.
1174
- */
1175
- async removeAllQueryDependencies() {
1176
- try {
1177
- if (!this.IsSaved)
1178
- return;
1179
- const rv = this.RunViewProviderToUse;
1180
- const existingResult = await rv.RunView({
1181
- EntityName: 'MJ: Query Dependencies',
1182
- ExtraFilter: `QueryID='${this.ID}'`,
1183
- ResultType: 'entity_object'
1184
- }, this.ContextCurrentUser);
1185
- if (!existingResult.Success) {
1186
- throw new Error(`Failed to load existing query dependencies: ${existingResult.ErrorMessage}`);
1187
- }
1188
- const existingDeps = existingResult.Results || [];
1189
- const deletePromises = existingDeps.map(dep => dep.Delete());
1190
- if (deletePromises.length > 0) {
1191
- await Promise.all(deletePromises);
1192
- }
1193
- }
1194
- catch (e) {
1195
- LogError('Failed to remove query dependencies:', e);
1196
- throw e;
1197
- }
1198
- }
194
+ // ─── Dialect Conversion ──────────────────────────────────────────────────────
1199
195
  /**
1200
196
  * Auto-converts the query's SQL to other dialects based on the queryDialects
1201
197
  * configuration stored in GlobalObjectStore. Best-effort: failures never block the save.
1202
198
  */
1203
199
  async autoConvertDialectsAsync() {
1204
200
  try {
1205
- // Read config from GlobalObjectStore (set by MJServer at startup)
1206
201
  const config = MJGlobal.Instance.GetGlobalObjectStore()?.['queryDialects'];
1207
202
  if (!config?.autoConvertOnSave || !config.targetPlatforms?.length) {
1208
- return; // Auto-convert disabled or no targets configured
203
+ return;
1209
204
  }
1210
205
  if (!this.SQL || this.SQL.trim().length === 0) {
1211
- return; // Nothing to convert
206
+ return;
1212
207
  }
1213
208
  const md = this.ProviderToUse;
1214
209
  const dialects = md.SQLDialects;
1215
- // Determine source dialect from query's SQLDialectID
1216
210
  const sourceDialect = this.SQLDialectID
1217
211
  ? dialects.find(d => UUIDsEqual(d.ID, this.SQLDialectID))
1218
- : dialects.find(d => d.PlatformKey === 'sqlserver'); // Default to T-SQL
212
+ : dialects.find(d => d.PlatformKey === 'sqlserver');
1219
213
  if (!sourceDialect) {
1220
214
  console.warn(`Query "${this.Name}" - Could not determine source dialect, skipping auto-convert`);
1221
215
  return;
1222
216
  }
1223
- // Process each target platform independently
1224
217
  for (const targetPlatformKey of config.targetPlatforms) {
1225
- try {
1226
- // Skip if target is same as source
1227
- if (targetPlatformKey === sourceDialect.PlatformKey) {
1228
- continue;
1229
- }
1230
- const targetDialect = dialects.find(d => d.PlatformKey === targetPlatformKey);
1231
- if (!targetDialect) {
1232
- console.warn(`Query "${this.Name}" - Target dialect "${targetPlatformKey}" not found, skipping`);
1233
- continue;
1234
- }
1235
- // Currently only T-SQL → PostgreSQL is supported
1236
- if (sourceDialect.PlatformKey !== 'sqlserver' || targetPlatformKey !== 'postgresql') {
1237
- console.warn(`Query "${this.Name}" - Conversion from "${sourceDialect.PlatformKey}" to "${targetPlatformKey}" not yet supported`);
1238
- continue;
1239
- }
1240
- const convertedSQL = this.convertTSQLToPostgreSQL(this.SQL);
1241
- await this.upsertQuerySQLRecord(targetDialect, convertedSQL, md);
1242
- }
1243
- catch (platformError) {
1244
- // Per-platform isolation: failing on one target doesn't affect others
1245
- console.warn(`Query "${this.Name}" - Auto-convert to "${targetPlatformKey}" failed:`, platformError);
1246
- }
218
+ await this.convertToTargetDialect(sourceDialect, targetPlatformKey, md, dialects);
1247
219
  }
1248
220
  }
1249
221
  catch (e) {
1250
- // Best-effort: never block the save
1251
222
  console.warn(`Query "${this.Name}" - Auto-convert dialects failed:`, e);
1252
223
  }
1253
224
  }
1254
225
  /**
1255
- * Converts T-SQL to PostgreSQL using SQLConverter ExpressionHelper transforms.
1256
- * Applies the standard transform pipeline for statement-level conversions.
226
+ * Converts and persists SQL for a single target dialect. Logs warnings on skip/failure.
1257
227
  */
1258
- convertTSQLToPostgreSQL(sql) {
1259
- let result = sql;
1260
- result = removeNPrefix(result);
1261
- result = convertIdentifiers(result);
1262
- result = convertCommonFunctions(result);
1263
- result = convertDateFunctions(result);
1264
- result = convertCharIndex(result);
1265
- result = convertStuff(result);
1266
- result = convertIIF(result);
1267
- result = convertConvertFunction(result);
1268
- result = convertCastTypes(result);
1269
- result = convertTopToLimit(result);
1270
- result = convertStringConcat(result);
1271
- result = removeCollate(result);
1272
- return result;
228
+ async convertToTargetDialect(sourceDialect, targetPlatformKey, md, dialects) {
229
+ try {
230
+ if (targetPlatformKey === sourceDialect.PlatformKey) {
231
+ return;
232
+ }
233
+ const targetDialect = dialects.find(d => d.PlatformKey === targetPlatformKey);
234
+ if (!targetDialect) {
235
+ console.warn(`Query "${this.Name}" - Target dialect "${targetPlatformKey}" not found, skipping`);
236
+ return;
237
+ }
238
+ if (sourceDialect.PlatformKey !== 'sqlserver' || targetPlatformKey !== 'postgresql') {
239
+ console.warn(`Query "${this.Name}" - Conversion from "${sourceDialect.PlatformKey}" to "${targetPlatformKey}" not yet supported`);
240
+ return;
241
+ }
242
+ const convertedSQL = ConvertTSQLToPostgreSQL(this.SQL);
243
+ await this.upsertQuerySQLRecord(targetDialect, convertedSQL, md);
244
+ }
245
+ catch (platformError) {
246
+ console.warn(`Query "${this.Name}" - Auto-convert to "${targetPlatformKey}" failed:`, platformError);
247
+ }
1273
248
  }
1274
249
  /**
1275
250
  * Creates or updates a QuerySQL record for the given dialect.
1276
251
  */
1277
252
  async upsertQuerySQLRecord(targetDialect, convertedSQL, md) {
1278
253
  const rv = this.RunViewProviderToUse;
1279
- // Look for existing QuerySQL record for this query + dialect
1280
254
  const existingResult = await rv.RunView({
1281
255
  EntityName: 'MJ: Query SQLs',
1282
256
  ExtraFilter: `QueryID='${this.ID}' AND SQLDialectID='${targetDialect.ID}'`,
@@ -1288,11 +262,9 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
1288
262
  }
1289
263
  let record;
1290
264
  if (existingResult.Results?.length > 0) {
1291
- // Update existing
1292
265
  record = existingResult.Results[0];
1293
266
  }
1294
267
  else {
1295
- // Create new
1296
268
  record = await md.GetEntityObject('MJ: Query SQLs', this.ContextCurrentUser);
1297
269
  record.QueryID = this.ID;
1298
270
  record.SQLDialectID = targetDialect.ID;
@@ -1329,6 +301,7 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
1329
301
  console.warn(`Query "${this.Name}" - Failed to remove QuerySQL records:`, e);
1330
302
  }
1331
303
  }
304
+ // ─── Load Overrides ──────────────────────────────────────────────────────────
1332
305
  async Load(ID, EntityRelationshipsToLoad) {
1333
306
  const result = await super.Load(ID, EntityRelationshipsToLoad);
1334
307
  await this.RefreshRelatedMetadata(false);
@@ -1339,25 +312,24 @@ let MJQueryEntityServer = class MJQueryEntityServer extends MJQueryEntity {
1339
312
  await this.RefreshRelatedMetadata(false);
1340
313
  return result;
1341
314
  }
315
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- BaseEntity.LoadFromData uses `any` for the data parameter
1342
316
  async LoadFromData(data, _replaceOldValues) {
1343
317
  const result = await super.LoadFromData(data, _replaceOldValues);
1344
318
  await this.RefreshRelatedMetadata(false);
1345
319
  return result;
1346
320
  }
321
+ // ─── Metadata Refresh ────────────────────────────────────────────────────────
1347
322
  /**
1348
323
  * Refreshes this record's related metadata from the provider, refreshing
1349
- * all the way up from the database if refreshFromDB is true, otherwise from
1350
- * cache.
1351
- * @param refreshFromDB
324
+ * all the way up from the database if refreshFromDB is true, otherwise from cache.
1352
325
  */
1353
326
  async RefreshRelatedMetadata(refreshFromDB) {
1354
327
  const md = this.ProviderToUse;
1355
328
  if (refreshFromDB) {
1356
329
  const globalMetadataProvider = Metadata.Provider;
1357
- await globalMetadataProvider.Refresh(md); // we pass in our metadata provider because that is the connection we want to use if we are in the midst of a transaction
330
+ await globalMetadataProvider.Refresh(md);
1358
331
  if (globalMetadataProvider !== md) {
1359
- // If the global metadata provider is different, we need to refresh it
1360
- await md.Refresh(); // will refresh FROM the global provider, meaning we do NOT hit the DB again, we just copy the data into our MD instance that is part of our trans scope
332
+ await md.Refresh();
1361
333
  }
1362
334
  }
1363
335
  this._queryPermissions = md.QueryPermissions.filter(p => UUIDsEqual(p.QueryID, this.ID));