@memberjunction/query-gen 3.4.0 → 4.1.0

package/src/cli/commands/generate.ts

@@ -12,8 +12,6 @@
 
 import ora from 'ora';
 import chalk from 'chalk';
-import * as fs from 'fs';
-import * as path from 'path';
 import { Metadata, DatabaseProviderBase, EntityInfo, LogStatus } from '@memberjunction/core';
 import { loadConfig } from '../config';
 import { getSystemUser } from '../../utils/user-helpers';
@@ -30,6 +28,9 @@ import { formatEntityMetadataForPrompt } from '../../utils/entity-helpers';
 import { extractErrorMessage } from '../../utils/error-handlers';
 import { buildQueryCategory, extractUniqueCategories } from '../../utils/category-builder';
 import { ValidatedQuery, GoldenQuery, EntityGroup, QueryCategoryInfo } from '../../data/schema';
+import { readFileSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
 
 /**
  * Execute the generate command
@@ -73,11 +74,42 @@ export async function generateCommand(options: Record<string, unknown>): Promise
     spinner.start('Filtering entities...');
     const md = new Metadata();
 
+    // DIAGNOSTIC: Log entity filtering context
+    if (config.verbose) {
+      LogStatus(`\n=== Entity Filtering Diagnostics ===`);
+      LogStatus(`Total entities in metadata: ${md.Entities.length}`);
+      LogStatus(`Excluded schemas: ${config.excludeSchemas.join(', ')}`);
+
+      // Show schema distribution BEFORE filtering
+      const schemasBefore = new Map<string, number>();
+      md.Entities.forEach(e => {
+        const schema = e.SchemaName || 'null';
+        schemasBefore.set(schema, (schemasBefore.get(schema) || 0) + 1);
+      });
+      LogStatus(`Schema distribution (before filtering):`);
+      Array.from(schemasBefore.entries())
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, 10)
+        .forEach(([schema, count]) => {
+          const excluded = config.excludeSchemas.includes(schema) ? ' [EXCLUDED]' : '';
+          LogStatus(`  ${schema}: ${count} entities${excluded}`);
+        });
+    }
+
     // Apply entity filtering (includeEntities takes precedence over excludeEntities)
     let filteredEntities = md.Entities.filter(
       e => !config.excludeSchemas.includes(e.SchemaName || '')
     );
 
+    if (config.verbose) {
+      LogStatus(`\nAfter schema filtering: ${filteredEntities.length} entities remaining`);
+      if (filteredEntities.length > 0) {
+        LogStatus(`Sample entity names: ${filteredEntities.slice(0, 5).map(e => e.Name).join(', ')}`);
+        const schemasAfter = new Set(filteredEntities.map(e => e.SchemaName || 'null'));
+        LogStatus(`Remaining schemas: ${Array.from(schemasAfter).join(', ')}`);
+      }
+    }
+
     if (config.includeEntities.length > 0) {
       // Allowlist: only include specified entities
       filteredEntities = filteredEntities.filter(e => config.includeEntities.includes(e.Name));
@@ -88,6 +120,11 @@ export async function generateCommand(options: Record<string, unknown>): Promise
       spinner.info(chalk.dim(`Excluded ${config.excludeEntities.length} entities`));
     }
 
+    if (config.verbose) {
+      LogStatus(`Final filtered entities: ${filteredEntities.length}`);
+      LogStatus('====================================\n');
+    }
+
     // 4. Group entities by schema and generate entity groups
     spinner.text = 'Analyzing entity relationships...';
 
@@ -282,36 +319,30 @@ export async function generateCommand(options: Record<string, unknown>): Promise
 }
 
 /**
- * Load golden queries from JSON file
+ * Load golden queries from compile-time imported JSON data
  *
  * Golden queries are example queries used for few-shot learning.
- * They are stored in the data/golden-queries.json file.
+ * They are imported at compile time from data/golden-queries.json.
+ *
+ * This approach is more efficient than runtime file I/O and provides
+ * type safety through TypeScript's resolveJsonModule feature.
  *
  * @param config - QueryGen configuration with verbose flag
- * @returns Array of golden queries, or empty array if file not found/invalid
+ * @returns Array of golden queries
  */
 async function loadGoldenQueries(config: { verbose: boolean }): Promise<GoldenQuery[]> {
   try {
-    // Resolve path to golden-queries.json in the data directory
-    // __dirname points to dist/cli/commands, so we go up to dist, then to data
-    const goldenQueriesPath = path.join(__dirname, '../../data/golden-queries.json');
-
-    // Check if file exists
-    if (!fs.existsSync(goldenQueriesPath)) {
-      if (config.verbose) {
-        LogStatus(`[Warning] Golden queries file not found at: ${goldenQueriesPath}`);
-      }
-      return [];
-    }
-
-    // Read and parse JSON file
-    const fileContent = fs.readFileSync(goldenQueriesPath, 'utf-8');
-    const goldenQueries = JSON.parse(fileContent) as GoldenQuery[];
+    // Load golden queries from JSON file using ES module compatible path resolution
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    const goldenQueriesPath = join(__dirname, '../../data/golden-queries.json');
+    const goldenQueriesData = JSON.parse(readFileSync(goldenQueriesPath, 'utf-8'));
+    const goldenQueries = goldenQueriesData as GoldenQuery[];
 
     // Validate that it's an array
     if (!Array.isArray(goldenQueries)) {
       if (config.verbose) {
-        LogStatus('[Warning] Golden queries file does not contain an array');
+        LogStatus('[Warning] Golden queries data is not an array');
       }
       return [];
     }

package/src/cli/config.ts

@@ -4,6 +4,13 @@
  * Loads configuration from mj.config.cjs and merges with CLI options
  */
 
+import fs from 'fs';
+import path from 'path';
+import { createRequire } from 'node:module';
+
+// Use createRequire to load CommonJS config files
+const require = createRequire(import.meta.url);
+
 /**
  * QueryGen configuration options
  */
@@ -21,6 +28,7 @@ export interface QueryGenConfig {
   questionsPerGroup: number;
   minGroupSize: number;        // Minimum entities per group
   maxGroupSize: number;        // Maximum entities per group
+  requireConnectivity: boolean; // Require entities in groups to be connected via relationships
 
   // AI Configuration
   modelOverride?: string;    // Override model for all prompts (e.g., "GPT-OSS-120B")
@@ -74,6 +82,7 @@ const DEFAULT_CONFIG: QueryGenConfig = {
   questionsPerGroup: 2,
   minGroupSize: 2,              // Multi-entity groups have at least 2 entities
   maxGroupSize: 3,              // Keep groups small for focused questions
+  requireConnectivity: true,    // Require entities to be connected (disable for sparse relationship graphs)
   embeddingModel: 'text-embedding-3-small',
   maxRefinementIterations: 3,
   maxFixingIterations: 5,
@@ -170,8 +179,7 @@ export function loadConfig(cliOptions: Record<string, unknown>): QueryGenConfig
  */
 function loadMjConfig(): { queryGen?: Partial<QueryGenConfig> } | null {
   try {
-    const configPath = require('path').join(process.cwd(), 'mj.config.cjs');
-    const fs = require('fs');
+    const configPath = path.join(process.cwd(), 'mj.config.cjs');
     if (fs.existsSync(configPath)) {
       return require(configPath);
     }

package/src/cli/index.ts

@@ -11,11 +11,14 @@
  */
 
 import { Command } from 'commander';
+import { createRequire } from 'node:module';
 import { generateCommand } from './commands/generate';
 import { validateCommand } from './commands/validate';
 import { exportCommand } from './commands/export';
 
-const packageJson = require('../../package.json');
+// Use createRequire to import JSON (compatible with ESM)
+const require = createRequire(import.meta.url);
+const packageJson = require('../../package.json') as { version: string };
 
 const program = new Command();
 

package/src/core/EntityGrouper.ts

@@ -74,6 +74,14 @@ export class EntityGrouper {
           LogStatus(`[${schemaIndex}/${totalSchemas}] Processing schema: ${schemaName} (${schemaEntities.length} entities)`);
         }
 
+        // Skip schemas with too few entities for grouping
+        if (schemaEntities.length < this.config.minGroupSize) {
+          if (this.config.verbose) {
+            LogStatus(`[${schemaIndex}/${totalSchemas}] Skipping schema ${schemaName}: only ${schemaEntities.length} entities (minimum ${this.config.minGroupSize} required)`);
+          }
+          continue;
+        }
+
         // 2a. Prepare schema data for LLM
         const schemaData = this.prepareSchemaData(schemaEntities, schemaName);
 
@@ -81,7 +89,8 @@ export class EntityGrouper {
         const llmResponse = await this.callLLMForGrouping(schemaData, contextUser);
 
         // 2c. Validate and convert to EntityGroup objects
-        const validatedGroups = this.validateAndConvertGroups(llmResponse, schemaEntities);
+        // Note: This may return an empty array if no groups pass validation
+        const validatedGroups = this.validateAndConvertGroups(llmResponse, schemaEntities, schemaName);
 
         if (this.config.verbose && totalSchemas > 1) {
           LogStatus(`[${schemaIndex}/${totalSchemas}] Generated ${validatedGroups.length} groups for schema: ${schemaName}`);
@@ -90,7 +99,12 @@ export class EntityGrouper {
         allGroups.push(...validatedGroups);
       }
 
-      // 3. Deduplicate any similar groups across all schemas
+      // 3. Check if we generated any groups across all schemas
+      if (allGroups.length === 0) {
+        throw new Error('No valid entity groups generated by LLM across any schema');
+      }
+
+      // 4. Deduplicate any similar groups across all schemas
       const deduplicatedGroups = this.deduplicateGroups(allGroups);
 
       return deduplicatedGroups;
@@ -223,14 +237,36 @@ export class EntityGrouper {
 
   /**
    * Validate LLM output and convert to EntityGroup objects
+   *
+   * @param schemaName - Optional schema name for better error messages
+   * @returns Array of validated groups (may be empty if no groups pass validation)
    */
   private validateAndConvertGroups(
     llmResponse: LLMEntityGroupResponse,
-    entities: EntityInfo[]
+    entities: EntityInfo[],
+    schemaName?: string
   ): EntityGroup[] {
     const entityMap = new Map(entities.map(e => [e.Name, e]));
     const validGroups: EntityGroup[] = [];
 
+    // DIAGNOSTIC: Log entity validation context
+    if (this.config.verbose) {
+      LogStatus('\n=== Entity Group Validation Diagnostics ===');
+      LogStatus(`Entities available for validation: ${entities.length}`);
+      if (entities.length > 0) {
+        LogStatus(`Entity names in map: ${Array.from(entityMap.keys()).slice(0, 10).join(', ')}${entityMap.size > 10 ? '...' : ''}`);
+        LogStatus(`Sample entity SchemaName values: ${entities.slice(0, 5).map(e => `"${e.SchemaName || 'null'}"`).join(', ')}`);
+      }
+      LogStatus(`Groups returned by LLM: ${llmResponse.groups.length}`);
+      if (llmResponse.groups.length > 0) {
+        LogStatus(`Sample LLM group entities: ${llmResponse.groups[0].entities.join(', ')}`);
+      }
+      LogStatus(`Connectivity validation: ${this.config.requireConnectivity ? 'ENABLED' : 'DISABLED'}`);
+      if (!this.config.requireConnectivity) {
+        LogStatus('⚠️  WARNING: Connectivity validation is disabled. Groups may include unrelated entities.');
+      }
+    }
+
     for (const llmGroup of llmResponse.groups) {
       try {
         // Validate all entity names exist
@@ -239,14 +275,21 @@ export class EntityGrouper {
           .filter((e): e is EntityInfo => e !== undefined);
 
         if (groupEntities.length !== llmGroup.entities.length) {
-          // Skip logging - verbose debug info that clutters output
+          if (this.config.verbose) {
+            const missingEntities = llmGroup.entities.filter(name => !entityMap.has(name));
+            LogStatus(`  ✗ Group rejected - Entity name mismatch:`);
+            LogStatus(`    LLM requested: ${llmGroup.entities.join(', ')}`);
+            LogStatus(`    Not found in metadata: ${missingEntities.join(', ')}`);
+          }
           continue;
         }
 
         // Validate primary entity exists
         const primaryEntity = entityMap.get(llmGroup.primaryEntity);
         if (!primaryEntity) {
-          // Skip logging - verbose debug info that clutters output
+          if (this.config.verbose) {
+            LogStatus(`  ✗ Group rejected - Primary entity not found: ${llmGroup.primaryEntity}`);
+          }
           continue;
         }
 
@@ -254,9 +297,20 @@ export class EntityGrouper {
         const relationships = this.extractRelationships(groupEntities);
 
         // Validate connectivity (all entities must be reachable from primary)
-        if (groupEntities.length > 1 && !this.isConnected(groupEntities, relationships)) {
-          // Skip logging - verbose debug info that clutters output
-          continue;
+        // Skip this check if requireConnectivity is disabled
+        if (this.config.requireConnectivity) {
+          if (groupEntities.length > 1 && !this.isConnected(groupEntities, relationships)) {
+            if (this.config.verbose) {
+              LogStatus(`  ✗ Group rejected - Entities not connected: ${groupEntities.map(e => e.Name).join(', ')}`);
+            }
+            continue;
+          }
+        } else if (this.config.verbose && groupEntities.length > 1) {
+          // Log warning when connectivity check is skipped
+          const hasRelationships = relationships.length > 0;
+          if (!hasRelationships) {
+            LogStatus(`  ⚠️  Group has no relationships (connectivity check disabled): ${groupEntities.map(e => e.Name).join(', ')}`);
+          }
         }
 
         // Create EntityGroup with LLM metadata
@@ -269,15 +323,25 @@ export class EntityGrouper {
           businessRationale: llmGroup.businessRationale,
           expectedQuestionTypes: llmGroup.expectedQuestionTypes
         });
+
+        if (this.config.verbose) {
+          LogStatus(`  ✓ Group accepted: ${groupEntities.map(e => e.Name).join(', ')}`);
+        }
       } catch (error: unknown) {
-        // Skip logging - verbose debug info that clutters output
+        if (this.config.verbose) {
+          LogStatus(`  ✗ Group validation error: ${extractErrorMessage(error, 'Group Validation')}`);
+        }
       }
     }
 
-    if (validGroups.length === 0) {
-      throw new Error('No valid entity groups generated by LLM');
+    if (this.config.verbose) {
+      const schemaInfo = schemaName ? ` (schema: ${schemaName})` : '';
+      LogStatus(`\nValidation Summary${schemaInfo}: ${validGroups.length}/${llmResponse.groups.length} groups accepted`);
+      LogStatus('==========================================\n');
     }
 
+    // Return empty array if no groups passed validation
+    // The caller will aggregate results from all schemas and check if total is 0
     return validGroups;
   }
 

package/src/core/QueryTester.ts

@@ -5,7 +5,7 @@
  * queries against the database. Handles error fixing with retry loop.
  */
 
-import * as nunjucks from 'nunjucks';
+import nunjucks from 'nunjucks';
 import {
   DatabaseProviderBase,
   RunQuerySQLFilterManager,

package/tsconfig.json

@@ -1,39 +1,9 @@
 {
+  "extends": "../../tsconfig.server.json",
   "compilerOptions": {
-    "target": "ES2022",
-    "module": "commonjs",
-    "lib": ["ES2022"],
-    "outDir": "./dist",
-    "rootDir": "./src",
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true,
-    "moduleResolution": "node",
-    "noImplicitAny": true,
-    "strictNullChecks": true,
-    "strictFunctionTypes": true,
-    "strictBindCallApply": true,
-    "strictPropertyInitialization": true,
-    "noImplicitThis": true,
-    "alwaysStrict": true,
-    "noUnusedLocals": false,
-    "noUnusedParameters": false,
-    "noImplicitReturns": true,
-    "noFallthroughCasesInSwitch": true,
-    "allowSyntheticDefaultImports": true
+    "outDir": "dist",
+    "rootDir": "src"
   },
-  "include": [
-    "src/**/*"
-  ],
-  "exclude": [
-    "node_modules",
-    "dist",
-    "**/*.spec.ts",
-    "**/*.test.ts"
-  ]
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"]
 }