@memberjunction/query-gen 0.0.1 → 2.126.1

package/src/core/EntityGrouper.ts

@@ -0,0 +1,318 @@
+/**
+ * LLM-based Entity Grouper
+ *
+ * Uses AI to generate semantically meaningful entity groupings based on
+ * business context and schema understanding, replacing the deterministic
+ * hub-and-spoke algorithm with intelligent semantic analysis.
+ */
+
+import { EntityInfo, EntityRelationshipInfo, UserInfo, LogStatus } from '@memberjunction/core';
+import { AIEngine } from '@memberjunction/aiengine';
+import { EntityGroup } from '../data/schema';
+import { QueryGenConfig } from '../cli/config';
+import { generateRelationshipGraph, formatEntitiesForPrompt } from '../utils/graph-helpers';
+import { extractErrorMessage } from '../utils/error-handlers';
+import { executePromptWithOverrides } from '../utils/prompt-helpers';
+
+/**
+ * LLM response format from Entity Group Generator prompt
+ */
+interface LLMEntityGroupResponse {
+  groups: Array<{
+    entities: string[];
+    primaryEntity: string;
+    businessDomain: string;
+    businessRationale: string;
+    relationshipType: 'single' | 'parent-child' | 'many-to-many';
+    expectedQuestionTypes: string[];
+  }>;
+}
+
+/**
+ * Generates entity groups using LLM-based semantic analysis
+ *
+ * This class replaces the deterministic hub-and-spoke algorithm with an
+ * intelligent approach that understands business context and generates
+ * meaningful entity combinations for query generation.
+ */
+export class EntityGrouper {
+  private readonly promptName = 'Entity Group Generator';
+  private readonly config: QueryGenConfig;
+
+  constructor(config: QueryGenConfig) {
+    this.config = config;
+  }
+
+  /**
+   * Generate semantically meaningful entity groups using LLM analysis
+   *
+   * Note: The LLM determines the optimal number and size of entity groups based on
+   * business domain understanding. Makes separate LLM calls for each schema to
+   * avoid mixing entities from different databases.
+   *
+   * @param entities - All entities to analyze
+   * @param contextUser - User context for server-side operations
+   * @returns Array of validated entity groups with business context
+   */
+  async generateEntityGroups(
+    entities: EntityInfo[],
+    contextUser: UserInfo
+  ): Promise<EntityGroup[]> {
+    try {
+      // 1. Group entities by schema
+      const entitiesBySchema = this.groupEntitiesBySchema(entities);
+
+      // 2. Process each schema separately
+      const allGroups: EntityGroup[] = [];
+      let schemaIndex = 0;
+      const totalSchemas = entitiesBySchema.size;
+
+      for (const [schemaName, schemaEntities] of entitiesBySchema.entries()) {
+        schemaIndex++;
+
+        if (this.config.verbose && totalSchemas > 1) {
+          LogStatus(`[${schemaIndex}/${totalSchemas}] Processing schema: ${schemaName} (${schemaEntities.length} entities)`);
+        }
+
+        // 2a. Prepare schema data for LLM
+        const schemaData = this.prepareSchemaData(schemaEntities, schemaName);
+
+        // 2b. Call LLM to generate groups for this schema
+        const llmResponse = await this.callLLMForGrouping(schemaData, contextUser);
+
+        // 2c. Validate and convert to EntityGroup objects
+        const validatedGroups = this.validateAndConvertGroups(llmResponse, schemaEntities);
+
+        if (this.config.verbose && totalSchemas > 1) {
+          LogStatus(`[${schemaIndex}/${totalSchemas}] Generated ${validatedGroups.length} groups for schema: ${schemaName}`);
+        }
+
+        allGroups.push(...validatedGroups);
+      }
+
+      // 3. Deduplicate any similar groups across all schemas
+      const deduplicatedGroups = this.deduplicateGroups(allGroups);
+
+      return deduplicatedGroups;
+    } catch (error: unknown) {
+      throw new Error(extractErrorMessage(error, 'EntityGrouper.generateEntityGroups'));
+    }
+  }
+
+  /**
+   * Group entities by schema name
+   */
+  private groupEntitiesBySchema(entities: EntityInfo[]): Map<string, EntityInfo[]> {
+    const schemaMap = new Map<string, EntityInfo[]>();
+
+    for (const entity of entities) {
+      const schemaName = entity.SchemaName || 'Unknown';
+      const schemaEntities = schemaMap.get(schemaName) || [];
+      schemaEntities.push(entity);
+      schemaMap.set(schemaName, schemaEntities);
+    }
+
+    return schemaMap;
+  }
+
+  /**
+   * Prepare schema data for LLM prompt
+   *
+   * Note: The LLM will determine appropriate group count based on business
+   * domain understanding and schema complexity. We provide the full entity
+   * schema and relationship graph for intelligent analysis.
+   *
+   * @param entities - Entities within a single schema
+   * @param schemaName - Name of the schema being processed
+   */
+  private prepareSchemaData(entities: EntityInfo[], schemaName: string): Record<string, unknown> {
+    const formattedEntities = formatEntitiesForPrompt(entities);
+    const relationshipGraph = generateRelationshipGraph(entities);
+
+    return {
+      schemaName,
+      entities: formattedEntities,
+      relationshipGraph,
+      minGroupSize: this.config.minGroupSize,
+      maxGroupSize: this.config.maxGroupSize
+    };
+  }
+
+  /**
+   * Call LLM via AIPromptRunner to generate entity groups
+   */
+  private async callLLMForGrouping(
+    schemaData: Record<string, unknown>,
+    contextUser: UserInfo
+  ): Promise<LLMEntityGroupResponse> {
+    // Get prompt entity from AIEngine
+    const prompt = AIEngine.Instance.Prompts.find(p => p.Name === this.promptName);
+    if (!prompt) {
+      throw new Error(`Prompt "${this.promptName}" not found. Ensure metadata has been synced to database.`);
+    }
+
+    // Execute with model/vendor overrides if specified in config
+    const result = await executePromptWithOverrides<LLMEntityGroupResponse>(
+      prompt,
+      schemaData,
+      contextUser,
+      this.config
+    );
+
+    if (!result.success) {
+      throw new Error(`LLM grouping failed: ${result.errorMessage || 'Unknown error'}`);
+    }
+
+    if (!result.result) {
+      throw new Error('LLM did not return structured data');
+    }
+
+    return result.result;
+  }
+
+  /**
+   * Validate LLM output and convert to EntityGroup objects
+   */
+  private validateAndConvertGroups(
+    llmResponse: LLMEntityGroupResponse,
+    entities: EntityInfo[]
+  ): EntityGroup[] {
+    const entityMap = new Map(entities.map(e => [e.Name, e]));
+    const validGroups: EntityGroup[] = [];
+
+    for (const llmGroup of llmResponse.groups) {
+      try {
+        // Validate all entity names exist
+        const groupEntities = llmGroup.entities
+          .map(name => entityMap.get(name))
+          .filter((e): e is EntityInfo => e !== undefined);
+
+        if (groupEntities.length !== llmGroup.entities.length) {
+          // Skip logging - verbose debug info that clutters output
+          continue;
+        }
+
+        // Validate primary entity exists
+        const primaryEntity = entityMap.get(llmGroup.primaryEntity);
+        if (!primaryEntity) {
+          // Skip logging - verbose debug info that clutters output
+          continue;
+        }
+
+        // Build relationships array (collect all relationships between entities in the group)
+        const relationships = this.extractRelationships(groupEntities);
+
+        // Validate connectivity (all entities must be reachable from primary)
+        if (groupEntities.length > 1 && !this.isConnected(groupEntities, relationships)) {
+          // Skip logging - verbose debug info that clutters output
+          continue;
+        }
+
+        // Create EntityGroup with LLM metadata
+        validGroups.push({
+          entities: groupEntities,
+          relationships,
+          primaryEntity,
+          relationshipType: llmGroup.relationshipType,
+          businessDomain: llmGroup.businessDomain,
+          businessRationale: llmGroup.businessRationale,
+          expectedQuestionTypes: llmGroup.expectedQuestionTypes
+        });
+      } catch (error: unknown) {
+        // Skip logging - verbose debug info that clutters output
+      }
+    }
+
+    if (validGroups.length === 0) {
+      throw new Error('No valid entity groups generated by LLM');
+    }
+
+    return validGroups;
+  }
+
+  /**
+   * Extract relationships between entities in a group
+   */
+  private extractRelationships(entities: EntityInfo[]): EntityRelationshipInfo[] {
+    const entityNames = new Set(entities.map(e => e.Name));
+    const relationships: EntityRelationshipInfo[] = [];
+
+    for (const entity of entities) {
+      for (const rel of entity.RelatedEntities) {
+        if (entityNames.has(rel.RelatedEntity)) {
+          relationships.push(rel);
+        }
+      }
+    }
+
+    return relationships;
+  }
+
+  /**
+   * Check if all entities in a group are connected by relationships
+   *
+   * Uses BFS to verify all entities are reachable from the first entity
+   */
+  private isConnected(entities: EntityInfo[], relationships: EntityRelationshipInfo[]): boolean {
+    if (entities.length <= 1) return true;
+
+    // Build adjacency map
+    const adjacency = new Map<string, Set<string>>();
+    for (const entity of entities) {
+      adjacency.set(entity.Name, new Set());
+    }
+
+    for (const rel of relationships) {
+      const entityName = entities.find(e =>
+        e.RelatedEntities.includes(rel)
+      )?.Name;
+
+      if (entityName) {
+        adjacency.get(entityName)?.add(rel.RelatedEntity);
+        adjacency.get(rel.RelatedEntity)?.add(entityName); // Bidirectional
+      }
+    }
+
+    // BFS from first entity
+    const visited = new Set<string>();
+    const queue = [entities[0].Name];
+    visited.add(entities[0].Name);
+
+    while (queue.length > 0) {
+      const current = queue.shift()!;
+      const neighbors = adjacency.get(current) || new Set();
+
+      for (const neighbor of neighbors) {
+        if (!visited.has(neighbor)) {
+          visited.add(neighbor);
+          queue.push(neighbor);
+        }
+      }
+    }
+
+    // All entities should be visited
+    return visited.size === entities.length;
+  }
+
+  /**
+   * Remove duplicate or highly similar groups
+   *
+   * Groups are considered duplicates if they contain the exact same set of entities
+   */
+  private deduplicateGroups(groups: EntityGroup[]): EntityGroup[] {
+    const seen = new Set<string>();
+    const unique: EntityGroup[] = [];
+
+    for (const group of groups) {
+      // Create normalized key (sorted entity names)
+      const key = group.entities.map(e => e.Name).sort().join('|');
+
+      if (!seen.has(key)) {
+        seen.add(key);
+        unique.push(group);
+      }
+    }
+
+    return unique;
+  }
+}

package/src/core/MetadataExporter.ts

@@ -0,0 +1,148 @@
+/**
+ * MetadataExporter - Exports validated queries to MJ metadata format
+ *
+ * Transforms validated queries into MemberJunction metadata JSON files
+ * that can be synced to the database using mj-sync.
+ */
+
+import { promises as fs } from 'fs';
+import * as path from 'path';
+import { ValidatedQuery, ExportResult, QueryMetadataRecord, QueryCategoryInfo } from '../data/schema';
+import { generateQueryName } from '../utils/query-helpers';
+
+/**
+ * MetadataExporter class
+ * Exports validated queries to MJ metadata JSON format
+ */
+export class MetadataExporter {
+  /**
+   * Export queries to metadata JSON files
+   *
+   * Transforms validated queries into MemberJunction metadata format
+   * and writes them to timestamped JSON files:
+   * - .query-categories-{timestamp}.json for the categories
+   * - .queries-{timestamp}.json for the queries
+   *
+   * @param validatedQueries - Array of validated queries to export
+   * @param uniqueCategories - Pre-built unique categories for all queries
+   * @param outputDirectory - Directory to write the queries file
+   * @param outputCategoryDirectory - Optional directory for categories file (defaults to outputDirectory)
+   * @returns Export result with file path and query count
+   */
+  async exportQueries(
+    validatedQueries: ValidatedQuery[],
+    uniqueCategories: QueryCategoryInfo[],
+    outputDirectory: string,
+    outputCategoryDirectory?: string
+  ): Promise<ExportResult> {
+    // Generate shared timestamp for both files
+    const timestamp = Date.now();
+
+    // Use category directory if provided, otherwise use queries directory
+    const categoryDir = outputCategoryDirectory || outputDirectory;
+
+    // 1. Transform to MJ metadata format
+    const queryMetadata = validatedQueries.map(q => this.toQueryMetadata(q));
+    const categoryMetadata = uniqueCategories.map(c => this.toCategoryMetadata(c));
+
+    // 2. Ensure output directories exist
+    await fs.mkdir(outputDirectory, { recursive: true });
+    if (categoryDir !== outputDirectory) {
+      await fs.mkdir(categoryDir, { recursive: true });
+    }
+
+    // 3. Write categories file (if categories exist)
+    if (categoryMetadata.length > 0) {
+      const categoriesPath = path.join(categoryDir, `.query-categories-${timestamp}.json`);
+      await fs.writeFile(
+        categoriesPath,
+        JSON.stringify(categoryMetadata, null, 2),
+        'utf-8'
+      );
+    }
+
+    // 4. Write queries file
+    const outputPath = path.join(outputDirectory, `.queries-${timestamp}.json`);
+    await fs.writeFile(
+      outputPath,
+      JSON.stringify(queryMetadata, null, 2),
+      'utf-8'
+    );
+
+    return {
+      success: true,
+      outputPath,
+      queryCount: queryMetadata.length
+    };
+  }
+
+  /**
+   * Transform a validated query into MJ metadata format
+   *
+   * Note: This method only creates the Query record.
+   * QueryFields and QueryParameters are automatically extracted
+   * by QueryEntity.server.ts using AI analysis of the SQL template.
+   * This happens asynchronously during the Save() operation.
+   *
+   * @param query - Validated query to transform
+   * @returns Query metadata record
+   */
+  private toQueryMetadata(query: ValidatedQuery): QueryMetadataRecord {
+    // Build category lookup path (e.g., "Golden-Queries/Members" becomes lookup filter)
+    const categoryLookup = this.buildCategoryLookup(query.category);
+
+    return {
+      fields: {
+        Name: generateQueryName(query.businessQuestion),
+        CategoryID: categoryLookup,
+        UserQuestion: query.businessQuestion.userQuestion,
+        Description: query.businessQuestion.description,
+        TechnicalDescription: query.businessQuestion.technicalDescription,
+        SQL: query.query.sql,
+        OriginalSQL: query.query.sql,
+        UsesTemplate: true,
+        Status: 'Pending'
+      }
+      // relatedEntities removed - QueryEntity.server.ts handles extraction automatically
+    };
+  }
+
+  /**
+   * Transform a QueryCategoryInfo into MJ metadata format
+   *
+   * @param category - Category information
+   * @returns Category metadata record
+   */
+  private toCategoryMetadata(category: QueryCategoryInfo) {
+    return {
+      fields: {
+        Name: category.name,
+        ParentID: category.parentName
+          ? `@lookup:Query Categories.Name=${category.parentName}`
+          : null,
+        Description: category.description,
+        UserID: '@lookup:Users.Name=System'
+      }
+    };
+  }
+
+  /**
+   * Build category lookup string for metadata
+   * Uses multi-field lookup to uniquely identify categories in hierarchies
+   *
+   * For root categories: Name=X&ParentID=null
+   * For child categories: Name=X&Parent=Y (where Parent is the parent category name)
+   *
+   * @param category - Category information
+   * @returns Lookup string for CategoryID field
+   */
+  private buildCategoryLookup(category: QueryCategoryInfo): string {
+    if (category.parentName) {
+      // Child category - use Name and Parent field (view field showing parent name)
+      return `@lookup:Query Categories.Name=${category.name}&Parent=${category.parentName}`;
+    } else {
+      // Root category - use Name and ParentID=null
+      return `@lookup:Query Categories.Name=${category.name}&ParentID=null`;
+    }
+  }
+}

package/src/core/QueryDatabaseWriter.ts

@@ -0,0 +1,187 @@
+/**
+ * QueryDatabaseWriter - Write validated queries directly to the database
+ *
+ * Creates Query entities in the database. QueryFields and QueryParameters
+ * are automatically extracted by QueryEntity.server.ts using AI analysis
+ * of the SQL template during the Save() operation.
+ */
+
+import { Metadata, RunView, UserInfo } from '@memberjunction/core';
+import {
+  QueryEntity,
+  QueryCategoryEntity
+} from '@memberjunction/core-entities';
+import { ValidatedQuery, WriteResult } from '../data/schema';
+import { extractErrorMessage } from '../utils/error-handlers';
+import { generateQueryName } from '../utils/query-helpers';
+
+/**
+ * QueryDatabaseWriter class
+ * Writes validated queries directly to the database
+ */
+export class QueryDatabaseWriter {
+  private categoryCache: Map<string, string> = new Map();
+
+  constructor() {
+    // No config needed - category info comes from ValidatedQuery
+  }
+  /**
+   * Write validated queries to the database
+   *
+   * Creates Query entities. QueryFields and QueryParameters are automatically
+   * extracted by QueryEntity.server.ts using AI analysis of the SQL template.
+   * This happens asynchronously during the Save() operation.
+   *
+   * Errors for individual queries are logged but don't stop the batch process.
+   *
+   * @param validatedQueries - Array of validated queries to write
+   * @param contextUser - User context for entity operations
+   * @returns Write result with success status and per-query results
+   */
+  async writeQueriesToDatabase(
+    validatedQueries: ValidatedQuery[],
+    contextUser: UserInfo
+  ): Promise<WriteResult> {
+    const md = new Metadata();
+    const results: string[] = [];
+
+    // Process each query
+    for (const vq of validatedQueries) {
+      try {
+        // Get or create the category for this query (with caching)
+        const categoryId = await this.getCategoryId(vq.category, contextUser);
+
+        // Create Query entity ONLY (NO manual fields/params creation)
+        // QueryEntity.server.ts will automatically:
+        // - Detect Nunjucks syntax
+        // - Extract parameters using AI
+        // - Create QueryParameter records
+        // - Create QueryField records
+        // - Set UsesTemplate flag
+        const query = await md.GetEntityObject<QueryEntity>('Queries', contextUser);
+        query.NewRecord();
+        query.Name = generateQueryName(vq.businessQuestion);
+        query.CategoryID = categoryId;
+        query.UserQuestion = vq.businessQuestion.userQuestion;
+        query.Description = vq.businessQuestion.description;
+        query.TechnicalDescription = vq.businessQuestion.technicalDescription;
+        query.SQL = vq.query.sql;
+        query.OriginalSQL = vq.query.sql;
+        query.Status = 'Pending';
+
+        const saved = await query.Save();
+        if (!saved) {
+          throw new Error(`Failed to save query: ${query.LatestResult?.Message}`);
+        }
+
+        results.push(`✓ ${query.Name} (ID: ${query.ID}) - AI extraction queued`);
+
+      } catch (error: unknown) {
+        results.push(`✗ ${vq.businessQuestion.userQuestion}: ${extractErrorMessage(error, 'Database Write')}`);
+      }
+    }
+
+    return {
+      success: true,
+      results
+    };
+  }
+
+  /**
+   * Get or create category ID from QueryCategoryInfo
+   * Uses caching to avoid repeated lookups/creations
+   * Ensures parent categories exist before creating children
+   */
+  private async getCategoryId(
+    category: { name: string; parentName: string | null; description: string; path: string },
+    contextUser: UserInfo
+  ): Promise<string> {
+    // Check cache first (by path for uniqueness)
+    if (this.categoryCache.has(category.path)) {
+      return this.categoryCache.get(category.path)!;
+    }
+
+    // If this category has a parent, ensure parent exists first
+    let parentId: string | null = null;
+    if (category.parentName) {
+      // Create parent category info (it's a root category)
+      const parentCategory = {
+        name: category.parentName,
+        parentName: null,
+        description: 'Automatically generated queries from query-gen tool',
+        path: category.parentName
+      };
+      parentId = await this.getCategoryId(parentCategory, contextUser);
+    }
+
+    // Find or create this category
+    const categoryId = await this.findOrCreateCategory(
+      category.name,
+      parentId,
+      category.description,
+      contextUser
+    );
+
+    // Cache for future use
+    this.categoryCache.set(category.path, categoryId);
+    return categoryId;
+  }
+
+  /**
+   * Find or create a query category
+   *
+   * Searches for an existing category with the given name and parent, or creates it if not found.
+   *
+   * @param categoryName - Name of the category to find or create
+   * @param parentCategoryId - Parent category ID (null for root categories)
+   * @param description - Description for new categories
+   * @param contextUser - User context for entity operations
+   * @returns Category ID
+   */
+  private async findOrCreateCategory(
+    categoryName: string,
+    parentCategoryId: string | null,
+    description: string,
+    contextUser: UserInfo
+  ): Promise<string> {
+    const rv = new RunView();
+
+    // Build filter to match both name and parent
+    let filter = `Name='${categoryName.replace(/'/g, "''")}'`;
+    if (parentCategoryId) {
+      filter += ` AND ParentID='${parentCategoryId}'`;
+    } else {
+      filter += ` AND ParentID IS NULL`;
+    }
+
+    const result = await rv.RunView<QueryCategoryEntity>({
+      EntityName: 'Query Categories',
+      ExtraFilter: filter,
+      ResultType: 'entity_object'
+    }, contextUser);
+
+    if (!result.Success) {
+      throw new Error(`Failed to search for category: ${result.ErrorMessage}`);
+    }
+
+    // If found, return existing category ID
+    if (result.Results && result.Results.length > 0) {
+      return result.Results[0].ID;
+    }
+
+    // Category doesn't exist, create it
+    const md = new Metadata();
+    const category = await md.GetEntityObject<QueryCategoryEntity>('Query Categories', contextUser);
+    category.NewRecord();
+    category.Name = categoryName;
+    category.ParentID = parentCategoryId;
+    category.Description = description;
+
+    const saved = await category.Save();
+    if (!saved) {
+      throw new Error(`Failed to create category: ${category.LatestResult?.Message}`);
+    }
+
+    return category.ID;
+  }
+}