@memberjunction/query-gen 0.0.1 → 2.126.1

package/src/data/schema.ts

@@ -0,0 +1,252 @@
+/**
+ * Type definitions for QueryGen package
+ */
+
+import { EntityInfo, EntityRelationshipInfo } from '@memberjunction/core';
+
+/**
+ * Represents a group of related entities for query generation
+ */
+export interface EntityGroup {
+  entities: EntityInfo[];
+  relationships: EntityRelationshipInfo[];
+  primaryEntity: EntityInfo;
+  relationshipType: 'single' | 'parent-child' | 'many-to-many';
+
+  // LLM-generated semantic metadata
+  businessDomain: string;           // e.g., "Sales Pipeline", "Inventory Management"
+  businessRationale: string;        // Why this grouping makes business sense
+  expectedQuestionTypes: string[];  // e.g., ["trend_analysis", "aggregation", "comparison"]
+}
+
+/**
+ * Entity metadata formatted for AI prompts
+ */
+export interface EntityMetadataForPrompt {
+  entityName: string;
+  description: string;
+  schemaName: string;
+  baseTable: string;
+  baseView: string;
+  fields: EntityFieldMetadata[];
+  relationships: EntityRelationshipMetadata[];
+}
+
+/**
+ * Entity field metadata for prompts
+ */
+export interface EntityFieldMetadata {
+  name: string;
+  displayName: string;
+  type: string;
+  sqlFullType: string;
+  description: string;
+  isPrimaryKey: boolean;
+  isForeignKey: boolean;
+  isVirtual: boolean;
+  allowsNull: boolean;
+  relatedEntity?: string;
+  isRequired: boolean;
+  defaultValue?: string;
+  possibleValues?: string[];
+}
+
+/**
+ * Entity relationship metadata for prompts
+ */
+export interface EntityRelationshipMetadata {
+  type: 'one-to-many' | 'many-to-one' | 'many-to-many';
+  relatedEntity: string;
+  relatedEntityView: string;
+  relatedEntitySchema: string;
+  foreignKeyField: string;
+  description: string;
+}
+
+/**
+ * Business question generated by AI
+ */
+export interface BusinessQuestion {
+  userQuestion: string;
+  description: string;
+  technicalDescription: string;
+  complexity: 'simple' | 'medium' | 'complex';
+  requiresAggregation: boolean;
+  requiresJoins: boolean;
+  entities: string[];
+}
+
+/**
+ * SQL query generated by AI
+ * Note: selectClause removed - QueryEntity automatically creates QueryFieldEntity records from SQL
+ */
+export interface GeneratedQuery {
+  sql: string;
+  parameters: QueryParameter[];
+}
+
+/**
+ * Query output field definition
+ */
+export interface QueryOutputField {
+  name: string;
+  description: string;
+  type: string;
+  optional: boolean;
+}
+
+/**
+ * Query parameter definition
+ */
+export interface QueryParameter {
+  name: string;
+  type: string;
+  isRequired: boolean;
+  description: string;
+  usage: string[];
+  defaultValue: string | null;
+  sampleValue: string;
+}
+
+/**
+ * Golden query for few-shot learning
+ */
+export interface GoldenQuery {
+  name: string;
+  userQuestion: string;
+  description: string;
+  technicalDescription: string;
+  sql: string;
+  parameters: QueryParameter[];
+  selectClause: QueryOutputField[];
+}
+
+/**
+ * Similarity search result
+ * Note: nameSim excluded since queries don't have names until after generation
+ */
+export interface SimilarQuery {
+  query: GoldenQuery;
+  similarity: number;
+  fieldScores: {
+    userQuestionSim: number;
+    descSim: number;
+    techDescSim: number;
+  };
+}
+
+/**
+ * Query test result
+ */
+export interface QueryTestResult {
+  success: boolean;
+  renderedSQL?: string;
+  rowCount?: number;
+  sampleRows?: unknown[];
+  attempts?: number;
+  error?: string;
+}
+
+/**
+ * Query evaluation result
+ */
+export interface QueryEvaluation {
+  answersQuestion: boolean;
+  confidence: number;
+  reasoning: string;
+  suggestions: string[];
+  needsRefinement: boolean;
+}
+
+/**
+ * Refined query result
+ */
+export interface RefinedQuery {
+  query: GeneratedQuery;
+  testResult: QueryTestResult;
+  evaluation: QueryEvaluation;
+  refinementCount: number;
+  reachedMaxRefinements?: boolean;
+}
+
+/**
+ * Query category information for metadata export and database write
+ */
+export interface QueryCategoryInfo {
+  /** Name of the category */
+  name: string;
+  /** Parent category name (null for root categories) */
+  parentName: string | null;
+  /** Category description */
+  description: string;
+  /** Full category path (e.g., "Golden-Queries/Members") */
+  path: string;
+}
+
+/**
+ * Validated query ready for export
+ */
+export interface ValidatedQuery {
+  businessQuestion: BusinessQuestion;
+  query: GeneratedQuery;
+  testResult: QueryTestResult;
+  evaluation: QueryEvaluation;
+  entityGroup: EntityGroup;
+  category: QueryCategoryInfo;
+}
+
+/**
+ * Metadata export result
+ */
+export interface ExportResult {
+  success: boolean;
+  outputPath: string;
+  queryCount: number;
+}
+
+/**
+ * Database write result
+ */
+export interface WriteResult {
+  success: boolean;
+  results: string[];
+}
+
+/**
+ * Query metadata record for MJ metadata format
+ *
+ * Note: QueryFields and QueryParameters are not included here.
+ * They are automatically extracted by QueryEntity.server.ts using
+ * AI analysis of the SQL template during the Save() operation.
+ */
+export interface QueryMetadataRecord {
+  fields: {
+    Name: string;
+    CategoryID: string;
+    UserQuestion: string;
+    Description: string;
+    TechnicalDescription: string;
+    SQL: string;
+    OriginalSQL: string;
+    UsesTemplate: boolean;
+    Status: string;
+  };
+}
+
+/**
+ * Embeddings for a query or golden query
+ * Note: name is excluded since queries don't have names until after generation
+ */
+export interface QueryEmbeddings {
+  userQuestion: number[];
+  description: number[];
+  technicalDescription: number[];
+}
+
+/**
+ * Golden query with embeddings
+ */
+export interface EmbeddedGoldenQuery {
+  query: GoldenQuery;
+  embeddings: QueryEmbeddings;
+}

package/src/index.ts

@@ -0,0 +1,49 @@
+/**
+ * QueryGen package main entry point
+ *
+ * @memberjunction/query-gen
+ *
+ * AI-powered generation of domain-specific SQL query templates with
+ * automatic testing, refinement, and metadata export.
+ */
+
+// Export core classes
+export { EntityGrouper } from './core/EntityGrouper';
+export { QuestionGenerator } from './core/QuestionGenerator';
+export { QueryWriter } from './core/QueryWriter';
+export { QueryTester } from './core/QueryTester';
+export { QueryFixer } from './core/QueryFixer';
+export { QueryRefiner } from './core/QueryRefiner';
+export { MetadataExporter } from './core/MetadataExporter';
+export { QueryDatabaseWriter } from './core/QueryDatabaseWriter';
+
+// Export utility classes
+export { SimilaritySearch } from './vectors/SimilaritySearch';
+export { EmbeddingService } from './vectors/EmbeddingService';
+
+// Export types
+export * from './data/schema';
+
+// Export prompt names
+export * from './prompts/PromptNames';
+
+// Export configuration
+export { QueryGenConfig, loadConfig } from './cli/config';
+
+// Export CLI commands
+export { generateCommand } from './cli/commands/generate';
+export { validateCommand } from './cli/commands/validate';
+export { exportCommand } from './cli/commands/export';
+
+// Export utilities
+export { extractErrorMessage, requireValue, getPropertyOrDefault } from './utils/error-handlers';
+export {
+  formatEntityMetadataForPrompt,
+  formatEntityGroupForPrompt,
+  findEntityById,
+  getPrimaryKeyFields,
+  getForeignKeyFields,
+  hasRelationships,
+  getRelationshipCount
+} from './utils/entity-helpers';
+export { generateQueryName } from './utils/query-helpers';

package/src/prompts/PromptNames.ts

@@ -0,0 +1,36 @@
+/**
+ * Static prompt name constants for QueryGen
+ *
+ * All AI prompts used by QueryGen system with their exact names
+ * as defined in the AI Prompt metadata.
+ */
+
+/**
+ * Business Question Generator prompt
+ * Generates 1-2 business questions per entity group
+ */
+export const PROMPT_BUSINESS_QUESTION_GENERATOR = 'Business Question Generator';
+
+/**
+ * SQL Query Writer prompt
+ * Generates Nunjucks SQL templates from business questions
+ */
+export const PROMPT_SQL_QUERY_WRITER = 'SQL Query Writer';
+
+/**
+ * SQL Query Fixer prompt
+ * Fixes SQL syntax and logic errors in generated queries
+ */
+export const PROMPT_SQL_QUERY_FIXER = 'SQL Query Fixer';
+
+/**
+ * Query Result Evaluator prompt
+ * Evaluates if a query correctly answers the business question
+ */
+export const PROMPT_QUERY_EVALUATOR = 'Query Result Evaluator';
+
+/**
+ * Query Refiner prompt
+ * Refines queries based on evaluation feedback
+ */
+export const PROMPT_QUERY_REFINER = 'Query Refiner';

package/src/utils/category-builder.ts

@@ -0,0 +1,97 @@
+/**
+ * CategoryBuilder - Builds QueryCategoryInfo from configuration and entity groups
+ *
+ * Centralizes the logic for determining what categories should be created
+ * for queries based on the QueryGen configuration.
+ */
+
+import { QueryCategoryInfo, EntityGroup } from '../data/schema';
+import { QueryGenConfig } from '../cli/config';
+
+/**
+ * Build category information for a query based on configuration
+ *
+ * @param config - QueryGen configuration
+ * @param entityGroup - Entity group for the query
+ * @returns QueryCategoryInfo with full category details
+ */
+export function buildQueryCategory(
+  config: QueryGenConfig,
+  entityGroup: EntityGroup
+): QueryCategoryInfo {
+  const rootCategoryName = config.rootQueryCategory || 'Auto-Generated';
+
+  if (config.autoCreateEntityQueryCategories) {
+    // Create entity-specific category under root
+    const entityName = entityGroup?.primaryEntity?.Name;
+
+    if (!entityName) {
+      console.warn('buildQueryCategory: primaryEntity.Name is undefined, falling back to root category');
+      return {
+        name: rootCategoryName,
+        parentName: null,
+        description: 'Automatically generated queries from query-gen tool',
+        path: rootCategoryName
+      };
+    }
+
+    return {
+      name: entityName,
+      parentName: rootCategoryName,
+      description: `Queries for the ${entityName} entity`,
+      path: `${rootCategoryName}/${entityName}`
+    };
+  } else {
+    // Use root category only
+    return {
+      name: rootCategoryName,
+      parentName: null,
+      description: 'Automatically generated queries from query-gen tool',
+      path: rootCategoryName
+    };
+  }
+}
+
+/**
+ * Extract all unique categories from validated queries
+ * Returns categories in hierarchical order (root first, then children)
+ *
+ * IMPORTANT: Also creates parent categories for any child categories
+ * (e.g., if "Golden-Queries/Members" is provided, also creates "Golden-Queries")
+ *
+ * @param categories - Array of QueryCategoryInfo from validated queries
+ * @returns Unique categories sorted hierarchically (includes auto-generated parents)
+ */
+export function extractUniqueCategories(categories: QueryCategoryInfo[]): QueryCategoryInfo[] {
+  const uniqueMap = new Map<string, QueryCategoryInfo>();
+
+  // Collect all unique categories (by path)
+  for (const cat of categories) {
+    if (!uniqueMap.has(cat.path)) {
+      uniqueMap.set(cat.path, cat);
+    }
+
+    // If this is a child category, ensure parent category exists
+    if (cat.parentName && !uniqueMap.has(cat.parentName)) {
+      uniqueMap.set(cat.parentName, {
+        name: cat.parentName,
+        parentName: null,
+        description: 'Automatically generated queries from query-gen tool',
+        path: cat.parentName
+      });
+    }
+  }
+
+  // Convert to array and sort: root categories first, then children
+  const result = Array.from(uniqueMap.values());
+  result.sort((a, b) => {
+    // Root categories (no parent) come first
+    if (a.parentName === null && b.parentName !== null) return -1;
+    if (a.parentName !== null && b.parentName === null) return 1;
+
+    // Then sort alphabetically by path
+    return a.path.localeCompare(b.path);
+  });
+
+  return result;
+}

package/src/utils/entity-helpers.ts

@@ -0,0 +1,203 @@
+/**
+ * Entity helper utilities
+ *
+ * Helper functions for working with entity metadata and formatting for AI prompts
+ */
+
+import { EntityInfo, EntityFieldInfo } from '@memberjunction/core';
+import {
+  EntityMetadataForPrompt,
+  EntityFieldMetadata,
+  EntityRelationshipMetadata,
+  EntityGroup
+} from '../data/schema';
+
+/**
+ * Format entity metadata for AI prompt consumption
+ * Includes schema name, base view, fields, and relationships
+ *
+ * CRITICAL: Must include schemaName and baseView for functional SQL generation
+ *
+ * @param entity - Entity to format
+ * @param allEntities - All available entities for relationship lookups
+ * @returns Formatted entity metadata ready for AI prompts
+ */
+export function formatEntityMetadataForPrompt(entity: EntityInfo, allEntities: EntityInfo[]): EntityMetadataForPrompt {
+  return {
+    entityName: entity.Name,
+    description: entity.Description || '',
+    schemaName: entity.SchemaName || 'dbo',
+    baseTable: entity.BaseTable || entity.Name,
+    baseView: entity.BaseView || `vw${entity.Name}`,
+    fields: formatEntityFields(entity),
+    relationships: formatEntityRelationships(entity, allEntities),
+  };
+}
+
+/**
+ * Format entity fields for AI prompt
+ * Includes field metadata with types, descriptions, and relationship info
+ * Labels internal (__mj_*) and virtual fields appropriately
+ */
+function formatEntityFields(entity: EntityInfo): EntityFieldMetadata[] {
+  return entity.Fields.map((field) => {
+    const isInternalField = field.Name.startsWith('__mj_');
+    const isVirtualField = field.IsVirtual || false;
+
+    let description = field.Description || '';
+
+    // Add labels for special fields
+    if (isInternalField) {
+      description = `[INTERNAL] ${description}`;
+    } else if (isVirtualField) {
+      description = `[VIRTUAL] ${description}`;
+    }
+
+    // Extract possible values from EntityFieldValues if available
+    const possibleValues = field.EntityFieldValues && field.EntityFieldValues.length > 0
+      ? field.EntityFieldValues.map(efv => efv.Value)
+      : undefined;
+
+    return {
+      name: field.Name,
+      displayName: field.DisplayName || field.Name,
+      type: field.Type,
+      sqlFullType: field.SQLFullType,
+      description,
+      isPrimaryKey: field.IsPrimaryKey || false,
+      isForeignKey: field.RelatedEntityID != null && field.RelatedEntityID.trim().length > 0,
+      isVirtual: isVirtualField,
+      allowsNull: field.AllowsNull,
+      relatedEntity: field.RelatedEntity || undefined,
+      isRequired: !field.AllowsNull,
+      defaultValue: field.DefaultValue || undefined,
+      possibleValues,
+    };
+  });
+}
+
+/**
+ * Format entity relationships for AI prompt
+ * Includes schema and view names for proper JOIN generation
+ * Only includes relationships to entities in the current entity group
+ * Uses EntityInfo.RelatedEntities getter for pre-computed relationships
+ */
+function formatEntityRelationships(entity: EntityInfo, allEntities: EntityInfo[]): EntityRelationshipMetadata[] {
+  // Create set of entity names in the group for filtering
+  const entityNamesInGroup = new Set(allEntities.map(e => e.Name));
+
+  // Use entity.RelatedEntities getter which returns EntityRelationshipInfo[]
+  return entity.RelatedEntities
+    .filter(rel => entityNamesInGroup.has(rel.RelatedEntity))  // Only include relationships within the group
+    .map(rel => {
+      const relatedEntity = findEntityById(rel.RelatedEntityID, allEntities);
+
+      // Determine the foreign key field based on relationship type and available fields
+      let foreignKeyField: string;
+      let joinDescription: string;
+
+      const currentSchema = entity.SchemaName || 'dbo';
+      const currentView = entity.BaseView || `vw${entity.Name}`;
+      const relatedSchema = relatedEntity?.SchemaName || 'dbo';
+      const relatedView = relatedEntity?.BaseView || `vw${rel.RelatedEntity}`;
+
+      if (rel.EntityKeyField && rel.EntityKeyField.trim() !== '') {
+        // Current entity has the foreign key
+        foreignKeyField = rel.EntityKeyField;
+        const relatedJoinField = rel.RelatedEntityJoinField || 'ID';
+        joinDescription = `${currentSchema}.${currentView}.${foreignKeyField} = ${relatedSchema}.${relatedView}.${relatedJoinField}`;
+      } else if (rel.RelatedEntityJoinField && rel.RelatedEntityJoinField.trim() !== '') {
+        // Related entity has the foreign key pointing back to this entity
+        foreignKeyField = rel.RelatedEntityJoinField;
+        joinDescription = `${relatedSchema}.${relatedView}.${foreignKeyField} = ${currentSchema}.${currentView}.ID`;
+      } else {
+        // No foreign key field specified (possibly many-to-many through join table)
+        foreignKeyField = '';
+        joinDescription = `Related via ${rel.JoinView || 'join table'}`;
+      }
+
+      return {
+        type: mapRelationshipType(rel.Type),
+        relatedEntity: rel.RelatedEntity,
+        relatedEntityView: relatedEntity?.BaseView || `vw${rel.RelatedEntity}`,
+        relatedEntitySchema: relatedEntity?.SchemaName || 'dbo',
+        foreignKeyField,
+        description: joinDescription,
+      };
+    });
+}
+
+/**
+ * Map MJ relationship types to QueryGen types
+ */
+function mapRelationshipType(mjType: string): 'one-to-many' | 'many-to-one' | 'many-to-many' {
+  const normalized = mjType.toLowerCase().replace(/\s+/g, '-');
+  if (normalized === 'many-to-one') return 'many-to-one';
+  if (normalized === 'one-to-many') return 'one-to-many';
+  if (normalized === 'many-to-many') return 'many-to-many';
+  return 'many-to-one'; // Default fallback
+}
+
+/**
+ * Check if a field is a foreign key field
+ */
+function isForeignKeyField(field: EntityFieldInfo): boolean {
+  return field.RelatedEntityID != null && field.RelatedEntityID.trim().length > 0;
+}
+
+/**
+ * Find entity by name in array
+ */
+function findEntityByName(name: string, entities: EntityInfo[]): EntityInfo | undefined {
+  return entities.find((e) => e.Name === name);
+}
+
+/**
+ * Find entity by ID in array
+ */
+export function findEntityById(id: string, entities: EntityInfo[]): EntityInfo | undefined {
+  return entities.find((e) => e.ID === id);
+}
+
+/**
+ * Get primary key field(s) for an entity
+ */
+export function getPrimaryKeyFields(entity: EntityInfo): EntityFieldInfo[] {
+  return entity.Fields.filter((f) => f.IsPrimaryKey);
+}
+
+/**
+ * Get foreign key fields for an entity
+ */
+export function getForeignKeyFields(entity: EntityInfo): EntityFieldInfo[] {
+  return entity.Fields.filter((f) => isForeignKeyField(f));
+}
+
+/**
+ * Check if an entity has any relationships
+ * Uses EntityInfo.RelatedEntities getter
+ */
+export function hasRelationships(entity: EntityInfo, allEntities: EntityInfo[]): boolean {
+  return entity.RelatedEntities.length > 0;
+}
+
+/**
+ * Get count of relationships for an entity
+ * Uses EntityInfo.RelatedEntities getter
+ */
+export function getRelationshipCount(entity: EntityInfo, allEntities: EntityInfo[]): number {
+  return entity.RelatedEntities.length;
+}
+
+/**
+ * Format an entire entity group for AI prompt consumption
+ * Converts all entities in the group to structured metadata
+ *
+ * @param entityGroup - Entity group to format
+ * @returns Array of formatted entity metadata for Nunjucks template
+ */
+export function formatEntityGroupForPrompt(entityGroup: EntityGroup): EntityMetadataForPrompt[] {
+  return entityGroup.entities.map((entity) =>
+    formatEntityMetadataForPrompt(entity, entityGroup.entities)
+  );
+}

package/src/utils/error-handlers.ts

@@ -0,0 +1,41 @@
+/**
+ * Error handling utilities
+ *
+ * Standardized error handling functions following MJ patterns
+ */
+
+/**
+ * Extract error message from unknown error type
+ * Uses MJ's extractErrorMessage pattern
+ */
+export function extractErrorMessage(error: unknown, context: string): string {
+  if (error instanceof Error) {
+    return `${context}: ${error.message}`;
+  }
+  if (typeof error === 'string') {
+    return `${context}: ${error}`;
+  }
+  return `${context}: Unknown error occurred`;
+}
+
+/**
+ * Validate required value is not null/undefined
+ */
+export function requireValue<T>(value: T | null | undefined, fieldName: string): T {
+  if (value === null || value === undefined) {
+    throw new Error(`Required value '${fieldName}' is missing`);
+  }
+  return value;
+}
+
+/**
+ * Get property with fallback value
+ */
+export function getPropertyOrDefault<T>(
+  obj: Record<string, unknown>,
+  key: string,
+  defaultValue: T
+): T {
+  const value = obj[key];
+  return value !== undefined ? (value as T) : defaultValue;
+}