@memberjunction/query-gen 0.0.1 → 2.126.1

package/package.json

@@ -1,10 +1,52 @@
 {
   "name": "@memberjunction/query-gen",
-  "version": "0.0.1",
-  "description": "OIDC trusted publishing setup package for @memberjunction/query-gen",
+  "version": "2.126.1",
+  "description": "AI-powered generation of domain-specific SQL query templates with automatic testing, refinement, and metadata export",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "bin": {
+    "mj-querygen": "./dist/cli/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "watch": "tsc --watch",
+    "clean": "rm -rf dist",
+    "lint": "eslint src --ext .ts",
+    "format": "prettier --write \"src/**/*.ts\""
+  },
   "keywords": [
-    "oidc",
-    "trusted-publishing",
-    "setup"
-  ]
+    "memberjunction",
+    "sql",
+    "query",
+    "generation",
+    "ai",
+    "templates"
+  ],
+  "author": "MemberJunction",
+  "license": "MIT",
+  "dependencies": {
+    "@memberjunction/ai": "2.126.1",
+    "@memberjunction/ai-core-plus": "2.126.1",
+    "@memberjunction/aiengine": "2.126.1",
+    "@memberjunction/ai-prompts": "2.126.1",
+    "@memberjunction/ai-vectors-memory": "2.126.1",
+    "@memberjunction/core": "2.126.1",
+    "@memberjunction/core-entities": "2.126.1",
+    "@memberjunction/global": "2.126.1",
+    "@memberjunction/sqlserver-dataprovider": "2.126.1",
+    "chalk": "^4.1.2",
+    "commander": "^11.1.0",
+    "nunjucks": "^3.2.4",
+    "ora": "^5.4.1"
+  },
+  "devDependencies": {
+    "@types/node": "20.14.2",
+    "@types/nunjucks": "^3.2.6",
+    "ts-node-dev": "^2.0.0",
+    "typescript": "^5.4.5"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/MemberJunction/MJ"
+  }
 }

package/src/cli/commands/export.ts

@@ -0,0 +1,173 @@
+/**
+ * Export command - Export queries from database to metadata files
+ *
+ * Reads existing queries from the database and exports them to metadata format:
+ * - Loads all queries from Queries table
+ * - Includes related Query Fields and Query Params
+ * - Exports to JSON files in MJ metadata format
+ */
+
+import ora from 'ora';
+import chalk from 'chalk';
+import * as fs from 'fs';
+import * as path from 'path';
+import { Metadata, RunView, UserInfo } from '@memberjunction/core';
+import { getSystemUser } from '../../utils/user-helpers';
+import { QueryEntity } from '@memberjunction/core-entities';
+import { extractErrorMessage } from '../../utils/error-handlers';
+import { QueryMetadataRecord } from '../../data/schema';
+
+/**
+ * Execute the export command
+ *
+ * Loads queries from database and exports them to metadata files.
+ */
+export async function exportCommand(options: Record<string, unknown>): Promise<void> {
+  const spinner = ora('Initializing export...').start();
+
+  try {
+    const outputPath = String(options.output || './metadata/queries');
+    const verbose = Boolean(options.verbose);
+
+    // 1. Get system user from UserCache (populated by provider initialization)
+    const contextUser = getSystemUser();
+
+    // 2. Verify database connection and load metadata
+    spinner.text = 'Loading metadata...';
+    // Assume provider is already configured by the calling application
+    if (!Metadata.Provider) {
+      throw new Error('Metadata provider not configured. Please ensure database connection is set up before running CLI.');
+    }
+    spinner.succeed('Metadata loaded');
+
+    // 3. Load queries from database
+    spinner.start('Loading queries from database...');
+    const queries = await loadQueriesFromDatabase(contextUser);
+    spinner.succeed(chalk.green(`Found ${queries.length} queries`));
+
+    // 4. Create output directory if it doesn't exist
+    if (!fs.existsSync(outputPath)) {
+      fs.mkdirSync(outputPath, { recursive: true });
+    }
+
+    // 5. Export each query to metadata format
+    let exportCount = 0;
+    const errors: Array<{ query: string; error: string }> = [];
+
+    for (let i = 0; i < queries.length; i++) {
+      const query = queries[i];
+      const queryPrefix = chalk.cyan(`[${i + 1}/${queries.length}]`);
+
+      spinner.start(`${queryPrefix} Exporting ${chalk.bold(query.Name)}...`);
+
+      try {
+        const metadataRecord = await convertQueryToMetadata(query, contextUser);
+        const filename = sanitizeFilename(query.Name) + '.json';
+        const fullPath = path.join(outputPath, filename);
+
+        fs.writeFileSync(fullPath, JSON.stringify(metadataRecord, null, 2), 'utf-8');
+        exportCount++;
+
+        if (verbose) {
+          spinner.info(`${queryPrefix} ${chalk.green('✓')} Exported ${query.Name}`);
+        }
+      } catch (error: unknown) {
+        const errorMsg = extractErrorMessage(error, 'Query Export');
+        errors.push({ query: query.Name, error: errorMsg });
+        if (verbose) {
+          spinner.warn(`${queryPrefix} ${chalk.red('✗')} ${query.Name}: ${errorMsg}`);
+        }
+      }
+    }
+
+    // 6. Summary
+    if (errors.length === 0) {
+      spinner.succeed(chalk.green.bold(`✓ All ${exportCount} queries exported successfully!`));
+      console.log('\n' + chalk.green.bold('✓ Export complete!\n'));
+      console.log(chalk.bold('Summary:'));
+      console.log(`  Total Queries: ${chalk.cyan(exportCount.toString())}`);
+      console.log(`  Exported: ${chalk.green(exportCount.toString())}`);
+      console.log(`  Output Location: ${chalk.dim(outputPath)}`);
+      process.exit(0);
+    } else {
+      spinner.fail(chalk.yellow(`Export completed with ${errors.length} errors`));
+      console.log('\n' + chalk.yellow.bold('⚠ Export completed with errors\n'));
+      console.log(chalk.bold('Summary:'));
+      console.log(`  Total Queries: ${chalk.cyan(queries.length.toString())}`);
+      console.log(`  Exported: ${chalk.green(exportCount.toString())}`);
+      console.log(`  Failed: ${chalk.red(errors.length.toString())}`);
+      console.log(`  Output Location: ${chalk.dim(outputPath)}`);
+
+      if (errors.length > 0) {
+        console.log('\n' + chalk.bold('Errors:'));
+        for (const { query, error } of errors) {
+          console.log(chalk.red(`  ${query}: ${error}`));
+        }
+      }
+      process.exit(1);
+    }
+
+  } catch (error: unknown) {
+    spinner.fail(chalk.red('Export failed'));
+    console.error(chalk.red(extractErrorMessage(error, 'Query Export')));
+    process.exit(1);
+  }
+}
+
+/**
+ * Load all queries from the database
+ */
+async function loadQueriesFromDatabase(contextUser: UserInfo): Promise<QueryEntity[]> {
+  const rv = new RunView();
+  const result = await rv.RunView<QueryEntity>(
+    {
+      EntityName: 'Queries',
+      ExtraFilter: '',
+      OrderBy: 'Name',
+      ResultType: 'entity_object',
+    },
+    contextUser
+  );
+
+  if (!result.Success) {
+    throw new Error(`Failed to load queries: ${result.ErrorMessage}`);
+  }
+
+  return result.Results || [];
+}
+
+/**
+ * Convert QueryEntity to metadata record format
+ *
+ * Note: For the export command, we only export the Query entity itself.
+ * QueryFields and QueryParameters are managed by QueryEntity.server.ts and
+ * will be automatically extracted when the query is imported/saved.
+ */
+async function convertQueryToMetadata(
+  query: QueryEntity,
+  contextUser: UserInfo
+): Promise<QueryMetadataRecord> {
+  return {
+    fields: {
+      Name: query.Name,
+      CategoryID: query.CategoryID || 'unknown',
+      UserQuestion: query.UserQuestion || '',
+      Description: query.Description || '',
+      TechnicalDescription: query.TechnicalDescription || '',
+      SQL: query.SQL || '',
+      OriginalSQL: query.OriginalSQL || query.SQL || '',
+      UsesTemplate: query.UsesTemplate || false,
+      Status: query.Status || 'Active',
+    },
+  };
+}
+
+/**
+ * Sanitize filename by removing invalid characters
+ */
+function sanitizeFilename(name: string): string {
+  return name
+    .replace(/[^a-zA-Z0-9-_\s]/g, '')
+    .replace(/\s+/g, '_')
+    .toLowerCase();
+}

package/src/cli/commands/generate.ts

@@ -0,0 +1,330 @@
+/**
+ * Generate command - Main query generation workflow
+ *
+ * Orchestrates the full query generation pipeline:
+ * 1. Load entity groups
+ * 2. Generate business questions
+ * 3. Generate SQL queries
+ * 4. Test and fix queries
+ * 5. Refine queries
+ * 6. Export results
+ */
+
+import ora from 'ora';
+import chalk from 'chalk';
+import * as fs from 'fs';
+import * as path from 'path';
+import { Metadata, DatabaseProviderBase, EntityInfo, LogStatus } from '@memberjunction/core';
+import { loadConfig } from '../config';
+import { getSystemUser } from '../../utils/user-helpers';
+import { EntityGrouper } from '../../core/EntityGrouper';
+import { QuestionGenerator } from '../../core/QuestionGenerator';
+import { QueryWriter } from '../../core/QueryWriter';
+import { QueryTester } from '../../core/QueryTester';
+import { QueryRefiner } from '../../core/QueryRefiner';
+import { MetadataExporter } from '../../core/MetadataExporter';
+import { QueryDatabaseWriter } from '../../core/QueryDatabaseWriter';
+import { EmbeddingService } from '../../vectors/EmbeddingService';
+import { SimilaritySearch } from '../../vectors/SimilaritySearch';
+import { formatEntityMetadataForPrompt } from '../../utils/entity-helpers';
+import { extractErrorMessage } from '../../utils/error-handlers';
+import { buildQueryCategory, extractUniqueCategories } from '../../utils/category-builder';
+import { ValidatedQuery, GoldenQuery, EntityGroup, QueryCategoryInfo } from '../../data/schema';
+
+/**
+ * Execute the generate command
+ *
+ * Full orchestration of query generation workflow with progress reporting.
+ * Uses ora for spinners and chalk for colored output.
+ */
+export async function generateCommand(options: Record<string, unknown>): Promise<void> {
+  const spinner = ora('Initializing query generation...').start();
+
+  try {
+    // 1. Load configuration
+    spinner.text = 'Loading configuration...';
+    const config = loadConfig(options);
+
+    // Show model/vendor overrides if configured
+    if (config.modelOverride || config.vendorOverride) {
+      const overrideMsg = [];
+      if (config.modelOverride) overrideMsg.push(`Model: ${config.modelOverride}`);
+      if (config.vendorOverride) overrideMsg.push(`Vendor: ${config.vendorOverride}`);
+      spinner.info(chalk.cyan(`Using overrides - ${overrideMsg.join(', ')}`));
+    }
+
+    if (config.verbose) {
+      spinner.info(chalk.dim('Configuration loaded'));
+      console.log(chalk.dim(JSON.stringify(config, null, 2)));
+    }
+
+    // 2. Get system user from UserCache (populated by provider initialization)
+    const contextUser = getSystemUser();
+
+    // 3. Verify database connection and metadata
+    spinner.text = 'Loading metadata...';
+    // Assume provider and AIEngine are already configured by the calling application (MJCLI)
+    if (!Metadata.Provider) {
+      throw new Error('Metadata provider not configured. Please ensure database connection is set up before running CLI.');
+    }
+    spinner.succeed('Metadata loaded');
+
+    // 4. Filter and build entity groups
+    spinner.start('Filtering entities...');
+    const md = new Metadata();
+
+    // Apply entity filtering (includeEntities takes precedence over excludeEntities)
+    let filteredEntities = md.Entities.filter(
+      e => !config.excludeSchemas.includes(e.SchemaName || '')
+    );
+
+    if (config.includeEntities.length > 0) {
+      // Allowlist: only include specified entities
+      filteredEntities = filteredEntities.filter(e => config.includeEntities.includes(e.Name));
+      spinner.info(chalk.dim(`Including only ${config.includeEntities.length} specified entities`));
+    } else if (config.excludeEntities.length > 0) {
+      // Denylist: exclude specified entities
+      filteredEntities = filteredEntities.filter(e => !config.excludeEntities.includes(e.Name));
+      spinner.info(chalk.dim(`Excluded ${config.excludeEntities.length} entities`));
+    }
+
+    // 4. Group entities by schema and generate entity groups
+    spinner.text = 'Analyzing entity relationships...';
+
+    // Count entities per schema for informational logging
+    const schemaCount = new Set(filteredEntities.map(e => e.SchemaName)).size;
+    if (config.verbose && schemaCount > 1) {
+      spinner.info(chalk.dim(`Processing ${schemaCount} schemas separately`));
+    }
+
+    const grouper = new EntityGrouper(config);
+    const entityGroups = await grouper.generateEntityGroups(filteredEntities, contextUser);
+    spinner.succeed(chalk.green(`Found ${entityGroups.length} entity groups across ${schemaCount} ${schemaCount === 1 ? 'schema' : 'schemas'}`));
+
+
+    // 5. Initialize vector similarity search
+    spinner.start('Embedding golden queries...');
+    const embeddingService = new EmbeddingService(config.embeddingModel);
+    const goldenQueries = await loadGoldenQueries(config);
+    const embeddedGolden = await embeddingService.embedGoldenQueries(goldenQueries);
+    spinner.succeed(chalk.green(`Embedded ${goldenQueries.length} golden queries`));
+
+    // 5b. Build category structure for all entity groups upfront
+    spinner.start('Building category structure...');
+    const categoryMap = new Map<string, QueryCategoryInfo>();
+    for (const group of entityGroups) {
+      const category = buildQueryCategory(config, group);
+      // Use primary entity name as key for lookup during query generation
+      categoryMap.set(group.primaryEntity.Name, category);
+    }
+    const uniqueCategories = extractUniqueCategories(Array.from(categoryMap.values()));
+    spinner.succeed(chalk.green(`Created ${uniqueCategories.length} ${uniqueCategories.length === 1 ? 'category' : 'categories'}`));
+
+    // 6. Generate queries for each entity group
+    const totalGroups = entityGroups.length;
+    let processedGroups = 0;
+    const allValidatedQueries: ValidatedQuery[] = [];
+
+    for (const group of entityGroups) {
+      processedGroups++;
+      const groupPrefix = chalk.cyan(`[${processedGroups}/${totalGroups}]`);
+      spinner.start(`${groupPrefix} Processing ${chalk.bold(group.primaryEntity.Name)}...`);
+
+      let queriesCreatedForGroup = 0;
+
+      try {
+        // 6a. Generate business questions for this entity group
+        const questionGenerator = new QuestionGenerator(contextUser, config);
+        const questions = await questionGenerator.generateQuestions(group);
+
+        if (config.verbose) {
+          spinner.info(`${groupPrefix} Generated ${questions.length} questions`);
+        }
+
+        // 6b. For each question, generate and validate query
+        for (const question of questions) {
+          spinner.text = `${groupPrefix} Generating query: ${chalk.italic(question.userQuestion)}`;
+
+          // Embed question for similarity search
+          const questionEmbedding = await embeddingService.embedQuery({
+            userQuestion: question.userQuestion,
+            description: question.description,
+            technicalDescription: question.technicalDescription,
+          });
+
+          // Find similar golden queries
+          const similaritySearch = new SimilaritySearch();
+          const fewShotResults = await similaritySearch.findSimilarQueries(
+            questionEmbedding,
+            embeddedGolden,
+            config.topSimilarQueries
+          );
+          const fewShotExamples = fewShotResults.map(s => s.query);
+
+          // Generate SQL query
+          const queryWriter = new QueryWriter(contextUser, config);
+          const generatedQuery = await queryWriter.generateQuery(
+            question,
+            group.entities.map((e: EntityInfo) => formatEntityMetadataForPrompt(e, group.entities)),
+            fewShotExamples
+          );
+
+          // Test and fix query
+          // Access the database provider through Metadata.Provider
+          const dataProvider = Metadata.Provider as DatabaseProviderBase;
+          const entityMetadata = group.entities.map((e: EntityInfo) => formatEntityMetadataForPrompt(e, group.entities));
+          const queryTester = new QueryTester(
+            dataProvider,
+            entityMetadata,
+            question,
+            contextUser,
+            config
+          );
+          const testResult = await queryTester.testQuery(
+            generatedQuery,
+            config.maxFixingIterations
+          );
+
+          if (!testResult.success) {
+            spinner.warn(
+              chalk.yellow(`${groupPrefix} Query failed after ${config.maxFixingIterations} attempts: ${question.userQuestion}`)
+            );
+            continue;
+          }
+
+          // Refine query
+          const queryRefiner = new QueryRefiner(queryTester, contextUser, config);
+          const refinedResult = await queryRefiner.refineQuery(
+            generatedQuery,
+            question,
+            entityMetadata,
+            config.maxRefinementIterations
+          );
+
+          // Get pre-built category from map
+          const category = categoryMap.get(group.primaryEntity.Name);
+          if (!category) {
+            throw new Error(`Category not found for entity group: ${group.primaryEntity.Name}`);
+          }
+
+          allValidatedQueries.push({
+            businessQuestion: question,
+            query: refinedResult.query,
+            testResult: refinedResult.testResult,
+            evaluation: refinedResult.evaluation,
+            entityGroup: group,
+            category,
+          });
+
+          queriesCreatedForGroup++;
+
+          if (config.verbose) {
+            spinner.info(`${groupPrefix} ${chalk.green('✓')} ${question.userQuestion}`);
+          }
+        }
+
+        // Format entity list with primary entity in bold
+        const entityDisplay = group.entities
+          .map(e => e.Name === group.primaryEntity.Name ? chalk.bold(e.Name) : e.Name)
+          .join(', ');
+
+        spinner.succeed(
+          `${groupPrefix} ${entityDisplay} complete (${chalk.green(queriesCreatedForGroup + ' queries')})`
+        );
+
+      } catch (error: unknown) {
+        // Format entity list with primary entity in bold
+        const entityDisplay = group.entities
+          .map(e => e.Name === group.primaryEntity.Name ? chalk.bold(e.Name) : e.Name)
+          .join(', ');
+
+        spinner.warn(
+          chalk.yellow(
+            `${groupPrefix} Error processing ${entityDisplay}: ${extractErrorMessage(error, 'Query Generation')}`
+          )
+        );
+      }
+    }
+
+    // 7. Export results
+    spinner.start(`Exporting ${allValidatedQueries.length} queries...`);
+
+    if (config.outputMode === 'metadata' || config.outputMode === 'both') {
+      const exporter = new MetadataExporter();
+      const exportResult = await exporter.exportQueries(
+        allValidatedQueries,
+        uniqueCategories,
+        config.outputDirectory,
+        config.outputCategoryDirectory
+      );
+      spinner.succeed(chalk.green(`Exported to ${exportResult.outputPath}`));
+    }
+
+    if (config.outputMode === 'database' || config.outputMode === 'both') {
+      const dbWriter = new QueryDatabaseWriter();
+      await dbWriter.writeQueriesToDatabase(allValidatedQueries, contextUser);
+      spinner.succeed(chalk.green(`Wrote ${allValidatedQueries.length} queries to database`));
+    }
+
+    // 8. Summary
+    console.log('\n' + chalk.green.bold('✓ Query generation complete!\n'));
+    console.log(chalk.bold('Summary:'));
+    console.log(`  Entity Groups Processed: ${chalk.cyan(processedGroups.toString())}`);
+    console.log(`  Queries Generated: ${chalk.green(allValidatedQueries.length.toString())}`);
+    console.log(`  Output Location: ${chalk.dim(config.outputDirectory)}`);
+
+    process.exit(0);
+
+  } catch (error: unknown) {
+    spinner.fail(chalk.red('Query generation failed'));
+    console.error(chalk.red(extractErrorMessage(error, 'Query Generation')));
+    process.exit(1);
+  }
+}
+
+/**
+ * Load golden queries from JSON file
+ *
+ * Golden queries are example queries used for few-shot learning.
+ * They are stored in the data/golden-queries.json file.
+ *
+ * @param config - QueryGen configuration with verbose flag
+ * @returns Array of golden queries, or empty array if file not found/invalid
+ */
+async function loadGoldenQueries(config: { verbose: boolean }): Promise<GoldenQuery[]> {
+  try {
+    // Resolve path to golden-queries.json in the data directory
+    // __dirname points to dist/cli/commands, so we go up to dist, then to data
+    const goldenQueriesPath = path.join(__dirname, '../../data/golden-queries.json');
+
+    // Check if file exists
+    if (!fs.existsSync(goldenQueriesPath)) {
+      if (config.verbose) {
+        LogStatus(`[Warning] Golden queries file not found at: ${goldenQueriesPath}`);
+      }
+      return [];
+    }
+
+    // Read and parse JSON file
+    const fileContent = fs.readFileSync(goldenQueriesPath, 'utf-8');
+    const goldenQueries = JSON.parse(fileContent) as GoldenQuery[];
+
+    // Validate that it's an array
+    if (!Array.isArray(goldenQueries)) {
+      if (config.verbose) {
+        LogStatus('[Warning] Golden queries file does not contain an array');
+      }
+      return [];
+    }
+
+    if (config.verbose) {
+      LogStatus(`[Info] Loaded ${goldenQueries.length} golden queries for few-shot learning`);
+    }
+    return goldenQueries;
+  } catch (error: unknown) {
+    if (config.verbose) {
+      LogStatus(`[Warning] Failed to load golden queries: ${extractErrorMessage(error, 'loadGoldenQueries')}`);
+    }
+    return [];
+  }
+}