fhirsmith 0.18.2 → 1.0.0

package/tx/importers/import-ndc.module.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const sqlite3 = require('sqlite3').verbose();
 const path = require('path');
 const readline = require('readline');
+const terminal = require('./terminal');
 
 class NdcModule extends BaseTerminologyModule {
   getName() {
@@ -94,18 +95,18 @@ class NdcModule extends BaseTerminologyModule {
     const inquirer = require('inquirer');
     const chalk = require('chalk');
 
-    console.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
-    console.log(`  Source: ${chalk.white(config.source)}`);
-    console.log(`  Destination: ${chalk.white(config.dest)}`);
-    console.log(`  Version: ${chalk.white(config.version)}`);
-    console.log(`  Products Only: ${chalk.white(config.productsOnly ? 'Yes' : 'No')}`);
-    console.log(`  Packages Only: ${chalk.white(config.packagesOnly ? 'Yes' : 'No')}`);
-    console.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
-    console.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
-    console.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
+    terminal.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
+    terminal.log(`  Source: ${chalk.white(config.source)}`);
+    terminal.log(`  Destination: ${chalk.white(config.dest)}`);
+    terminal.log(`  Version: ${chalk.white(config.version)}`);
+    terminal.log(`  Products Only: ${chalk.white(config.productsOnly ? 'Yes' : 'No')}`);
+    terminal.log(`  Packages Only: ${chalk.white(config.packagesOnly ? 'Yes' : 'No')}`);
+    terminal.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
+    terminal.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
+    terminal.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
 
     if (config.estimatedDuration) {
-      console.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
+      terminal.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
     }
 
     const { confirmed } = await inquirer.prompt({
@@ -135,14 +136,14 @@ class NdcModule extends BaseTerminologyModule {
       const stats = await this.validateNdcDirectory(options.source, parseInt(options.sample));
 
       this.logSuccess('Directory validation passed');
-      console.log(`  Versions found: ${stats.versions.length}`);
-      console.log(`  Total products estimated: ${stats.totalProducts.toLocaleString()}`);
-      console.log(`  Total packages estimated: ${stats.totalPackages.toLocaleString()}`);
-      console.log(`  Versions: ${stats.versions.join(', ')}`);
+      terminal.log(`  Versions found: ${stats.versions.length}`);
+      terminal.log(`  Total products estimated: ${stats.totalProducts.toLocaleString()}`);
+      terminal.log(`  Total packages estimated: ${stats.totalPackages.toLocaleString()}`);
+      terminal.log(`  Versions: ${stats.versions.join(', ')}`);
 
       if (stats.warnings.length > 0) {
         this.logWarning('Validation warnings:');
-        stats.warnings.forEach(warning => console.log(`    ${warning}`));
+        stats.warnings.forEach(warning => terminal.log(`    ${warning}`));
       }
 
     } catch (error) {
@@ -166,7 +167,7 @@ class NdcModule extends BaseTerminologyModule {
 
       this.logSuccess(`Found ${versions.length} NDC versions:`);
       versions.forEach((version, index) => {
-        console.log(`  ${index + 1}. ${version}`);
+        terminal.log(`  ${index + 1}. ${version}`);
       });
 
     } catch (error) {
@@ -188,13 +189,13 @@ class NdcModule extends BaseTerminologyModule {
       const stats = await this.getDatabaseStats(dbPath);
 
       this.logSuccess('Database status:');
-      console.log(`  Versions: ${stats.versions.join(', ')}`);
-      console.log(`  Products: ${stats.productCount.toLocaleString()}`);
-      console.log(`  Packages: ${stats.packageCount.toLocaleString()}`);
-      console.log(`  Organizations: ${stats.orgCount.toLocaleString()}`);
-      console.log(`  Product Types: ${stats.typeCount.toLocaleString()}`);
-      console.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
-      console.log(`  Last Modified: ${stats.lastModified}`);
+      terminal.log(`  Versions: ${stats.versions.join(', ')}`);
+      terminal.log(`  Products: ${stats.productCount.toLocaleString()}`);
+      terminal.log(`  Packages: ${stats.packageCount.toLocaleString()}`);
+      terminal.log(`  Organizations: ${stats.orgCount.toLocaleString()}`);
+      terminal.log(`  Product Types: ${stats.typeCount.toLocaleString()}`);
+      terminal.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
+      terminal.log(`  Last Modified: ${stats.lastModified}`);
 
     } catch (error) {
       this.logError(`Status check failed: ${error.message}`);
@@ -427,7 +428,7 @@ class NdcModule extends BaseTerminologyModule {
       db.serialize(() => {
         indexes.forEach(sql => {
           db.run(sql, (err) => {
-            if (err) console.warn(`Index creation warning: ${err.message}`);
+            if (err) terminal.warn(`Index creation warning: ${err.message}`);
           });
         });
       });
@@ -447,7 +448,7 @@ class NdcDataMigrator {
   }
 
   async migrate(sourceDir, destFile = 'unknown', options = {}) {
-    if (options.verbose) console.log('Starting NDC data migration...');
+    if (options.verbose) terminal.log('Starting NDC data migration...');
 
     // Ensure destination directory exists
     const destDir = path.dirname(destFile);
@@ -471,12 +472,12 @@ class NdcDataMigrator {
       const versions = await this.#findVersions(sourceDir);
 
       if (options.verbose) {
-        console.log(`Found ${versions.length} versions: ${versions.join(', ')}`);
+        terminal.log(`Found ${versions.length} versions: ${versions.join(', ')}`);
       }
 
       // Process each version
       for (const versionName of versions) {
-        if (options.verbose) console.log(`Processing version: ${versionName}`);
+        if (options.verbose) terminal.log(`Processing version: ${versionName}`);
 
         await this.#processVersion(db, sourceDir, versionName, options);
       }
@@ -489,7 +490,7 @@ class NdcDataMigrator {
       // Create lookup tables
       await this.#createLookupTables(db, options.verbose);
 
-      if (options.verbose) console.log('NDC data migration completed successfully');
+      if (options.verbose) terminal.log('NDC data migration completed successfully');
     } finally {
       await this.#closeDatabase(db, options.verbose);
     }
@@ -511,7 +512,7 @@ class NdcDataMigrator {
   }
 
   async #createTables(db, verbose = true) {
-    if (verbose) console.log('Creating database tables...');
+    if (verbose) terminal.log('Creating database tables...');
 
     const tableSQL = [
       `CREATE TABLE NDCProducts (
@@ -574,7 +575,7 @@ class NdcDataMigrator {
           });
         });
 
-        if (verbose) console.log('Database tables created');
+        if (verbose) terminal.log('Database tables created');
         resolve();
       });
     });
@@ -599,11 +600,11 @@ class NdcDataMigrator {
     const productFile = path.join(versionDir, 'product.txt');
 
     if (!fs.existsSync(productFile)) {
-      if (options.verbose) console.warn(`Product file not found: ${productFile}`);
+      if (options.verbose) terminal.warn(`Product file not found: ${productFile}`);
       return;
     }
 
-    if (options.verbose) console.log(`Processing products from ${versionName}...`);
+    if (options.verbose) terminal.log(`Processing products from ${versionName}...`);
 
     const rl = readline.createInterface({
       input: fs.createReadStream(productFile),
@@ -668,7 +669,7 @@ class NdcDataMigrator {
         batch = [];
 
         if (processedCount % 10000 === 0 && options.verbose) {
-          console.log(`  Processed ${processedCount} products...`);
+          terminal.log(`  Processed ${processedCount} products...`);
         }
       }
     }
@@ -690,7 +691,7 @@ class NdcDataMigrator {
     this.codesMap = codesMap;
 
     if (options.verbose) {
-      console.log(`  Completed products: ${processedCount} records`);
+      terminal.log(`  Completed products: ${processedCount} records`);
     }
   }
 
@@ -698,11 +699,11 @@ class NdcDataMigrator {
     const packageFile = path.join(versionDir, 'package.txt');
 
     if (!fs.existsSync(packageFile)) {
-      if (options.verbose) console.warn(`Package file not found: ${packageFile}`);
+      if (options.verbose) terminal.warn(`Package file not found: ${packageFile}`);
       return;
     }
 
-    if (options.verbose) console.log(`Processing packages from ${versionName}...`);
+    if (options.verbose) terminal.log(`Processing packages from ${versionName}...`);
 
     const rl = readline.createInterface({
       input: fs.createReadStream(packageFile),
@@ -763,7 +764,7 @@ class NdcDataMigrator {
         batch = [];
 
         if (processedCount % 10000 === 0 && options.verbose) {
-          console.log(`  Processed ${processedCount} packages...`);
+          terminal.log(`  Processed ${processedCount} packages...`);
         }
       }
     }
@@ -778,7 +779,7 @@ class NdcDataMigrator {
     insertPackage.finalize();
 
     if (options.verbose) {
-      console.log(`  Completed packages: ${processedCount} records`);
+      terminal.log(`  Completed packages: ${processedCount} records`);
     }
   }
 
@@ -968,7 +969,7 @@ class NdcDataMigrator {
   }
 
   async #createLookupTables(db, verbose = true) {
-    if (verbose) console.log('Creating lookup tables...');
+    if (verbose) terminal.log('Creating lookup tables...');
 
     const lookups = [
       { table: 'NDCProductTypes', map: this.typesMap },
@@ -977,8 +978,12 @@ class NdcDataMigrator {
       { table: 'NDCRoutes', map: this.routesMap }
     ];
 
-    return new Promise((resolve) => {
+    // One transaction for all of them: otherwise each row commits (and waits on a disk sync)
+    // on its own, and there are thousands of organisations. A row that fails is warned about
+    // and skipped, as before: in SQLite a failed INSERT doesn't end the transaction
+    return new Promise((resolve, reject) => {
       db.serialize(() => {
+        db.run('BEGIN TRANSACTION');
         lookups.forEach(lookup => {
           if (lookup.map) {
             for (const [name, id] of lookup.map) {
@@ -987,7 +992,7 @@ class NdcDataMigrator {
                 [id, name.substring(0, 500)],
                 (err) => {
                   if (err && verbose) {
-                    console.warn(`Warning inserting into ${lookup.table}: ${err.message}`);
+                    terminal.warn(`Warning inserting into ${lookup.table}: ${err.message}`);
                   }
                 }
               );
@@ -995,8 +1000,15 @@ class NdcDataMigrator {
           }
         });
 
-        if (verbose) console.log('Lookup tables created');
-        resolve();
+        // resolves once the rows are in, not when they have only been queued
+        db.run('COMMIT', (err) => {
+          if (err) {
+            reject(err);
+          } else {
+            if (verbose) terminal.log('Lookup tables created');
+            resolve();
+          }
+        });
       });
     });
   }
@@ -1005,7 +1017,7 @@ class NdcDataMigrator {
     return new Promise((resolve) => {
       db.close((err) => {
         if (err && verbose) {
-          console.error('Error closing database:', err);
+          terminal.error('Error closing database:', err);
         }
         resolve();
       });

package/tx/importers/import-rxnorm.module.js

@@ -4,6 +4,7 @@ const sqlite3 = require('sqlite3').verbose();
 const path = require('path');
 const readline = require('readline');
 const chalk = require('chalk');
+const terminal = require('./terminal');
 
 class RxNormModule extends BaseTerminologyModule {
   constructor() {
@@ -98,7 +99,7 @@ class RxNormModule extends BaseTerminologyModule {
     } catch (error) {
       this.logError(`Import command failed: ${error.message}`);
       if (options.verbose) {
-        console.error(error.stack);
+        terminal.error(error.stack);
       }
       throw error;
     }
@@ -121,17 +122,17 @@ class RxNormModule extends BaseTerminologyModule {
   async confirmImport(config) {
     const inquirer = require('inquirer');
 
-    console.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
-    console.log(`  Source: ${chalk.white(config.source)}`);
-    console.log(`  Destination: ${chalk.white(config.dest)}`);
-    console.log(`  Version: ${chalk.white(config.version || 'Auto-detect')}`);
-    console.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
-    console.log(`  Create Stems: ${chalk.white(config.createStems ? 'Yes' : 'No')}`);
-    console.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
-    console.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
+    terminal.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
+    terminal.log(`  Source: ${chalk.white(config.source)}`);
+    terminal.log(`  Destination: ${chalk.white(config.dest)}`);
+    terminal.log(`  Version: ${chalk.white(config.version || 'Auto-detect')}`);
+    terminal.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
+    terminal.log(`  Create Stems: ${chalk.white(config.createStems ? 'Yes' : 'No')}`);
+    terminal.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
+    terminal.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
 
     if (config.estimatedDuration) {
-      console.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
+      terminal.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
     }
 
     const { confirmed } = await inquirer.prompt({
@@ -161,14 +162,14 @@ class RxNormModule extends BaseTerminologyModule {
       const stats = await this.validateRxNormDirectory(options.source);
 
       this.logSuccess('Directory validation passed');
-      console.log(`  Required files found: ${stats.requiredFiles.length}/3`);
-      console.log(`  Optional files found: ${stats.optionalFiles.length}/3`);
-      console.log(`  Estimated concepts: ${stats.estimatedConcepts.toLocaleString()}`);
-      console.log(`  Estimated relationships: ${stats.estimatedRelationships.toLocaleString()}`);
+      terminal.log(`  Required files found: ${stats.requiredFiles.length}/3`);
+      terminal.log(`  Optional files found: ${stats.optionalFiles.length}/3`);
+      terminal.log(`  Estimated concepts: ${stats.estimatedConcepts.toLocaleString()}`);
+      terminal.log(`  Estimated relationships: ${stats.estimatedRelationships.toLocaleString()}`);
 
       if (stats.warnings.length > 0) {
         this.logWarning('Validation warnings:');
-        stats.warnings.forEach(warning => console.log(`    ${warning}`));
+        stats.warnings.forEach(warning => terminal.log(`    ${warning}`));
       }
 
     } catch (error) {
@@ -190,13 +191,13 @@ class RxNormModule extends BaseTerminologyModule {
       const stats = await this.getDatabaseStats(dbPath);
 
       this.logSuccess('Database status:');
-      console.log(`  Version: ${stats.version}`);
-      console.log(`  RXNCONSO Records: ${stats.rxnconsoCount.toLocaleString()}`);
-      console.log(`  RXNREL Records: ${stats.rxnrelCount.toLocaleString()}`);
-      console.log(`  RXNSTY Records: ${stats.rxnstyCount.toLocaleString()}`);
-      console.log(`  Stem Records: ${stats.stemCount.toLocaleString()}`);
-      console.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
-      console.log(`  Last Modified: ${stats.lastModified}`);
+      terminal.log(`  Version: ${stats.version}`);
+      terminal.log(`  RXNCONSO Records: ${stats.rxnconsoCount.toLocaleString()}`);
+      terminal.log(`  RXNREL Records: ${stats.rxnrelCount.toLocaleString()}`);
+      terminal.log(`  RXNSTY Records: ${stats.rxnstyCount.toLocaleString()}`);
+      terminal.log(`  Stem Records: ${stats.stemCount.toLocaleString()}`);
+      terminal.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
+      terminal.log(`  Last Modified: ${stats.lastModified}`);
 
     } catch (error) {
       this.logError(`Status check failed: ${error.message}`);
@@ -428,15 +429,20 @@ class RxNormModule extends BaseTerminologyModule {
       'CREATE INDEX IF NOT EXISTS idx_rxnrel_rel ON RXNREL(REL)',
       'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela ON RXNREL(RELA)',
       'CREATE INDEX IF NOT EXISTS X_RXNSTY_2 ON RXNSTY(TUI)',
+      'CREATE INDEX IF NOT EXISTS idx_rxnsty_rxcui ON RXNSTY(RXCUI)',
+      'CREATE INDEX IF NOT EXISTS idx_rxnrel_rxcui2 ON RXNREL(RXCUI2, SAB)',
+      'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxcui1 ON RXNREL(RELA, RXCUI1)',
+      'CREATE INDEX IF NOT EXISTS idx_rxnrel_rela_rxaui1 ON RXNREL(RELA, RXAUI1)',
       'CREATE INDEX IF NOT EXISTS idx_rxnstems_stem_cui ON RXNSTEMS(stem, CUI)',
-      'CREATE INDEX IF NOT EXISTS idx_rxnstems_cui_stem ON RXNSTEMS(CUI, stem)'
+      'CREATE INDEX IF NOT EXISTS idx_rxnstems_cui_stem ON RXNSTEMS(CUI, stem)',
+      'CREATE INDEX IF NOT EXISTS idx_rxnatomarchive_rxcui_sab ON RXNATOMARCHIVE(RXCUI, SAB)'
     ];
 
     return new Promise((resolve, reject) => {
       db.serialize(() => {
         indexes.forEach(sql => {
           db.run(sql, (err) => {
-            if (err) console.warn(`Index creation warning: ${err.message}`);
+            if (err) terminal.warn(`Index creation warning: ${err.message}`);
           });
         });
       });
@@ -472,7 +478,7 @@ class RxNormImporter {
   }
 
   async import() {
-    if (this.options.verbose) console.log('Starting RxNorm import...');
+    if (this.options.verbose) terminal.log('Starting RxNorm import...');
 
     // Ensure destination directory exists
     const destDir = path.dirname(this.destFile);
@@ -501,7 +507,7 @@ class RxNormImporter {
         await this.makeStems(db);
       }
 
-      if (this.options.verbose) console.log('RxNorm import completed successfully');
+      if (this.options.verbose) terminal.log('RxNorm import completed successfully');
     } finally {
       await this.closeDatabase(db);
     }
@@ -634,12 +640,12 @@ class RxNormImporter {
         tableSQL.forEach(sql => {
           db.run(sql, (err) => {
             if (err && this.options.verbose) {
-              console.warn(`Table creation warning: ${err.message}`);
+              terminal.warn(`Table creation warning: ${err.message}`);
             }
           });
         });
 
-        if (this.options.verbose) console.log('Database tables created');
+        if (this.options.verbose) terminal.log('Database tables created');
         resolve();
       });
     });
@@ -701,83 +707,116 @@ class RxNormImporter {
   async loadRRFFile(db, fileName, insertSQL, extractValues) {
     const filePath = path.join(this.sourceDir, fileName);
     if (!fs.existsSync(filePath)) {
-      if (this.options.verbose) console.warn(`File not found: ${fileName}`);
+      if (this.options.verbose) terminal.warn(`File not found: ${fileName}`);
       return;
     }
 
     this.updateProgress(0, `Loading ${fileName}`);
 
+    // Rows go in as multi-row INSERTs, as many rows per statement as fit in SQLite's 999
+    // parameters: one statement (and one callback) per row made the import slow, and slower
+    // still when the process is busy collecting a large heap - the test suite runs it in a
+    // jest process already holding GBs. OR IGNORE keeps the old per-row behaviour, where a row
+    // that broke a constraint was dropped and the rest of the file went in
+    const m = insertSQL.match(/^\s*INSERT\s+(?:OR\s+IGNORE\s+)?INTO([\s\S]*?VALUES\s*)(\([^)]*\))\s*$/i);
+    if (!m) {
+      throw new Error(`Unexpected insert statement for ${fileName}: ${insertSQL}`);
+    }
+    const head = 'INSERT OR IGNORE INTO' + m[1];
+    const group = m[2];
+    const cols = (group.match(/\?/g) || []).length;
+    const batchRows = Math.max(1, Math.floor(999 / cols));
+    const sqlFor = (n) => head + new Array(n).fill(group).join(', ');
+    const fullSql = sqlFor(batchRows);
+
     const rl = readline.createInterface({
       input: fs.createReadStream(filePath),
       crlfDelay: Infinity
     });
 
     return new Promise((resolve, reject) => {
-      db.serialize(() => {
-        db.run('BEGIN TRANSACTION');
-
-        const stmt = db.prepare(insertSQL);
-        let lineCount = 0;
-        let processedCount = 0;
-        let errorCount = 0;
-
-        rl.on('line', (line) => {
-          lineCount++;
-
-          const items = line.split('|');
-          if (items.length === 0) return;
-
-          try {
-            const values = extractValues(items);
-            stmt.run(values, (err) => {
-              if (err && err.code !== 'SQLITE_CONSTRAINT') {
-                errorCount++;
-                if (this.options.verbose && errorCount <= 10) {
-                  console.warn(`Error processing line ${lineCount} in ${fileName}: ${err.message}`);
-                }
-              }
-            });
-            processedCount++;
+      // serialized: BEGIN, every batch, and COMMIT run in the order they are queued
+      db.serialize();
+      db.run('BEGIN TRANSACTION');
 
-            if (processedCount % 1000 === 0) {
-              this.updateProgress(1000);
-            }
-          } catch (error) {
+      let lineCount = 0;
+      let processedCount = 0;
+      let errorCount = 0;
+      let pending = [];
+      let pendingRows = 0;
+
+      const flush = () => {
+        if (pendingRows === 0) {
+          return;
+        }
+        const rows = pendingRows;
+        db.run(rows === batchRows ? fullSql : sqlFor(rows), pending, (err) => {
+          if (err) {
             errorCount++;
             if (this.options.verbose && errorCount <= 10) {
-              console.warn(`Error processing line ${lineCount} in ${fileName}: ${error.message}`);
+              terminal.warn(`Error loading ${rows} rows of ${fileName}: ${err.message}`);
             }
           }
         });
+        pending = [];
+        pendingRows = 0;
+      };
 
-        rl.on('close', () => {
-          stmt.finalize();
-          db.run('COMMIT', (err) => {
-            if (err) reject(err);
-            else {
-              // Update progress for remaining records
-              const remaining = processedCount % 1000;
-              if (remaining > 0) {
-                this.updateProgress(remaining);
-              }
+      rl.on('line', (line) => {
+        lineCount++;
 
-              if (this.options.verbose) {
-                console.log(`  Loaded ${processedCount} records from ${fileName}${errorCount > 0 ? ` (${errorCount} errors)` : ''}`);
-              }
-              resolve();
+        const items = line.split('|');
+        if (items.length === 0) return;
+
+        try {
+          const values = extractValues(items);
+          for (let i = 0; i < cols; i++) {
+            pending.push(values[i] === undefined ? null : values[i]);
+          }
+          pendingRows++;
+          if (pendingRows === batchRows) {
+            flush();
+          }
+          processedCount++;
+
+          if (processedCount % 1000 === 0) {
+            this.updateProgress(1000);
+          }
+        } catch (error) {
+          errorCount++;
+          if (this.options.verbose && errorCount <= 10) {
+            terminal.warn(`Error processing line ${lineCount} in ${fileName}: ${error.message}`);
+          }
+        }
+      });
+
+      rl.on('close', () => {
+        flush();
+        db.run('COMMIT', (err) => {
+          if (err) reject(err);
+          else {
+            // Update progress for remaining records
+            const remaining = processedCount % 1000;
+            if (remaining > 0) {
+              this.updateProgress(remaining);
             }
-          });
-        });
 
-        rl.on('error', reject);
+            if (this.options.verbose) {
+              terminal.log(`  Loaded ${processedCount} records from ${fileName}${errorCount > 0 ? ` (${errorCount} errors)` : ''}`);
+            }
+            resolve();
+          }
+        });
       });
+
+      rl.on('error', reject);
     });
   }
 
   async makeStems(db) {
     this.updateProgress(0, 'Generating Stems');
 
-    if (this.options.verbose) console.log('Generating word stems...');
+    if (this.options.verbose) terminal.log('Generating word stems...');
 
     // Simple English stemmer implementation
     var natural = require('natural/lib/natural/stemmers'); // just the stemmers: the package index also loads its storage backends, and dotenv with them
@@ -833,7 +872,7 @@ class RxNormImporter {
               }
 
               if (this.options.verbose) {
-                console.log(`  Generated ${totalInserts} stem entries from ${processedStems} unique stems`);
+                terminal.log(`  Generated ${totalInserts} stem entries from ${processedStems} unique stems`);
               }
               resolve();
             }
@@ -857,7 +896,7 @@ class RxNormImporter {
     return new Promise((resolve) => {
       db.close((err) => {
         if (err && this.options.verbose) {
-          console.error('Error closing database:', err);
+          terminal.error('Error closing database:', err);
         }
         resolve();
       });