fhirsmith 0.18.2 → 1.0.0

package/tx/importers/import-sct.module.js

@@ -12,6 +12,7 @@ const {
   SnomedRelationshipList, SnomedReferenceSetMembers, SnomedReferenceSetIndex
 } = require('../sct/structures');
 const {SnomedExpressionServices} = require("../sct/expressions");
+const terminal = require('./terminal');
 
 // definitionStatusId values from the RF2 concept file
 const SNOMED_DEFINITION_STATUS_PRIMITIVE = 900000000000074008n;
@@ -106,16 +107,16 @@ class SnomedModule extends BaseTerminologyModule {
     } catch (error) {
       this.logError(`Import command failed: ${error.message}`);
       if (options.verbose) {
-        console.error(error.stack);
+        terminal.error(error.stack);
       }
       throw error;
     }
   }
 
   async confirmImport(config) {
-    console.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
-    console.log(`  Source: ${chalk.white(config.source)}`);
-    console.log(`  Destination: ${chalk.white(config.dest)}`);
+    terminal.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
+    terminal.log(`  Source: ${chalk.white(config.source)}`);
+    terminal.log(`  Destination: ${chalk.white(config.dest)}`);
 
     if (config.edition) {
       const editions = {
@@ -125,26 +126,26 @@ class SnomedModule extends BaseTerminologyModule {
         // ... other editions
       };
       const editionName = editions[config.edition] || `Edition ${config.edition}`;
-      console.log(`  Edition: ${chalk.white(editionName)} (${config.edition})`);
+      terminal.log(`  Edition: ${chalk.white(editionName)} (${config.edition})`);
     }
 
     if (config.version) {
-      console.log(`  Version: ${chalk.white(config.version)}`);
+      terminal.log(`  Version: ${chalk.white(config.version)}`);
     }
 
     if (config.uri) {
-      console.log(`  Version URI: ${chalk.white(config.uri)}`);
+      terminal.log(`  Version URI: ${chalk.white(config.uri)}`);
     }
 
     if (config.language) {
-      console.log(`  Language: ${chalk.white(config.language)}`);
+      terminal.log(`  Language: ${chalk.white(config.language)}`);
     }
 
-    console.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
-    console.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
+    terminal.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
+    terminal.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
 
     if (config.estimatedDuration) {
-      console.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
+      terminal.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
     }
 
     const { confirmed } = await inquirer.prompt({
@@ -357,7 +358,7 @@ class SnomedModule extends BaseTerminologyModule {
     } catch (error) {
       // Silent fail - auto-detection is best-effort
       if (this.config?.verbose) {
-        console.log(`Auto-detection warning: ${error.message}`);
+        terminal.log(`Auto-detection warning: ${error.message}`);
       }
     }
 
@@ -661,7 +662,7 @@ class SnomedModule extends BaseTerminologyModule {
 
   async runImportWithoutConfigSaving(config) {
     try {
-      console.log(chalk.blue.bold(`🏥 Starting ${this.getName()} Import...\n`));
+      terminal.log(chalk.blue.bold(`🏥 Starting ${this.getName()} Import...\n`));
 
       // Pre-flight checks
       this.logInfo('Running pre-flight checks...');
@@ -680,7 +681,7 @@ class SnomedModule extends BaseTerminologyModule {
       this.stopProgress();
       this.logError(`${this.getName()} import failed: ${error.message}`);
       if (config.verbose) {
-        console.error(error.stack);
+        terminal.error(error.stack);
       }
       process.exit(1);
     }
@@ -816,10 +817,10 @@ class SnomedModule extends BaseTerminologyModule {
       const stats = await this.validateSnomedDirectory(options.source);
 
       this.logSuccess('Directory validation passed');
-      console.log(`  Concept files: ${stats.conceptFiles}`);
-      console.log(`  Description files: ${stats.descriptionFiles}`);
-      console.log(`  Relationship files: ${stats.relationshipFiles}`);
-      console.log(`  Refset directories: ${stats.refsetDirectories}`);
+      terminal.log(`  Concept files: ${stats.conceptFiles}`);
+      terminal.log(`  Description files: ${stats.descriptionFiles}`);
+      terminal.log(`  Relationship files: ${stats.relationshipFiles}`);
+      terminal.log(`  Refset directories: ${stats.refsetDirectories}`);
 
     } catch (error) {
       this.logError(`Validation failed: ${error.message}`);
@@ -843,26 +844,26 @@ class SnomedModule extends BaseTerminologyModule {
       const data = await reader.loadSnomedData();
 
       this.logSuccess('Cache file status:');
-      console.log(`  Cache Version: ${data.cacheVersion}`);
-      console.log(`  Version URI: ${data.versionUri}`);
-      console.log(`  Version Date: ${data.versionDate}`);
-      console.log(`  Edition: ${data.edition}`);
-      console.log(`  SNOMED Version: ${data.version}`);
+      terminal.log(`  Cache Version: ${data.cacheVersion}`);
+      terminal.log(`  Version URI: ${data.versionUri}`);
+      terminal.log(`  Version Date: ${data.versionDate}`);
+      terminal.log(`  Edition: ${data.edition}`);
+      terminal.log(`  SNOMED Version: ${data.version}`);
 
       // Create structure instances to get counts
       const concepts = new SnomedConceptList(data.concept);
       const descriptions = new SnomedDescriptions(data.desc);
       const relationships = new SnomedRelationshipList(data.rel);
 
-      console.log(`  Concepts: ${concepts.count().toLocaleString()}`);
-      console.log(`  Descriptions: ${descriptions.count().toLocaleString()}`);
-      console.log(`  Relationships: ${relationships.count().toLocaleString()}`);
-      console.log(`  Active Roots: ${data.activeRoots.length}`);
-      console.log(`  Inactive Roots: ${data.inactiveRoots.length}`);
+      terminal.log(`  Concepts: ${concepts.count().toLocaleString()}`);
+      terminal.log(`  Descriptions: ${descriptions.count().toLocaleString()}`);
+      terminal.log(`  Relationships: ${relationships.count().toLocaleString()}`);
+      terminal.log(`  Active Roots: ${data.activeRoots.length}`);
+      terminal.log(`  Inactive Roots: ${data.inactiveRoots.length}`);
 
       const fileStat = fs.statSync(cachePath);
-      console.log(`  File Size: ${(fileStat.size / (1024 * 1024 * 1024)).toFixed(2)} GB`);
-      console.log(`  Last Modified: ${fileStat.mtime.toISOString()}`);
+      terminal.log(`  File Size: ${(fileStat.size / (1024 * 1024 * 1024)).toFixed(2)} GB`);
+      terminal.log(`  Last Modified: ${fileStat.mtime.toISOString()}`);
 
     } catch (error) {
       this.logError(`Status check failed: ${error.message}`);
@@ -924,7 +925,7 @@ class SnomedImporterWithProgress {
         message += ` (${rate.toLocaleString()} items/sec)`;
       }
 
-      console.log(message);
+      terminal.log(message);
 
       // Clean up
       this.taskStartTimes.delete(taskName);
@@ -1161,7 +1162,7 @@ class SnomedImporter {
       await this.saveCache();
 
     } catch (error) {
-      console.error('DEBUG: Import failed with error:', error);
+      terminal.error('DEBUG: Import failed with error:', error);
       throw new Error(`Import failed: ${error.message}`);
     }
   }
@@ -1181,7 +1182,7 @@ class SnomedImporter {
     // extension content is layered on top.
     if (this.config.base) {
       if (this.config.verbose) {
-        console.log(`Loading base edition from: ${this.config.base}`);
+        terminal.log(`Loading base edition from: ${this.config.base}`);
       }
       this._scanDirectory(this.config.base, files);
     }
@@ -1232,7 +1233,7 @@ class SnomedImporter {
         files.concreteValues.push(filePath);
       }
     } catch (error) {
-      console.log(`DEBUG: Error reading file ${filePath}: ${error.message}`);
+      terminal.log(`DEBUG: Error reading file ${filePath}: ${error.message}`);
     }
   }
 
@@ -1528,14 +1529,15 @@ class SnomedImporter {
     const month = parseInt(dateStr.substring(4, 6));
     const day = parseInt(dateStr.substring(6, 8));
 
-    // Create target date
-    const targetDate = new Date(year, month - 1, day);
-
-    // Pascal TDateTime epoch: December 30, 1899
-    const pascalEpoch = new Date(1899, 11, 30); // Month is 0-based in JS
+    // Days since the Pascal TDateTime epoch (December 30, 1899), counted in UTC. Local dates are
+    // not whole days apart: 1899 local time is the zone's historical mean time (e.g. +9:39:52 in
+    // Melbourne) and today's has daylight saving, so the difference came out a day short in zones
+    // ahead of UTC, and every effectiveTime in a cache built there was a day early
+    const targetDate = Date.UTC(year, month - 1, day);
+    const pascalEpoch = Date.UTC(1899, 11, 30); // Month is 0-based in JS
 
     // Calculate days difference
-    const daysDiff = Math.floor((targetDate - pascalEpoch) / (1000 * 60 * 60 * 24));
+    const daysDiff = Math.round((targetDate - pascalEpoch) / (1000 * 60 * 60 * 24));
 
     // Ensure it fits in 16 bits (0-65535) and is positive
     if (daysDiff < 0 || daysDiff > 65535) {
@@ -1784,7 +1786,7 @@ class SnomedImporter {
     }
     this.identifiers.sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1));
     if (this.identifiers.length > 0 || missing > 0) {
-      console.log(`Alternate identifiers: ${this.identifiers.length}${missing ? ` (${missing} for concepts not in this edition, ignored)` : ''}`);
+      terminal.log(`Alternate identifiers: ${this.identifiers.length}${missing ? ` (${missing} for concepts not in this edition, ignored)` : ''}`);
     }
   }
 
@@ -1836,7 +1838,7 @@ class SnomedImporter {
     }
     this.concreteValues.sort((a, b) => (a.join('|') < b.join('|') ? -1 : 1));
     if (this.concreteValues.length > 0 || missing > 0) {
-      console.log(`Concrete values: ${this.concreteValues.length}${missing ? ` (${missing} for concepts not in this edition, ignored)` : ''}`);
+      terminal.log(`Concrete values: ${this.concreteValues.length}${missing ? ` (${missing} for concepts not in this edition, ignored)` : ''}`);
     }
   }
 
@@ -2251,7 +2253,7 @@ class SnomedImporter {
 
   async processRefsets() {
     if (this.files.refsetDirectories.length === 0) {
-      console.log('No reference set directories found');
+      terminal.log('No reference set directories found');
       this.refsetIndex.doneBuild();
       this.refsetMembers.doneBuild();
       return;
@@ -2367,7 +2369,7 @@ class SnomedImporter {
       const parts = fileName.split('_');
 
       if (parts.length < 3) {
-        console.log(`Skipping file with unexpected name format: ${fileName}`);
+        terminal.log(`Skipping file with unexpected name format: ${fileName}`);
         return;
       }
 
@@ -2391,19 +2393,16 @@ class SnomedImporter {
         lineNumber++;
 
         if (lineNumber === 1) {
-          // Parse headers
+          // Parse headers. Map reference sets (simple, complex, extended) are loaded like
+          // any other: ECL ^ and ?fhir_vs=refset/ can name any reference set, and the
+          // ICD-10 maps are among the most used.
           headers = line.split('\t').map(h => h.trim());
-
-          // Skip if this looks like a mapping reference set
-          if (line.toLowerCase().includes('map')) {
-            return;
-          }
           continue;
         }
 
         const fields = line.split('\t');
         if (fields.length < 6) {
-          console.log(`Skipping line ${lineNumber} with insufficient fields: ${fields.length}`);
+          terminal.log(`Skipping line ${lineNumber} with insufficient fields: ${fields.length}`);
           continue; // Minimum fields required
         }
 
@@ -2439,14 +2438,14 @@ class SnomedImporter {
         try {
           member.id = this.parseGUID(id);
         } catch (error) {
-          console.log(`Invalid GUID in ${fileName}: ${id}`);
+          terminal.log(`Invalid GUID in ${fileName}: ${id}`);
           continue;
         }
 
         // Find module concept
         const moduleConcept = this.conceptMap.get(BigInt(moduleId));
         if (!moduleConcept) {
-          console.log(`Module concept not found: ${moduleId}`);
+          terminal.log(`Module concept not found: ${moduleId}`);
           continue;
         }
         member.module = moduleConcept.index;
@@ -2479,7 +2478,7 @@ class SnomedImporter {
             // Try to find as relationship (simplified - would need relationship ID lookup)
             member.kind = 3; // Other/unknown
             member.ref = 0;
-            console.log(`Component not found: ${referencedComponentId}`);
+            terminal.log(`Component not found: ${referencedComponentId}`);
             continue;
           }
         }
@@ -2502,7 +2501,7 @@ class SnomedImporter {
       }
 
     } catch (error) {
-      console.error(`Error processing reference set ${filePath}:`, error);
+      terminal.error(`Error processing reference set ${filePath}:`, error);
     }
   }
 
@@ -2590,7 +2589,7 @@ class SnomedImporter {
               value = descResult.index;
               type = 2; // Description
             } else {
-              console.log(`Referenced component not found: ${field}`);
+              terminal.log(`Referenced component not found: ${field}`);
               value = 0;
               type = 1;
             }
@@ -2847,7 +2846,7 @@ class SnomedImporter {
       } catch (error) {
         // Log the error but continue processing other concepts
         if (this.config.verbose) {
-          console.warn(`Warning: Could not build normal form for concept ${concept.id}: ${error.message}`);
+          terminal.warn(`Warning: Could not build normal form for concept ${concept.id}: ${error.message}`);
         }
       }
 
@@ -2862,7 +2861,7 @@ class SnomedImporter {
     }
 
     if (this.config.verbose) {
-      console.log(`Normal forms: ${normalFormsAdded} concepts have non-trivial normal forms`);
+      terminal.log(`Normal forms: ${normalFormsAdded} concepts have non-trivial normal forms`);
     }
   }
 

package/tx/importers/import-unii.module.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const sqlite3 = require('sqlite3').verbose();
 const path = require("path");
 const readline = require("readline");
+const terminal = require('./terminal');
 
 class UniiModule extends BaseTerminologyModule {
   getName() {
@@ -81,16 +82,16 @@ class UniiModule extends BaseTerminologyModule {
     const inquirer = require('inquirer');
     const chalk = require('chalk');
 
-    console.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
-    console.log(`  Source: ${chalk.white(config.source)}`);
-    console.log(`  Destination: ${chalk.white(config.dest)}`);
-    console.log(`  Version: ${chalk.white(config.version)}`);
-    console.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
-    console.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
-    console.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
+    terminal.log(chalk.cyan(`\n📋 ${this.getName()} Import Configuration:`));
+    terminal.log(`  Source: ${chalk.white(config.source)}`);
+    terminal.log(`  Destination: ${chalk.white(config.dest)}`);
+    terminal.log(`  Version: ${chalk.white(config.version)}`);
+    terminal.log(`  Create Indexes: ${chalk.white(config.createIndexes ? 'Yes' : 'No')}`);
+    terminal.log(`  Overwrite: ${chalk.white(config.overwrite ? 'Yes' : 'No')}`);
+    terminal.log(`  Verbose: ${chalk.white(config.verbose ? 'Yes' : 'No')}`);
 
     if (config.estimatedDuration) {
-      console.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
+      terminal.log(`  Estimated Duration: ${chalk.white(config.estimatedDuration)}`);
     }
 
     const { confirmed } = await inquirer.prompt({
@@ -120,14 +121,14 @@ class UniiModule extends BaseTerminologyModule {
       const stats = await this.validateUniiFile(options.source, parseInt(options.sample));
       
       this.logSuccess('File validation passed');
-      console.log(`  Lines: ${stats.totalLines}`);
-      console.log(`  Estimated UNII codes: ${stats.estimatedCodes}`);
-      console.log(`  Estimated descriptions: ${stats.estimatedDescriptions}`);
-      console.log(`  Sample data format: ${stats.formatValid ? 'Valid' : 'Invalid'}`);
+      terminal.log(`  Lines: ${stats.totalLines}`);
+      terminal.log(`  Estimated UNII codes: ${stats.estimatedCodes}`);
+      terminal.log(`  Estimated descriptions: ${stats.estimatedDescriptions}`);
+      terminal.log(`  Sample data format: ${stats.formatValid ? 'Valid' : 'Invalid'}`);
       
       if (stats.warnings.length > 0) {
         this.logWarning('Validation warnings:');
-        stats.warnings.forEach(warning => console.log(`    ${warning}`));
+        stats.warnings.forEach(warning => terminal.log(`    ${warning}`));
       }
       
     } catch (error) {
@@ -149,11 +150,11 @@ class UniiModule extends BaseTerminologyModule {
       const stats = await this.getDatabaseStats(dbPath);
       
       this.logSuccess('Database status:');
-      console.log(`  Version: ${stats.version}`);
-      console.log(`  UNII Codes: ${stats.uniiCount.toLocaleString()}`);
-      console.log(`  Descriptions: ${stats.descCount.toLocaleString()}`);
-      console.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
-      console.log(`  Last Modified: ${stats.lastModified}`);
+      terminal.log(`  Version: ${stats.version}`);
+      terminal.log(`  UNII Codes: ${stats.uniiCount.toLocaleString()}`);
+      terminal.log(`  Descriptions: ${stats.descCount.toLocaleString()}`);
+      terminal.log(`  Database Size: ${stats.sizeGB.toFixed(2)} GB`);
+      terminal.log(`  Last Modified: ${stats.lastModified}`);
       
     } catch (error) {
       this.logError(`Status check failed: ${error.message}`);
@@ -304,7 +305,7 @@ class UniiModule extends BaseTerminologyModule {
       db.serialize(() => {
         indexes.forEach(sql => {
           db.run(sql, (err) => {
-            if (err) console.warn(`Index creation warning: ${err.message}`);
+            if (err) terminal.warn(`Index creation warning: ${err.message}`);
           });
         });
       });
@@ -327,7 +328,7 @@ class UniiDataMigrator {
    * @returns {Promise<void>}
    */
   async migrate(sourceFile, destFile, version = 'unknown', verbose = true) {
-    if (verbose) console.log('Starting UNII data migration...');
+    if (verbose) terminal.log('Starting UNII data migration...');
 
     // Ensure destination directory exists
     const destDir = path.dirname(destFile);
@@ -350,7 +351,7 @@ class UniiDataMigrator {
       // Process source file
       await this.#processSourceFile(db, sourceFile, verbose);
 
-      if (verbose) console.log('UNII data migration completed successfully');
+      if (verbose) terminal.log('UNII data migration completed successfully');
     } finally {
       await this.#closeDatabase(db, verbose);
     }
@@ -361,7 +362,7 @@ class UniiDataMigrator {
    * @private
    */
   async #createTables(db, version, verbose = true) {
-    if (verbose) console.log('Creating database tables...');
+    if (verbose) terminal.log('Creating database tables...');
 
     return new Promise((resolve, reject) => {
       db.serialize(() => {
@@ -400,7 +401,7 @@ class UniiDataMigrator {
         // Insert version
         db.run('INSERT INTO UniiVersion (Version) VALUES (?)', [version], (err) => {
           if (err) return reject(err);
-          if (verbose) console.log('Database tables created');
+          if (verbose) terminal.log('Database tables created');
           resolve();
         });
       });
@@ -412,7 +413,7 @@ class UniiDataMigrator {
    * @private
    */
   async #processSourceFile(db, sourceFile, verbose = true) {
-    if (verbose) console.log('Processing source file:', sourceFile);
+    if (verbose) terminal.log('Processing source file:', sourceFile);
 
     if (!fs.existsSync(sourceFile)) {
       throw new Error(`Source file not found: ${sourceFile}`);
@@ -438,7 +439,7 @@ class UniiDataMigrator {
       throw new Error('Source file is empty');
     }
 
-    if (verbose) console.log(`Read ${lines.length} lines, processing...`);
+    if (verbose) terminal.log(`Read ${lines.length} lines, processing...`);
 
     // Track processed codes and auto-increment keys
     const codeMap = new Map(); // code -> UniiKey
@@ -467,7 +468,7 @@ class UniiDataMigrator {
 
             // Need at least 3 columns (Display, Type, Code)
             if (cols.length < 3) {
-              if (verbose) console.warn(`Skipping line ${i + 1}: insufficient columns`);
+              if (verbose) terminal.warn(`Skipping line ${i + 1}: insufficient columns`);
               continue;
             }
 
@@ -476,7 +477,7 @@ class UniiDataMigrator {
             const code = cols[2] || '';
 
             if (!code) {
-              if (verbose) console.warn(`Skipping line ${i + 1}: empty UNII code`);
+              if (verbose) terminal.warn(`Skipping line ${i + 1}: empty UNII code`);
               continue;
             }
 
@@ -510,14 +511,14 @@ class UniiDataMigrator {
       });
 
       if (processedLines % 10000 === 0) {
-        console.log(`Processed ${processedLines} lines...`);
+        terminal.log(`Processed ${processedLines} lines...`);
       }
     }
 
     if (verbose) {
-      console.log(`Processing completed. Total lines processed: ${processedLines}`);
-      console.log(`Unique UNII codes: ${codeMap.size}`);
-      console.log(`Total descriptions: ${lastUniiDescKey}`);
+      terminal.log(`Processing completed. Total lines processed: ${processedLines}`);
+      terminal.log(`Unique UNII codes: ${codeMap.size}`);
+      terminal.log(`Total descriptions: ${lastUniiDescKey}`);
     }
   }
 
@@ -529,7 +530,7 @@ class UniiDataMigrator {
     return new Promise((resolve) => {
       db.close((err) => {
         if (err && verbose) {
-          console.error('Error closing database:', err);
+          terminal.error('Error closing database:', err);
         }
         resolve();
       });
@@ -558,8 +559,8 @@ class UniiDataMigratorWithProgress {
     let processedLines = 0;
     
     // Override the progress reporting in the original migrator
-    const originalConsoleLog = console.log;
-    console.log = (...args) => {
+    const originalLog = terminal.log;
+    terminal.log = (...args) => {
       const message = args.join(' ');
       if (message.includes('Processed') && message.includes('lines')) {
         const match = message.match(/Processed (\d+) lines/);
@@ -568,13 +569,13 @@ class UniiDataMigratorWithProgress {
           this.module.updateProgress(processedLines);
         }
       }
-      if (verbose) originalConsoleLog(...args);
+      if (verbose) originalLog(...args);
     };
 
     try {
       await originalMigrator.migrate(sourceFile, destFile, version, verbose);
     } finally {
-      console.log = originalConsoleLog;
+      terminal.log = originalLog;
       this.module.stopProgress();
     }
   }