dicom-curate 0.4.1 → 0.5.0

package/dist/esm/curateOne.js

@@ -31,8 +31,20 @@ export function curateOne(_a) {
             dicomData = dcmjs.data.DicomMessage.readFile(fileArrayBuffer);
         }
         catch (error) {
+            console.warn(`[dicom-curate] Could not parse ${fileInfo.name} as DICOM data:`, error);
+            // Create a more informative error result
             const mapResults = {
-                anomalies: [`Could not parse ${fileInfo.name} as dicom data`],
+                anomalies: [`Could not parse ${fileInfo.name} as DICOM data`],
+                errors: [`File ${fileInfo.name} is not a valid DICOM file or is corrupted`],
+                sourceInstanceUID: `invalid_${fileInfo.name.replace(/[^a-zA-Z0-9]/g, '_')}`,
+                outputFilePath: `${fileInfo.path}/${fileInfo.name}`,
+                // Add metadata about the failed file
+                fileInfo: {
+                    name: fileInfo.name,
+                    size: fileInfo.size,
+                    path: fileInfo.path,
+                    parseError: error instanceof Error ? error.message : String(error)
+                }
             };
             return mapResults;
         }

package/dist/esm/index.js

@@ -27,8 +27,11 @@ export { specVersion } from './config/specVersion.js';
 export { sample2PassCurationSpecification as sampleSpecification } from './config/sample2PassCurationSpecification.js';
 export { csvTextToRows } from './csvMapping.js';
 const mappingWorkerCount = navigator.hardwareConcurrency;
+// Update the type to include scan anomalies
 let filesToProcess = [];
 let directoryScanFinished = false;
+// Track scan anomalies separately since they don't go through the processing pipeline
+let scanAnomalies = [];
 function requiresDateOffset(deIdOpts) {
     return (deIdOpts !== 'Off' &&
         deIdOpts.retainLongitudinalTemporalInformationOptions === 'Offset');
@@ -47,24 +50,34 @@ function requiresDateOffset(deIdOpts) {
 function initializeFileListWorker() {
     filesToProcess = [];
     directoryScanFinished = false;
+    scanAnomalies = [];
     const fileListWorker = new Worker(new URL('./scanDirectoryWorker.js', import.meta.url), { type: 'module' });
     fileListWorker.addEventListener('message', (event) => {
         switch (event.data.response) {
             case 'file':
                 const { fileIndex, fileInfo } = event.data;
-                filesToProcess.push({ fileIndex, fileInfo });
+                filesToProcess.push({
+                    fileIndex,
+                    fileInfo,
+                    scanAnomalies: [], // Files sent to processing have no scan anomalies
+                });
                 // Could do some throttling:
                 // if (filesToProcess.length > 10) {
                 //   fileListWorker.postMessage({ request: 'stop' })
                 // }
                 dispatchMappingJobs();
                 break;
+            case 'scanAnomalies':
+                // Handle scan anomalies separately - they don't go to processing
+                const { fileInfo: anomalyFileInfo, anomalies } = event.data;
+                scanAnomalies.push({ fileInfo: anomalyFileInfo, anomalies });
+                break;
             case 'done':
                 console.log('directoryScanFinished');
                 directoryScanFinished = true;
                 break;
             default:
-                // @ts-expect-error: response is string here, not never
+                // @ts-expect-error: response is string here, not never
                 console.error(`Unknown response from worker ${event.data.response}`);
         }
         dispatchMappingJobs();
@@ -143,6 +156,22 @@ function dispatchMappingJobs() {
         clearCaches();
         console.log(`Finished mapping ${mapResultsList.length} files`);
         console.log('job is finished');
+        // Create individual mapResults entries for each scan anomaly
+        // Only do this during actual processing (not first pass)
+        if (!mappingWorkerOptions.skipWrite) {
+            scanAnomalies.forEach(({ fileInfo, anomalies }) => {
+                const scanAnomalyResult = {
+                    sourceInstanceUID: `scan_${fileInfo.name.replace(/[^a-zA-Z0-9]/g, '_')}`,
+                    outputFilePath: `${fileInfo.path}/${fileInfo.name}`, // Use the actual file path
+                    mappings: {},
+                    anomalies: anomalies, // Keep the original anomalies array
+                    errors: [],
+                    quarantine: {},
+                };
+                // Add each scan anomaly result to the final results
+                mapResultsList.push(scanAnomalyResult);
+            });
+        }
         progressCallback({
             response: 'done',
             mapResultsList: mapResultsList,
@@ -197,9 +226,14 @@ function queueFilesForMapping(organizeOptions) {
             kind: 'blob',
             blob: inputFile,
         };
-        filesToProcess.push({ fileInfo, fileIndex });
-        dispatchMappingJobs();
+        filesToProcess.push({
+            fileInfo,
+            fileIndex,
+            scanAnomalies: [],
+        });
     });
+    // Dispatch jobs once after all files are queued to prevent race conditions
+    dispatchMappingJobs();
 }
 let progressCallback;
 function curateMany(organizeOptions, onProgress) {
@@ -225,9 +259,12 @@ function curateMany(organizeOptions, onProgress) {
                 //
                 if (organizeOptions.inputType === 'directory') {
                     const fileListWorker = initializeFileListWorker();
+                    const curationSpec = organizeOptions.curationSpec();
+                    const specExcludedFiletypes = curationSpec.excludedFiletypes;
                     fileListWorker.postMessage({
                         request: 'scan',
                         directoryHandle: organizeOptions.inputDirectory,
+                        excludedFiletypes: specExcludedFiletypes,
                     });
                 }
                 else if (organizeOptions.inputType === 'files') {

package/dist/esm/scanDirectoryWorker.js

@@ -14,11 +14,59 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
     function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
     function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
 };
+// Case-insensitive filetypes to ALWAYS exclude from processing
+const DEFAULT_EXCLUDED_FILETYPES = ['dicomdir', 'dicomdir.dir', 'dicomdir.dat', 'dicomdir.bak', 'thumbs.db', '.ds_store'];
 let keepScanning = true;
+let excludedFiletypes = [];
+/**
+ * Check if a file should be processed based on filtering rules
+ * @param file - The file to check
+ * @param fileAnomalies - Array to collect anomalies for this specific file e.g. excluded files
+ * @returns Promise<boolean> - True if the file should be processed
+ */
+function shouldProcessFile(file, fileAnomalies) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const allExcludedFiletypes = [
+            ...DEFAULT_EXCLUDED_FILETYPES,
+            ...excludedFiletypes
+        ];
+        try {
+            // Check if the file is in the list of excluded files
+            if (allExcludedFiletypes.some(excluded => file.name.toLowerCase() === excluded.toLowerCase())) {
+                fileAnomalies.push(`Skipped excluded file: ${file.name}`);
+                return false;
+            }
+            // Check filesize - (valid) DICOM files are at least 132 bytes (128-byte preamble + 4-byte signature)
+            if (file.size < 132) {
+                fileAnomalies.push(`Skipped very small file: ${file.name} (${file.size} bytes)`);
+                return false;
+            }
+            // Check for DICOM signature "DICM" at offset 128
+            const headerBytes = yield file.slice(128, 132).arrayBuffer();
+            const headerView = new Uint8Array(headerBytes);
+            const dicomSignature = String.fromCharCode(headerView[0], headerView[1], headerView[2], headerView[3]);
+            if (dicomSignature === 'DICM') {
+                return true;
+            }
+            // Don't parse file without DICOM signature
+            fileAnomalies.push(`Skipped file without DICOM signature: ${file.name}`);
+            return false;
+        }
+        catch (error) {
+            fileAnomalies.push(`Unable to determine file validity - processing anyway: ${file.name} - ${error}`);
+            // If vetting process fails, let the parser decide
+            return true;
+        }
+    });
+}
 self.addEventListener('message', (event) => {
     switch (event.data.request) {
         case 'scan':
             console.log(`Starting directory scan of ${event.data.directoryHandle.name}`);
+            // Update excluded filetypes if provided
+            if (event.data.excludedFiletypes) {
+                excludedFiletypes = event.data.excludedFiletypes;
+            }
             keepScanning = true;
             scanDirectory(event.data.directoryHandle);
             break;
@@ -57,17 +105,36 @@ function scanDirectory(dir) {
                 for (const entry of entries) {
                     if (entry.kind === 'file' && keepScanning) {
                         const file = yield entry.getFile();
-                        self.postMessage({
-                            response: 'file',
-                            fileIndex: fileIndex++,
-                            fileInfo: {
-                                path: prefix,
-                                name: entry.name,
-                                size: file.size,
-                                kind: 'handle',
-                                fileHandle: entry,
-                            },
-                        });
+                        const fileAnomalies = [];
+                        if (yield shouldProcessFile(file, fileAnomalies)) {
+                            // Send file to processing pipeline
+                            self.postMessage({
+                                response: 'file',
+                                fileIndex: fileIndex++,
+                                fileInfo: {
+                                    path: prefix,
+                                    name: entry.name,
+                                    size: file.size,
+                                    kind: 'handle',
+                                    fileHandle: entry,
+                                },
+                            });
+                        }
+                        else if (fileAnomalies.length > 0) {
+                            // Send scan anomalies as separate messsage so they are not sent to processing (curate)
+                            self.postMessage({
+                                response: 'scanAnomalies',
+                                fileIndex: fileIndex++,
+                                fileInfo: {
+                                    path: prefix,
+                                    name: entry.name,
+                                    size: file.size,
+                                    kind: 'handle',
+                                    fileHandle: entry,
+                                },
+                                anomalies: fileAnomalies,
+                            });
+                        }
                     }
                     else if (entry.kind === 'directory' && keepScanning) {
                         yield traverse(entry, prefix + '/' + entry.name);

package/dist/types/scanDirectoryWorker.d.ts

@@ -4,6 +4,11 @@ export type FileScanMsg = {
     response: 'file';
     fileIndex: number;
     fileInfo: TFileInfo;
+} | {
+    response: 'scanAnomalies';
+    fileIndex: number;
+    fileInfo: TFileInfo;
+    anomalies: string[];
 } | {
     response: 'done';
 };

package/dist/types/types.d.ts

@@ -137,6 +137,7 @@ export type TCurationSpecification = {
     additionalData?: {
         mapping: TMappedValues;
     } & (TMappingInputDirect | TMappingInputTwoPass);
+    excludedFiletypes?: string[];
 };
 type TProgressMessageBase = {
     totalFiles?: number;