dicom-curate 0.2.0 → 0.4.0

package/README.md

@@ -77,11 +77,11 @@ const columnMappings = extractColumnMappings([
   { subjectID: 'SubjectID2', blindedID: 'BlindedID2' },
 ])
 
-curateOne(
+curateOne({
   fileInfo, // path, name, size, kind, blob
-  undefined,
-  { curationSpec, columnMappings },
-)
+  mappingOptions: { curationSpec, columnMappings },
+})
+
 
 // Cache clean-up responsibility, e.g. for consistent UID mapping in `retainUIDsOption: 'Off'` is with caller
 clearCaches()

package/dist/esm/applyMappingsWorker.js

@@ -6,7 +6,12 @@ self.addEventListener('message', (event) => {
             const { serializedMappingOptions } = event.data;
             const mappingOptions = deserializeMappingOptions(serializedMappingOptions);
             try {
-                curateOne(event.data.fileInfo, event.data.outputDirectory, mappingOptions).then((mapResults) => {
+                curateOne({
+                    fileInfo: event.data.fileInfo,
+                    fileIndex: event.data.fileIndex,
+                    outputDirectory: event.data.outputDirectory,
+                    mappingOptions,
+                }).then((mapResults) => {
                     // Send finished message for completion
                     self.postMessage({
                         response: 'finished',

package/dist/esm/collectMappings.js

@@ -14,7 +14,7 @@ import deidentifyPS315E, { defaultPs315Options } from './deidentifyPS315E.js';
 import getParser from './getParser.js';
 import { specVersion } from './config/specVersion.js';
 import { get as _get } from 'lodash';
-export default function collectMappings(inputFilePath, dicomData, mappingOptions) {
+export default function collectMappings(inputFilePath, inputFileIndex, dicomData, mappingOptions) {
     var _a;
     const mapResults = {
         // original UID for this dicomData
@@ -33,13 +33,15 @@ export default function collectMappings(inputFilePath, dicomData, mappingOptions
     let finalSpec = {
         dicomPS315EOptions: defaultPs315Options,
         inputPathPattern: '',
-        modifications: () => ({
-            dicomHeader: {},
-            outputFilePathComponents: [
-                parser.getDicom('SeriesInstanceUID'),
-                parser.getFilePathComp(parser.FILENAME),
-            ],
-        }),
+        modifications(parser) {
+            return {
+                dicomHeader: {},
+                outputFilePathComponents: [
+                    parser.protectUid(parser.getDicom('SeriesInstanceUID')),
+                    parser.getFilePathComp(parser.FILENAME),
+                ],
+            };
+        },
         validation: () => ({ errors: [] }),
     };
     const _b = mappingOptions.curationSpec(), { modifications, validation } = _b, restSpec = __rest(_b, ["modifications", "validation"]);
@@ -54,8 +56,13 @@ export default function collectMappings(inputFilePath, dicomData, mappingOptions
     if (!mappingOptions.skipValidation) {
         finalSpec.validation = validation;
     }
+    // protect filename if we de-identify
+    const finalFilePath = finalSpec.dicomPS315EOptions === 'Off'
+        ? inputFilePath
+        : inputFilePath.slice(0, inputFilePath.lastIndexOf('/') + 1) +
+            `${String(inputFileIndex + 1).padStart(5, '0')}.dcm`;
     // create a parser object to be used in the eval'ed mappingFunctions
-    const parser = getParser(finalSpec.inputPathPattern, inputFilePath, naturalData, mappingOptions.columnMappings, finalSpec.additionalData);
+    const parser = getParser(finalSpec.inputPathPattern, finalFilePath, naturalData, finalSpec.dicomPS315EOptions, mappingOptions.columnMappings, finalSpec.additionalData);
     let modificationMap = finalSpec.modifications(parser);
     // List all validation errors
     mapResults.errors = finalSpec

package/dist/esm/curateDict.js

@@ -2,11 +2,11 @@ import * as dcmjs from 'dcmjs';
 import collectMappings from './collectMappings.js';
 import mapMetaheader from './mapMetaheader.js';
 import { set as _set, unset as _unset, cloneDeep as _cloneDeep } from 'lodash';
-export default function curateDict(inputFilePath, dicomData, mappingOptions) {
+export default function curateDict(inputFilePath, inputFileIndex, dicomData, mappingOptions) {
     //
     // Collect the mappings and apply them to the data
     //
-    const [naturalData, mapResults] = collectMappings(inputFilePath, dicomData, mappingOptions);
+    const [naturalData, mapResults] = collectMappings(inputFilePath, inputFileIndex, dicomData, mappingOptions);
     for (let tagPath in mapResults.mappings) {
         const [, operation, , mappedValue] = mapResults.mappings[tagPath];
         switch (operation) {

package/dist/esm/curateOne.js

@@ -10,8 +10,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import * as dcmjs from 'dcmjs';
 import createNestedDirectories from './createNestedDirectories.js';
 import curateDict from './curateDict.js';
-export function curateOne(fileInfo, outputDirectory, mappingOptions) {
-    return __awaiter(this, void 0, void 0, function* () {
+export function curateOne(_a) {
+    return __awaiter(this, arguments, void 0, function* ({ fileInfo, fileIndex = 0, outputDirectory, mappingOptions, }) {
         //
         // First, read the dicom instance data from the file handle or blob
         //
@@ -36,7 +36,7 @@ export function curateOne(fileInfo, outputDirectory, mappingOptions) {
             };
             return mapResults;
         }
-        const { dicomData: mappedDicomData, mapResults: clonedMapResults } = curateDict(`${fileInfo.path}/${fileInfo.name}`, dicomData, mappingOptions);
+        const { dicomData: mappedDicomData, mapResults: clonedMapResults } = curateDict(`${fileInfo.path}/${fileInfo.name}`, fileIndex, dicomData, mappingOptions);
         if (!mappingOptions.skipWrite) {
             // Finally, write the results
             const dirPath = clonedMapResults.outputFilePath

package/dist/esm/deidentifyPS315E.js

@@ -33,6 +33,9 @@ function temporalVr(vr) {
 function removeRetiredPrefix(name) {
     return name.startsWith('RETIRED_') ? name.slice(8) : name;
 }
+export function protectUid(uid, retainUIDsOption) {
+    return retainUIDsOption === 'Hashed' ? hashUid(uid) : replaceUid(uid);
+}
 const elementNamesToAlwaysKeepSet = new Set(elementNamesToAlwaysKeep);
 // Special conditions for some PS3.15 E1.1 elements.
 const ps315EElements = rawPs315EElements.map((elm) => {
@@ -235,7 +238,7 @@ export default function deidentifyPS315E({ naturalData, dicomPS315EOptions, date
                         // UID is not a known class UID.
                         !(uid in uidRegistryPS3_06_A1)) {
                         // UIDs that need to be mapped
-                        const mappedUID = retainUIDsOption === 'Hashed' ? hashUid(uid) : replaceUid(uid);
+                        const mappedUID = protectUid(uid, retainUIDsOption);
                         mapResults.mappings[attrPath] = [
                             uid,
                             'replace',

package/dist/esm/getParser.js

@@ -1,4 +1,5 @@
 import * as dcmjs from 'dcmjs';
+import { protectUid as rawProtectUid } from './deidentifyPS315E.js';
 import { getCsvMapping } from './csvMapping.js';
 import { UniqueNumbers } from './UniqueNumbers.js';
 export const FILEBASENAME = Symbol('fileBasename');
@@ -25,7 +26,15 @@ const { isUniqueInGroup, clearUniqueInGroupCache } = (function () {
     };
 })();
 export { clearUniqueNumberCache, clearUniqueInGroupCache };
-export default function getParser(inputPathPattern, inputFilePath, naturalData, columnMappings, additionalData) {
+export default function getParser(inputPathPattern, inputFilePath, naturalData, dicomPS315EOptions, columnMappings, additionalData) {
+    function protectUid(uid) {
+        let protectedUid = uid;
+        if (dicomPS315EOptions !== 'Off') {
+            const { retainUIDsOption } = dicomPS315EOptions;
+            protectedUid = rawProtectUid(uid, retainUIDsOption);
+        }
+        return protectedUid;
+    }
     function getDicom(attrName) {
         if (attrName in dcmjs.data.DicomMetaDictionary.dictionary) {
             // if in hex like "(0008,0100)", convert to text key
@@ -92,6 +101,7 @@ export default function getParser(inputPathPattern, inputFilePath, naturalData,
         getMapping,
         getDicom,
         missingDicom,
+        protectUid,
         // TODO: Phase this out in favor of ISO8601 duration handling.
         // Example of this logic:
         // ContentDate:

package/dist/esm/index.js

@@ -51,7 +51,8 @@ function initializeFileListWorker() {
     fileListWorker.addEventListener('message', (event) => {
         switch (event.data.response) {
             case 'file':
-                filesToProcess.push(event.data.fileInfo);
+                const { fileIndex, fileInfo } = event.data;
+                filesToProcess.push({ fileIndex, fileInfo });
                 // Could do some throttling:
                 // if (filesToProcess.length > 10) {
                 //   fileListWorker.postMessage({ request: 'stop' })
@@ -63,6 +64,7 @@ function initializeFileListWorker() {
                 directoryScanFinished = true;
                 break;
             default:
+                // @ts-expect-error: response is string here, not never
                 console.error(`Unknown response from worker ${event.data.response}`);
         }
         dispatchMappingJobs();
@@ -96,14 +98,12 @@ function initializeMappingWorkers() {
                     mapResultsList.push(event.data.mapResults);
                     workersActive -= 1;
                     // Report progress
-                    if (progressCallback) {
-                        progressCallback({
-                            response: 'progress',
-                            mapResults: event.data.mapResults,
-                            processedFiles: mapResultsList.length,
-                            totalFiles: filesToProcess.length + mapResultsList.length + workersActive,
-                        });
-                    }
+                    progressCallback({
+                        response: 'progress',
+                        mapResults: event.data.mapResults,
+                        processedFiles: mapResultsList.length,
+                        totalFiles: filesToProcess.length + mapResultsList.length + workersActive,
+                    });
                     dispatchMappingJobs();
                     if (mapResultsList.length % 100 === 0) {
                         console.log(`Finished mapping ${mapResultsList.length} files`);
@@ -119,7 +119,7 @@ function initializeMappingWorkers() {
 }
 function dispatchMappingJobs() {
     while (filesToProcess.length > 0 && availableMappingWorkers.length > 0) {
-        const fileInfo = filesToProcess.pop();
+        const { fileInfo, fileIndex } = filesToProcess.pop();
         const mappingWorker = availableMappingWorkers.pop();
         const _a = 
         // Not partial anymore.
@@ -127,6 +127,7 @@ function dispatchMappingJobs() {
         mappingWorker.postMessage({
             request: 'apply',
             fileInfo,
+            fileIndex,
             outputDirectory,
             serializedMappingOptions: serializeMappingOptions(mappingOptions),
         });
@@ -142,6 +143,12 @@ function dispatchMappingJobs() {
         clearCaches();
         console.log(`Finished mapping ${mapResultsList.length} files`);
         console.log('job is finished');
+        progressCallback({
+            response: 'done',
+            mapResultsList: mapResultsList,
+            processedFiles: mapResultsList.length,
+            totalFiles: mapResultsList.length,
+        });
     }
 }
 function collectMappingOptions(organizeOptions) {
@@ -182,7 +189,7 @@ function collectMappingOptions(organizeOptions) {
     });
 }
 function queueFilesForMapping(organizeOptions) {
-    organizeOptions.inputFiles.forEach((inputFile) => {
+    organizeOptions.inputFiles.forEach((inputFile, fileIndex) => {
         const fileInfo = {
             path: '',
             name: inputFile.name,
@@ -190,38 +197,51 @@ function queueFilesForMapping(organizeOptions) {
             kind: 'blob',
             blob: inputFile,
         };
-        filesToProcess.push(fileInfo);
+        filesToProcess.push({ fileInfo, fileIndex });
         dispatchMappingJobs();
     });
 }
 let progressCallback;
 function curateMany(organizeOptions, onProgress) {
     return __awaiter(this, void 0, void 0, function* () {
-        progressCallback = onProgress;
-        // create the mapping workers
-        initializeMappingWorkers();
-        // Set global mappingWorkerOptions
-        mappingWorkerOptions = (yield collectMappingOptions(organizeOptions));
-        //
-        // If the request provides a directory, then use the worker
-        // to recursively convert to fileSystemHandles.
-        // If the request provides a list of File objects,
-        // send them to the mapping workers directly.
-        //
-        if (organizeOptions.inputType === 'directory') {
-            const fileListWorker = initializeFileListWorker();
-            fileListWorker.postMessage({
-                request: 'scan',
-                directoryHandle: organizeOptions.inputDirectory,
-            });
-        }
-        else if (organizeOptions.inputType === 'files') {
-            queueFilesForMapping(organizeOptions);
-        }
-        else {
-            console.error('`inputType` should be "directory" or "files"');
-        }
-        dispatchMappingJobs();
+        return new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () {
+            // Resolve promise if progressCallback gets called with 'done'
+            progressCallback = (msg) => {
+                onProgress === null || onProgress === void 0 ? void 0 : onProgress(msg);
+                if (msg.response === 'done') {
+                    resolve(msg);
+                }
+            };
+            try {
+                // create the mapping workers
+                initializeMappingWorkers();
+                // Set global mappingWorkerOptions
+                mappingWorkerOptions = (yield collectMappingOptions(organizeOptions));
+                //
+                // If the request provides a directory, then use the worker
+                // to recursively convert to fileSystemHandles.
+                // If the request provides a list of File objects,
+                // send them to the mapping workers directly.
+                //
+                if (organizeOptions.inputType === 'directory') {
+                    const fileListWorker = initializeFileListWorker();
+                    fileListWorker.postMessage({
+                        request: 'scan',
+                        directoryHandle: organizeOptions.inputDirectory,
+                    });
+                }
+                else if (organizeOptions.inputType === 'files') {
+                    queueFilesForMapping(organizeOptions);
+                }
+                else {
+                    console.error('`inputType` should be "directory" or "files"');
+                }
+                dispatchMappingJobs();
+            }
+            catch (error) {
+                reject(error);
+            }
+        }));
     });
 }
 export { curateMany, curateOne, extractColumnMappings, clearCaches };

package/dist/esm/scanDirectoryWorker.js

@@ -34,27 +34,14 @@ function scanDirectory(dir) {
         function traverse(dir, prefix) {
             return __awaiter(this, void 0, void 0, function* () {
                 var _a, e_1, _b, _c;
+                // First, collect sorted dir entries
+                const entries = [];
                 try {
                     for (var _d = true, _e = __asyncValues(dir.values()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
                         _c = _f.value;
                         _d = false;
                         const entry = _c;
-                        if (entry.kind === 'file' && keepScanning) {
-                            const file = yield entry.getFile();
-                            self.postMessage({
-                                response: 'file',
-                                fileInfo: {
-                                    path: prefix,
-                                    name: entry.name,
-                                    size: file.size,
-                                    kind: 'handle',
-                                    fileHandle: entry,
-                                },
-                            });
-                        }
-                        else if (entry.kind === 'directory' && keepScanning) {
-                            yield traverse(entry, prefix + '/' + entry.name);
-                        }
+                        entries.push(entry);
                     }
                 }
                 catch (e_1_1) { e_1 = { error: e_1_1 }; }
@@ -64,6 +51,28 @@ function scanDirectory(dir) {
                     }
                     finally { if (e_1) throw e_1.error; }
                 }
+                entries.sort((a, b) => a.name.localeCompare(b.name));
+                // Assign sorted index to files
+                let fileIndex = 0;
+                for (const entry of entries) {
+                    if (entry.kind === 'file' && keepScanning) {
+                        const file = yield entry.getFile();
+                        self.postMessage({
+                            response: 'file',
+                            fileIndex: fileIndex++,
+                            fileInfo: {
+                                path: prefix,
+                                name: entry.name,
+                                size: file.size,
+                                kind: 'handle',
+                                fileHandle: entry,
+                            },
+                        });
+                    }
+                    else if (entry.kind === 'directory' && keepScanning) {
+                        yield traverse(entry, prefix + '/' + entry.name);
+                    }
+                }
             });
         }
         yield traverse(dir, dir.name);

package/dist/types/applyMappingsWorker.d.ts

@@ -1 +1,8 @@
-export {};
+import type { TFileInfo, TSerializedMappingOptions } from './types';
+export type MappingRequest = {
+    request: 'apply';
+    fileInfo: TFileInfo;
+    fileIndex: number;
+    outputDirectory?: FileSystemDirectoryHandle;
+    serializedMappingOptions: TSerializedMappingOptions;
+};

package/dist/types/collectMappings.d.ts

@@ -1,3 +1,3 @@
 import type { TMappingOptions, TMapResults } from './types';
 import type { TDicomData, TNaturalData } from 'dcmjs';
-export default function collectMappings(inputFilePath: string, dicomData: TDicomData, mappingOptions: TMappingOptions): [TNaturalData, TMapResults];
+export default function collectMappings(inputFilePath: string, inputFileIndex: number, dicomData: TDicomData, mappingOptions: TMappingOptions): [TNaturalData, TMapResults];

package/dist/types/curateDict.d.ts

@@ -1,7 +1,7 @@
 import * as dcmjs from 'dcmjs';
 import type { TDicomData } from 'dcmjs';
 import type { TMappingOptions } from './types';
-export default function curateDict(inputFilePath: string, dicomData: TDicomData, mappingOptions: TMappingOptions): {
+export default function curateDict(inputFilePath: string, inputFileIndex: number, dicomData: TDicomData, mappingOptions: TMappingOptions): {
     dicomData: dcmjs.data.DicomDict;
     mapResults: import("./types").TMapResults;
 };

package/dist/types/curateOne.d.ts

@@ -1,4 +1,10 @@
 import type { TFileInfo, TMappingOptions, TMapResults } from './types';
-export declare function curateOne(fileInfo: TFileInfo, outputDirectory: FileSystemDirectoryHandle | undefined, mappingOptions: TMappingOptions): Promise<Omit<Partial<TMapResults>, 'anomalies'> & {
+export type TCurateOneArgs = {
+    fileInfo: TFileInfo;
+    fileIndex?: number;
+    outputDirectory: FileSystemDirectoryHandle | undefined;
+    mappingOptions: TMappingOptions;
+};
+export declare function curateOne({ fileInfo, fileIndex, outputDirectory, mappingOptions, }: TCurateOneArgs): Promise<Omit<Partial<TMapResults>, 'anomalies'> & {
     anomalies: TMapResults['anomalies'];
 }>;

package/dist/types/deidentifyPS315E.d.ts

@@ -1,5 +1,6 @@
 import type { TNaturalData } from 'dcmjs';
 import type { Iso8601Duration, TPs315Options, TMapResults } from './types';
+export declare function protectUid(uid: string, retainUIDsOption: string): string;
 export default function deidentifyPS315E({ naturalData, dicomPS315EOptions, dateOffset, mapResults, }: {
     naturalData: TNaturalData;
     dicomPS315EOptions: TPs315Options;

package/dist/types/getParser.d.ts

@@ -6,4 +6,4 @@ export declare const FILENAME: symbol;
 declare const clearUniqueNumberCache: () => void;
 declare const clearUniqueInGroupCache: () => void;
 export { clearUniqueNumberCache, clearUniqueInGroupCache };
-export default function getParser(inputPathPattern: string, inputFilePath: string, naturalData: TNaturalData, columnMappings?: TColumnMappings, additionalData?: TCurationSpecification['additionalData']): TParser;
+export default function getParser(inputPathPattern: string, inputFilePath: string, naturalData: TNaturalData, dicomPS315EOptions: TCurationSpecification['dicomPS315EOptions'], columnMappings?: TColumnMappings, additionalData?: TCurationSpecification['additionalData']): TParser;

package/dist/types/index.d.ts

@@ -1,12 +1,13 @@
 import { extractColumnMappings } from './csvMapping';
 import { clearCaches } from './clearCaches';
 import { curateOne } from './curateOne';
-import type { OrganizeOptions, TProgressMessage } from './types';
+import type { OrganizeOptions, TProgressMessage, TProgressMessageDone } from './types';
 export type ProgressCallback = (message: TProgressMessage) => void;
 export type { TPs315Options, TMapResults, TProgressMessage, OrganizeOptions, TCurationSpecification, } from './types';
+export { TCurateOneArgs } from './curateOne';
 export { specVersion } from './config/specVersion';
 export { sample2PassCurationSpecification as sampleSpecification } from './config/sample2PassCurationSpecification';
 export { csvTextToRows } from './csvMapping';
 export type { Row } from './csvMapping';
-declare function curateMany(organizeOptions: OrganizeOptions, onProgress?: ProgressCallback): Promise<void>;
+declare function curateMany(organizeOptions: OrganizeOptions, onProgress?: ProgressCallback): Promise<TProgressMessageDone>;
 export { curateMany, curateOne, extractColumnMappings, clearCaches };

package/dist/types/scanDirectoryWorker.d.ts

@@ -1 +1,9 @@
+import type { TFileInfo } from './types';
 export {};
+export type FileScanMsg = {
+    response: 'file';
+    fileIndex: number;
+    fileInfo: TFileInfo;
+} | {
+    response: 'done';
+};

package/dist/types/serializeMappingOptions.d.ts

@@ -1,7 +1,3 @@
-import { TMappingOptions } from './types';
-type TSerializedMappingOptions = Omit<TMappingOptions, 'curationSpec'> & {
-    curationSpecStr: string;
-};
+import { TMappingOptions, TSerializedMappingOptions } from './types';
 export declare function serializeMappingOptions(mappingOptions: TMappingOptions): TSerializedMappingOptions;
 export declare function deserializeMappingOptions(serializedMappingOptions: TSerializedMappingOptions): TMappingOptions;
-export {};

package/dist/types/types.d.ts

@@ -34,6 +34,9 @@ export type TMappingOptions = {
     skipValidation?: boolean;
     dateOffset?: Iso8601Duration;
 };
+export type TSerializedMappingOptions = Omit<TMappingOptions, 'curationSpec'> & {
+    curationSpecStr: string;
+};
 export type TFileInfo = {
     path: string;
     name: string;
@@ -92,6 +95,7 @@ export type TParser = {
     getMapping: ((value: string) => string | number) | undefined;
     getDicom: (attrName: string) => any;
     missingDicom: (attrName: string) => boolean;
+    protectUid: (uid: string) => string;
     addDays: (dicomDateString: string, offsetDays: number) => string;
     FILENAME: symbol;
     FILEBASENAME: symbol;
@@ -134,11 +138,18 @@ export type TCurationSpecification = {
         mapping: TMappedValues;
     } & (TMappingInputDirect | TMappingInputTwoPass);
 };
-export type TProgressMessage = {
-    response: 'start' | 'progress' | 'finished' | 'error';
+type TProgressMessageBase = {
     totalFiles?: number;
     processedFiles?: number;
+};
+type TProgressMessageProgress = TProgressMessageBase & {
+    response: 'progress';
     mapResults?: TMapResults;
     error?: Error;
 };
+export type TProgressMessageDone = TProgressMessageBase & {
+    response: 'done';
+    mapResultsList: TMapResults[];
+};
+export type TProgressMessage = TProgressMessageProgress | TProgressMessageDone;
 export {};