dicom-curate 0.30.0 → 0.32.0

package/dist/esm/applyMappingsWorker.js

@@ -80107,8 +80107,8 @@ async function curateOne({
           console.error(
             `Upload failed for ${uploadUrl}: ${resp.status} ${resp.statusText}`
           );
-          clonedMapResults.errors = clonedMapResults.errors ?? [];
-          clonedMapResults.errors.push(
+          clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+          clonedMapResults.uploadErrors.push(
             `Upload failed: ${resp.status} ${resp.statusText}`
           );
         } else {
@@ -80121,8 +80121,8 @@ async function curateOne({
         }
       } catch (e4) {
         console.error("Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }
@@ -80161,8 +80161,8 @@ async function curateOne({
         };
       } catch (e4) {
         console.error("S3 Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `S3 Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }

package/dist/esm/curateOne.js

@@ -73816,8 +73816,8 @@ async function curateOne({
           console.error(
             `Upload failed for ${uploadUrl}: ${resp.status} ${resp.statusText}`
           );
-          clonedMapResults.errors = clonedMapResults.errors ?? [];
-          clonedMapResults.errors.push(
+          clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+          clonedMapResults.uploadErrors.push(
             `Upload failed: ${resp.status} ${resp.statusText}`
           );
         } else {
@@ -73830,8 +73830,8 @@ async function curateOne({
         }
       } catch (e4) {
         console.error("Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }
@@ -73870,8 +73870,8 @@ async function curateOne({
         };
       } catch (e4) {
         console.error("S3 Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `S3 Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }

package/dist/esm/index.js

@@ -81667,8 +81667,8 @@ async function curateOne({
           console.error(
             `Upload failed for ${uploadUrl}: ${resp.status} ${resp.statusText}`
           );
-          clonedMapResults.errors = clonedMapResults.errors ?? [];
-          clonedMapResults.errors.push(
+          clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+          clonedMapResults.uploadErrors.push(
             `Upload failed: ${resp.status} ${resp.statusText}`
           );
         } else {
@@ -81681,8 +81681,8 @@ async function curateOne({
         }
       } catch (e4) {
         console.error("Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }
@@ -81721,8 +81721,8 @@ async function curateOne({
         };
       } catch (e4) {
         console.error("S3 Upload error", e4);
-        clonedMapResults.errors = clonedMapResults.errors ?? [];
-        clonedMapResults.errors.push(
+        clonedMapResults.uploadErrors = clonedMapResults.uploadErrors ?? [];
+        clonedMapResults.uploadErrors.push(
           `S3 Upload error: ${e4 instanceof Error ? e4.message : String(e4)}`
         );
       }
@@ -87495,6 +87495,7 @@ var progressCallback = () => {
 };
 var scanResumeCallback = null;
 var scanPaused = false;
+var totalDiscoveredFiles = void 0;
 var LOW_WATER_MARK = 50;
 function setMappingWorkerOptions(opts) {
   mappingWorkerOptions = opts;
@@ -87503,6 +87504,9 @@ function setScanResumeCallback(cb) {
   scanResumeCallback = cb;
   scanPaused = false;
 }
+function setTotalDiscoveredFiles(n4) {
+  totalDiscoveredFiles = n4;
+}
 function markScanPaused() {
   scanPaused = true;
 }
@@ -87538,6 +87542,7 @@ async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, p
   filesToProcess = [];
   directoryScanFinished = false;
   scanAnomalies = [];
+  totalDiscoveredFiles = void 0;
   if (progressCb)
     progressCallback = progressCb;
   const effectiveWorkerCount = workerCount ?? Math.min(await getHardwareConcurrency(), 8);
@@ -87642,7 +87647,7 @@ function recoverCrashedWorker(mappingWorker, errorMessage) {
     response: "progress",
     mapResults: errorMapResults,
     processedFiles: filesMapped,
-    totalFiles: filesToProcess.length + filesMapped + workersActive
+    totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
   });
   dispatchMappingJobs();
   pendingReplacements += 1;
@@ -87704,7 +87709,7 @@ async function createMappingWorker() {
           response: "progress",
           mapResults: event.data.mapResults,
           processedFiles: filesMapped,
-          totalFiles: filesToProcess.length + filesMapped + workersActive
+          totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
         });
         dispatchMappingJobs();
         if (filesMapped % 100 === 0) {
@@ -87730,7 +87735,7 @@ async function createMappingWorker() {
           response: "progress",
           mapResults: errorMapResults,
           processedFiles: filesMapped,
-          totalFiles: filesToProcess.length + filesMapped + workersActive
+          totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
         });
         dispatchMappingJobs();
         break;
@@ -87795,6 +87800,10 @@ async function initializeFileListWorker(rejectCallback) {
           scanAnomalies.push({ fileInfo: anomalyFileInfo, anomalies });
           break;
         }
+        case "count": {
+          setTotalDiscoveredFiles(event.data.totalDiscovered);
+          break;
+        }
         case "done": {
           console.log("directoryScanFinished");
           setDirectoryScanFinished(true);
@@ -87810,7 +87819,11 @@ async function initializeFileListWorker(rejectCallback) {
           break;
         }
         default: {
-          console.error(`Unknown response from worker ${event.data.response}`);
+          console.error(
+            // @ts-expect-error: event.data is never here (all cases handled),
+            // but we log anyway as a defensive guard against future message types.
+            `Unknown response from worker ${event.data.response}`
+          );
         }
       }
       dispatchMappingJobs();

package/dist/esm/mappingWorkerPool.js

@@ -12067,6 +12067,7 @@ var progressCallback = () => {
 };
 var scanResumeCallback = null;
 var scanPaused = false;
+var totalDiscoveredFiles = void 0;
 var LOW_WATER_MARK = 50;
 function setMappingWorkerOptions(opts) {
   mappingWorkerOptions = opts;
@@ -12075,6 +12076,9 @@ function setScanResumeCallback(cb) {
   scanResumeCallback = cb;
   scanPaused = false;
 }
+function setTotalDiscoveredFiles(n) {
+  totalDiscoveredFiles = n;
+}
 function markScanPaused() {
   scanPaused = true;
 }
@@ -12113,6 +12117,7 @@ async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, p
   filesToProcess = [];
   directoryScanFinished = false;
   scanAnomalies = [];
+  totalDiscoveredFiles = void 0;
   if (progressCb)
     progressCallback = progressCb;
   const effectiveWorkerCount = workerCount ?? Math.min(await getHardwareConcurrency(), 8);
@@ -12217,7 +12222,7 @@ function recoverCrashedWorker(mappingWorker, errorMessage) {
     response: "progress",
     mapResults: errorMapResults,
     processedFiles: filesMapped,
-    totalFiles: filesToProcess.length + filesMapped + workersActive
+    totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
   });
   dispatchMappingJobs();
   pendingReplacements += 1;
@@ -12279,7 +12284,7 @@ async function createMappingWorker() {
           response: "progress",
           mapResults: event.data.mapResults,
           processedFiles: filesMapped,
-          totalFiles: filesToProcess.length + filesMapped + workersActive
+          totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
         });
         dispatchMappingJobs();
         if (filesMapped % 100 === 0) {
@@ -12305,7 +12310,7 @@ async function createMappingWorker() {
           response: "progress",
           mapResults: errorMapResults,
           processedFiles: filesMapped,
-          totalFiles: filesToProcess.length + filesMapped + workersActive
+          totalFiles: totalDiscoveredFiles ?? filesToProcess.length + filesMapped + workersActive
         });
         dispatchMappingJobs();
         break;
@@ -12339,5 +12344,6 @@ export {
   setDirectoryScanFinished,
   setMappingWorkerOptions,
   setScanResumeCallback,
+  setTotalDiscoveredFiles,
   terminateAllWorkers
 };

package/dist/esm/scanDirectoryWorker.js

@@ -37139,8 +37139,12 @@ var DEFAULT_EXCLUDED_FILETYPES = [
 var keepScanning = true;
 var pauseResolve = null;
 var pausePromise = null;
+var totalDiscovered = 0;
+var countingMode = false;
+var fileBuffer = [];
 function pauseScanning() {
   if (!pausePromise) {
+    countingMode = true;
     pausePromise = new Promise((resolve) => {
       pauseResolve = resolve;
     });
@@ -37148,6 +37152,7 @@ function pauseScanning() {
 }
 function resumeScanning() {
   if (pauseResolve) {
+    countingMode = false;
     pauseResolve();
     pauseResolve = null;
     pausePromise = null;
@@ -37281,6 +37286,113 @@ async function shouldProcessFileNode(filePath, fileName, fileSize, fileAnomalies
     return true;
   }
 }
+function cheapFilter(fileName, fileSize, filePath) {
+  const allExcludedFiletypes = [
+    ...noDefaultExclusions ? [] : DEFAULT_EXCLUDED_FILETYPES,
+    ...excludedFiletypes
+  ];
+  if (excludedPathRegexes.some((regex) => regex.test(filePath))) {
+    return false;
+  }
+  if (allExcludedFiletypes.some(
+    (excluded) => fileName.toLowerCase() === excluded.toLowerCase()
+  )) {
+    return false;
+  }
+  if (noDicomSignatureCheck) {
+    return true;
+  }
+  if (fileSize < 132) {
+    return false;
+  }
+  return true;
+}
+async function drainBuffer() {
+  while (fileBuffer.length > 0 && keepScanning) {
+    const item = fileBuffer.shift();
+    const fileAnomalies = [];
+    if (item.kind === "handle") {
+      const filePath = `${item.prefix}/${item.name}`;
+      if (await shouldProcessFile(item.file, fileAnomalies, filePath)) {
+        globalThis.postMessage({
+          response: "file",
+          fileInfo: {
+            path: item.prefix,
+            name: item.name,
+            size: item.size,
+            kind: "handle",
+            fileHandle: item.entry
+          },
+          previousFileInfo: item.prev
+        });
+      } else {
+        totalDiscovered--;
+        globalThis.postMessage({
+          response: "count",
+          totalDiscovered
+        });
+        if (fileAnomalies.length > 0) {
+          globalThis.postMessage({
+            response: "scanAnomalies",
+            fileInfo: {
+              path: item.prefix,
+              name: item.name,
+              size: item.size,
+              kind: "handle",
+              fileHandle: item.entry
+            },
+            anomalies: fileAnomalies,
+            previousFileInfo: item.prev
+          });
+        }
+      }
+    } else {
+      if (await shouldProcessFileNode(
+        item.filePath,
+        item.name,
+        item.size,
+        fileAnomalies,
+        `${item.prefix}/${item.name}`
+      )) {
+        globalThis.postMessage({
+          response: "file",
+          fileInfo: {
+            path: item.prefix,
+            name: item.name,
+            size: item.size,
+            kind: "path",
+            fullPath: item.filePath
+          },
+          previousFileInfo: item.prev
+        });
+      } else {
+        totalDiscovered--;
+        globalThis.postMessage({
+          response: "count",
+          totalDiscovered
+        });
+        if (fileAnomalies.length > 0) {
+          globalThis.postMessage({
+            response: "scanAnomalies",
+            fileInfo: {
+              path: item.prefix,
+              name: item.name,
+              size: item.size,
+              kind: "path",
+              fullPath: item.filePath
+            },
+            anomalies: fileAnomalies,
+            previousFileInfo: item.prev
+          });
+        }
+      }
+    }
+    if (!await waitIfPaused())
+      return;
+    if (countingMode)
+      return;
+  }
+}
 fixupNodeWorkerEnvironment().then(() => {
   globalThis.addEventListener("message", (event) => {
     switch (event.data.request) {
@@ -37304,6 +37416,9 @@ fixupNodeWorkerEnvironment().then(() => {
         noDicomSignatureCheck = event.data.noDicomSignatureCheck ?? false;
         noDefaultExclusions = event.data.noDefaultExclusions ?? false;
         keepScanning = true;
+        totalDiscovered = 0;
+        countingMode = false;
+        fileBuffer.length = 0;
         if ("path" in eventData) {
           scanDirectoryNode(eventData.path);
         } else if ("directoryHandle" in eventData) {
@@ -37349,6 +37464,7 @@ async function scanS3Bucket(bucketOptions) {
         for (const item of data2.Contents) {
           const fileAnomalies = [];
           if (item.Key && item.Size !== void 0 && await shouldProcessFileItem(item, fileAnomalies)) {
+            totalDiscovered++;
             const prev = previousIndex ? previousIndex[item.Key] : void 0;
             globalThis.postMessage({
               response: "file",
@@ -37397,18 +37513,54 @@ async function scanDirectory(dir) {
     for await (const entry of dir2.values()) {
       if (!keepScanning)
         return;
-      if (!await waitIfPaused())
-        return;
       if (entry.kind === "file") {
         const file = await entry.getFile();
+        const key = `${prefix}/${entry.name}`;
+        const prev = previousIndex ? previousIndex[key] : void 0;
+        if (countingMode) {
+          if (cheapFilter(entry.name, file.size, key)) {
+            totalDiscovered++;
+            fileBuffer.push({
+              kind: "handle",
+              entry,
+              file,
+              prefix,
+              name: entry.name,
+              size: file.size,
+              prev
+            });
+            globalThis.postMessage({
+              response: "count",
+              totalDiscovered
+            });
+          }
+          continue;
+        }
+        if (fileBuffer.length > 0) {
+          await drainBuffer();
+          if (countingMode) {
+            if (cheapFilter(entry.name, file.size, key)) {
+              totalDiscovered++;
+              fileBuffer.push({
+                kind: "handle",
+                entry,
+                file,
+                prefix,
+                name: entry.name,
+                size: file.size,
+                prev
+              });
+              globalThis.postMessage({
+                response: "count",
+                totalDiscovered
+              });
+            }
+            continue;
+          }
+        }
         const fileAnomalies = [];
-        if (await shouldProcessFile(
-          file,
-          fileAnomalies,
-          `${prefix}/${entry.name}`
-        )) {
-          const key = `${prefix}/${entry.name}`;
-          const prev = previousIndex ? previousIndex[key] : void 0;
+        if (await shouldProcessFile(file, fileAnomalies, key)) {
+          totalDiscovered++;
           globalThis.postMessage({
             response: "file",
             fileInfo: {
@@ -37421,8 +37573,6 @@ async function scanDirectory(dir) {
             previousFileInfo: prev
           });
         } else if (fileAnomalies.length > 0) {
-          const key = `${prefix}/${entry.name}`;
-          const prev = previousIndex ? previousIndex[key] : void 0;
           globalThis.postMessage({
             response: "scanAnomalies",
             fileInfo: {
@@ -37436,6 +37586,8 @@ async function scanDirectory(dir) {
             previousFileInfo: prev
           });
         }
+        if (!await waitIfPaused())
+          return;
       } else if (entry.kind === "directory") {
         await traverse(
           entry,
@@ -37446,6 +37598,10 @@ async function scanDirectory(dir) {
   }
   try {
     await traverse(dir, dir.name);
+    if (fileBuffer.length > 0) {
+      countingMode = false;
+      await drainBuffer();
+    }
     globalThis.postMessage({ response: "done" });
   } catch (error2) {
     globalThis.postMessage({
@@ -37466,13 +37622,51 @@ async function scanDirectoryNode(dirPath) {
       for (const entry of entries) {
         if (!keepScanning)
           return;
-        if (!await waitIfPaused())
-          return;
         if (entry.isFile()) {
           const filePath = path.join(currentPath, entry.name);
           const stats = await fs.stat(filePath);
-          const fileAnomalies = [];
           const key = `${prefix}/${entry.name}`;
+          const prev = previousIndex ? previousIndex[key] : void 0;
+          if (countingMode) {
+            if (cheapFilter(entry.name, stats.size, key)) {
+              totalDiscovered++;
+              fileBuffer.push({
+                kind: "path",
+                filePath,
+                name: entry.name,
+                size: stats.size,
+                prefix,
+                prev
+              });
+              globalThis.postMessage({
+                response: "count",
+                totalDiscovered
+              });
+            }
+            continue;
+          }
+          if (fileBuffer.length > 0) {
+            await drainBuffer();
+            if (countingMode) {
+              if (cheapFilter(entry.name, stats.size, key)) {
+                totalDiscovered++;
+                fileBuffer.push({
+                  kind: "path",
+                  filePath,
+                  name: entry.name,
+                  size: stats.size,
+                  prefix,
+                  prev
+                });
+                globalThis.postMessage({
+                  response: "count",
+                  totalDiscovered
+                });
+              }
+              continue;
+            }
+          }
+          const fileAnomalies = [];
           if (await shouldProcessFileNode(
             filePath,
             entry.name,
@@ -37480,7 +37674,7 @@ async function scanDirectoryNode(dirPath) {
             fileAnomalies,
             key
           )) {
-            const prev = previousIndex ? previousIndex[key] : void 0;
+            totalDiscovered++;
             globalThis.postMessage({
               response: "file",
               fileInfo: {
@@ -37493,7 +37687,6 @@ async function scanDirectoryNode(dirPath) {
               previousFileInfo: prev
             });
           } else if (fileAnomalies.length > 0) {
-            const prev = previousIndex ? previousIndex[key] : void 0;
             globalThis.postMessage({
               response: "scanAnomalies",
               fileInfo: {
@@ -37507,6 +37700,8 @@ async function scanDirectoryNode(dirPath) {
               previousFileInfo: prev
             });
           }
+          if (!await waitIfPaused())
+            return;
         } else if (entry.isDirectory()) {
           await traverse(
             path.join(currentPath, entry.name),
@@ -37517,6 +37712,10 @@ async function scanDirectoryNode(dirPath) {
     }
     const dirName = await import("path").then((p4) => p4.basename(dirPath));
     await traverse(dirPath, dirName);
+    if (fileBuffer.length > 0) {
+      countingMode = false;
+      await drainBuffer();
+    }
     globalThis.postMessage({ response: "done" });
   } catch (error2) {
     globalThis.postMessage({

package/dist/types/curateOne.d.ts

@@ -1,4 +1,4 @@
-import type { TFileInfo, THashMethod, TOutputTarget, TMappingOptions, TMapResults } from './types';
+import type { TFileInfo, THashMethod, TMappingOptions, TMapResults, TOutputTarget } from './types';
 export type TCurateOneArgs = {
     fileInfo: TFileInfo;
     outputTarget: TOutputTarget;

package/dist/types/mappingWorkerPool.d.ts

@@ -34,6 +34,11 @@ export declare function setMappingWorkerOptions(opts: TMappingWorkerOptions): vo
  * after the scan worker is created.
  */
 export declare function setScanResumeCallback(cb: (() => void) | null): void;
+/**
+ * Update the total discovered file count from the scan worker's 'count'
+ * messages. Pass undefined to reset (e.g. at the start of a new run).
+ */
+export declare function setTotalDiscoveredFiles(n: number | undefined): void;
 /**
  * Mark the scan as paused. Called from the scan worker message handler in
  * index.ts when the queue exceeds the high-water mark.

package/dist/types/scanDirectoryWorker.d.ts

@@ -20,6 +20,9 @@ export type FileScanMsg = {
 } | {
     response: 'error';
     error: string;
+} | {
+    response: 'count';
+    totalDiscovered: number;
 } | {
     response: 'done';
 };

package/dist/types/types.d.ts

@@ -1,6 +1,6 @@
-import { TColumnMappings, TMappedValues, Row } from './csvMapping';
 import type { TNaturalData } from 'dcmjs';
 import type { SpecPart } from './composeSpecs';
+import type { Row, TColumnMappings, TMappedValues } from './csvMapping';
 export type Iso8601Duration = string;
 export type TPs315Options = {
     cleanDescriptorsOption: boolean;
@@ -131,6 +131,9 @@ export type TMapResults = {
     };
     anomalies: string[];
     errors: string[];
+    /** Upload/write failures only (retryable). Separate from `errors` which
+     *  contains DICOM validation issues that cannot be resolved by retrying. */
+    uploadErrors?: string[];
     quarantine: {
         [objectPath: string]: string;
     };