dicom-curate 0.28.1 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/applyMappingsWorker.js +12 -6
- package/dist/esm/curateOne.js +12 -6
- package/dist/esm/index.js +84 -13
- package/dist/esm/mappingWorkerPool.js +37 -1
- package/dist/types/mappingWorkerPool.d.ts +12 -0
- package/dist/types/types.d.ts +2 -0
- package/dist/umd/dicom-curate.umd.js +108 -9
- package/dist/umd/dicom-curate.umd.js.map +1 -1
- package/dist/umd/dicom-curate.umd.min.js +2 -2
- package/dist/umd/dicom-curate.umd.min.js.map +1 -1
- package/package.json +1 -1
|
@@ -79926,7 +79926,7 @@ async function curateOne({
|
|
|
79926
79926
|
canSkip = previousSourceFileInfo.size === fileInfo.size && previousSourceFileInfo.mtime === mtime;
|
|
79927
79927
|
}
|
|
79928
79928
|
}
|
|
79929
|
-
const noMapResult = (outputFilePath) => {
|
|
79929
|
+
const noMapResult = (outputFilePath, knownPostMappedHash) => {
|
|
79930
79930
|
const retval = {
|
|
79931
79931
|
sourceInstanceUID: `unchanged_${fileInfo.name.replace(/[^a-zA-Z0-9]/g, "_")}`,
|
|
79932
79932
|
mappings: {},
|
|
@@ -79939,7 +79939,10 @@ async function curateOne({
|
|
|
79939
79939
|
size: fileInfo.size,
|
|
79940
79940
|
path: fileInfo.path,
|
|
79941
79941
|
mtime: previousSourceFileInfo?.mtime,
|
|
79942
|
-
preMappedHash
|
|
79942
|
+
preMappedHash,
|
|
79943
|
+
...knownPostMappedHash !== void 0 && {
|
|
79944
|
+
postMappedHash: knownPostMappedHash
|
|
79945
|
+
}
|
|
79943
79946
|
},
|
|
79944
79947
|
// include curationTime even when skipped to measure hashing/check time
|
|
79945
79948
|
curationTime: performance.now() - startTime,
|
|
@@ -80041,7 +80044,7 @@ async function curateOne({
|
|
|
80041
80044
|
fileArrayBuffer = null;
|
|
80042
80045
|
const previousPostMappedHash = previousMappedFileInfo ? (await previousMappedFileInfo(clonedMapResults.outputFilePath))?.postMappedHash : void 0;
|
|
80043
80046
|
if (previousPostMappedHash !== void 0 && previousPostMappedHash === postMappedHash) {
|
|
80044
|
-
return noMapResult(clonedMapResults.outputFilePath);
|
|
80047
|
+
return noMapResult(clonedMapResults.outputFilePath, postMappedHash);
|
|
80045
80048
|
}
|
|
80046
80049
|
if (typeof outputTarget?.directory === "object" && "getFileHandle" in outputTarget.directory) {
|
|
80047
80050
|
const subDirectoryHandle = await createNestedDirectories(
|
|
@@ -80109,9 +80112,11 @@ async function curateOne({
|
|
|
80109
80112
|
`Upload failed: ${resp.status} ${resp.statusText}`
|
|
80110
80113
|
);
|
|
80111
80114
|
} else {
|
|
80115
|
+
const etag = resp.headers.get("etag") ?? void 0;
|
|
80112
80116
|
clonedMapResults.outputUpload = clonedMapResults.outputUpload ?? {
|
|
80113
80117
|
url: uploadUrl,
|
|
80114
|
-
status: resp.status
|
|
80118
|
+
status: resp.status,
|
|
80119
|
+
etag
|
|
80115
80120
|
};
|
|
80116
80121
|
}
|
|
80117
80122
|
} catch (e4) {
|
|
@@ -80132,7 +80137,7 @@ async function curateOne({
|
|
|
80132
80137
|
try {
|
|
80133
80138
|
const prefix = outputTarget.s3.prefix ? outputTarget.s3.prefix.endsWith("/") ? outputTarget.s3.prefix : outputTarget.s3.prefix + "/" : "";
|
|
80134
80139
|
const key = prefix + clonedMapResults.outputFilePath;
|
|
80135
|
-
await client.send(
|
|
80140
|
+
const putResponse = await client.send(
|
|
80136
80141
|
new s32.PutObjectCommand({
|
|
80137
80142
|
Bucket: outputTarget.s3.bucketName,
|
|
80138
80143
|
Key: key,
|
|
@@ -80151,7 +80156,8 @@ async function curateOne({
|
|
|
80151
80156
|
const uploadUrl = `s3://${outputTarget.s3.bucketName}/${key}`;
|
|
80152
80157
|
clonedMapResults.outputUpload = {
|
|
80153
80158
|
url: uploadUrl,
|
|
80154
|
-
status: 200
|
|
80159
|
+
status: 200,
|
|
80160
|
+
etag: putResponse.ETag ?? void 0
|
|
80155
80161
|
};
|
|
80156
80162
|
} catch (e4) {
|
|
80157
80163
|
console.error("S3 Upload error", e4);
|
package/dist/esm/curateOne.js
CHANGED
|
@@ -73635,7 +73635,7 @@ async function curateOne({
|
|
|
73635
73635
|
canSkip = previousSourceFileInfo.size === fileInfo.size && previousSourceFileInfo.mtime === mtime;
|
|
73636
73636
|
}
|
|
73637
73637
|
}
|
|
73638
|
-
const noMapResult = (outputFilePath) => {
|
|
73638
|
+
const noMapResult = (outputFilePath, knownPostMappedHash) => {
|
|
73639
73639
|
const retval = {
|
|
73640
73640
|
sourceInstanceUID: `unchanged_${fileInfo.name.replace(/[^a-zA-Z0-9]/g, "_")}`,
|
|
73641
73641
|
mappings: {},
|
|
@@ -73648,7 +73648,10 @@ async function curateOne({
|
|
|
73648
73648
|
size: fileInfo.size,
|
|
73649
73649
|
path: fileInfo.path,
|
|
73650
73650
|
mtime: previousSourceFileInfo?.mtime,
|
|
73651
|
-
preMappedHash
|
|
73651
|
+
preMappedHash,
|
|
73652
|
+
...knownPostMappedHash !== void 0 && {
|
|
73653
|
+
postMappedHash: knownPostMappedHash
|
|
73654
|
+
}
|
|
73652
73655
|
},
|
|
73653
73656
|
// include curationTime even when skipped to measure hashing/check time
|
|
73654
73657
|
curationTime: performance.now() - startTime,
|
|
@@ -73750,7 +73753,7 @@ async function curateOne({
|
|
|
73750
73753
|
fileArrayBuffer = null;
|
|
73751
73754
|
const previousPostMappedHash = previousMappedFileInfo ? (await previousMappedFileInfo(clonedMapResults.outputFilePath))?.postMappedHash : void 0;
|
|
73752
73755
|
if (previousPostMappedHash !== void 0 && previousPostMappedHash === postMappedHash) {
|
|
73753
|
-
return noMapResult(clonedMapResults.outputFilePath);
|
|
73756
|
+
return noMapResult(clonedMapResults.outputFilePath, postMappedHash);
|
|
73754
73757
|
}
|
|
73755
73758
|
if (typeof outputTarget?.directory === "object" && "getFileHandle" in outputTarget.directory) {
|
|
73756
73759
|
const subDirectoryHandle = await createNestedDirectories(
|
|
@@ -73818,9 +73821,11 @@ async function curateOne({
|
|
|
73818
73821
|
`Upload failed: ${resp.status} ${resp.statusText}`
|
|
73819
73822
|
);
|
|
73820
73823
|
} else {
|
|
73824
|
+
const etag = resp.headers.get("etag") ?? void 0;
|
|
73821
73825
|
clonedMapResults.outputUpload = clonedMapResults.outputUpload ?? {
|
|
73822
73826
|
url: uploadUrl,
|
|
73823
|
-
status: resp.status
|
|
73827
|
+
status: resp.status,
|
|
73828
|
+
etag
|
|
73824
73829
|
};
|
|
73825
73830
|
}
|
|
73826
73831
|
} catch (e4) {
|
|
@@ -73841,7 +73846,7 @@ async function curateOne({
|
|
|
73841
73846
|
try {
|
|
73842
73847
|
const prefix = outputTarget.s3.prefix ? outputTarget.s3.prefix.endsWith("/") ? outputTarget.s3.prefix : outputTarget.s3.prefix + "/" : "";
|
|
73843
73848
|
const key = prefix + clonedMapResults.outputFilePath;
|
|
73844
|
-
await client.send(
|
|
73849
|
+
const putResponse = await client.send(
|
|
73845
73850
|
new s32.PutObjectCommand({
|
|
73846
73851
|
Bucket: outputTarget.s3.bucketName,
|
|
73847
73852
|
Key: key,
|
|
@@ -73860,7 +73865,8 @@ async function curateOne({
|
|
|
73860
73865
|
const uploadUrl = `s3://${outputTarget.s3.bucketName}/${key}`;
|
|
73861
73866
|
clonedMapResults.outputUpload = {
|
|
73862
73867
|
url: uploadUrl,
|
|
73863
|
-
status: 200
|
|
73868
|
+
status: 200,
|
|
73869
|
+
etag: putResponse.ETag ?? void 0
|
|
73864
73870
|
};
|
|
73865
73871
|
} catch (e4) {
|
|
73866
73872
|
console.error("S3 Upload error", e4);
|
package/dist/esm/index.js
CHANGED
|
@@ -48613,14 +48613,14 @@ var require_dist_cjs70 = __commonJS({
|
|
|
48613
48613
|
const exitConditions = [runPolling(params, input, acceptorChecks)];
|
|
48614
48614
|
const finalize = [];
|
|
48615
48615
|
if (options.abortSignal) {
|
|
48616
|
-
const { aborted, clearListener } = abortTimeout(options.abortSignal);
|
|
48616
|
+
const { aborted: aborted2, clearListener } = abortTimeout(options.abortSignal);
|
|
48617
48617
|
finalize.push(clearListener);
|
|
48618
|
-
exitConditions.push(
|
|
48618
|
+
exitConditions.push(aborted2);
|
|
48619
48619
|
}
|
|
48620
48620
|
if (options.abortController?.signal) {
|
|
48621
|
-
const { aborted, clearListener } = abortTimeout(options.abortController.signal);
|
|
48621
|
+
const { aborted: aborted2, clearListener } = abortTimeout(options.abortController.signal);
|
|
48622
48622
|
finalize.push(clearListener);
|
|
48623
|
-
exitConditions.push(
|
|
48623
|
+
exitConditions.push(aborted2);
|
|
48624
48624
|
}
|
|
48625
48625
|
return Promise.race(exitConditions).then((result) => {
|
|
48626
48626
|
for (const fn of finalize) {
|
|
@@ -81486,7 +81486,7 @@ async function curateOne({
|
|
|
81486
81486
|
canSkip = previousSourceFileInfo.size === fileInfo.size && previousSourceFileInfo.mtime === mtime;
|
|
81487
81487
|
}
|
|
81488
81488
|
}
|
|
81489
|
-
const noMapResult = (outputFilePath) => {
|
|
81489
|
+
const noMapResult = (outputFilePath, knownPostMappedHash) => {
|
|
81490
81490
|
const retval = {
|
|
81491
81491
|
sourceInstanceUID: `unchanged_${fileInfo.name.replace(/[^a-zA-Z0-9]/g, "_")}`,
|
|
81492
81492
|
mappings: {},
|
|
@@ -81499,7 +81499,10 @@ async function curateOne({
|
|
|
81499
81499
|
size: fileInfo.size,
|
|
81500
81500
|
path: fileInfo.path,
|
|
81501
81501
|
mtime: previousSourceFileInfo?.mtime,
|
|
81502
|
-
preMappedHash
|
|
81502
|
+
preMappedHash,
|
|
81503
|
+
...knownPostMappedHash !== void 0 && {
|
|
81504
|
+
postMappedHash: knownPostMappedHash
|
|
81505
|
+
}
|
|
81503
81506
|
},
|
|
81504
81507
|
// include curationTime even when skipped to measure hashing/check time
|
|
81505
81508
|
curationTime: performance.now() - startTime,
|
|
@@ -81601,7 +81604,7 @@ async function curateOne({
|
|
|
81601
81604
|
fileArrayBuffer = null;
|
|
81602
81605
|
const previousPostMappedHash = previousMappedFileInfo ? (await previousMappedFileInfo(clonedMapResults.outputFilePath))?.postMappedHash : void 0;
|
|
81603
81606
|
if (previousPostMappedHash !== void 0 && previousPostMappedHash === postMappedHash) {
|
|
81604
|
-
return noMapResult(clonedMapResults.outputFilePath);
|
|
81607
|
+
return noMapResult(clonedMapResults.outputFilePath, postMappedHash);
|
|
81605
81608
|
}
|
|
81606
81609
|
if (typeof outputTarget?.directory === "object" && "getFileHandle" in outputTarget.directory) {
|
|
81607
81610
|
const subDirectoryHandle = await createNestedDirectories(
|
|
@@ -81669,9 +81672,11 @@ async function curateOne({
|
|
|
81669
81672
|
`Upload failed: ${resp.status} ${resp.statusText}`
|
|
81670
81673
|
);
|
|
81671
81674
|
} else {
|
|
81675
|
+
const etag = resp.headers.get("etag") ?? void 0;
|
|
81672
81676
|
clonedMapResults.outputUpload = clonedMapResults.outputUpload ?? {
|
|
81673
81677
|
url: uploadUrl,
|
|
81674
|
-
status: resp.status
|
|
81678
|
+
status: resp.status,
|
|
81679
|
+
etag
|
|
81675
81680
|
};
|
|
81676
81681
|
}
|
|
81677
81682
|
} catch (e4) {
|
|
@@ -81692,7 +81697,7 @@ async function curateOne({
|
|
|
81692
81697
|
try {
|
|
81693
81698
|
const prefix = outputTarget.s3.prefix ? outputTarget.s3.prefix.endsWith("/") ? outputTarget.s3.prefix : outputTarget.s3.prefix + "/" : "";
|
|
81694
81699
|
const key = prefix + clonedMapResults.outputFilePath;
|
|
81695
|
-
await client.send(
|
|
81700
|
+
const putResponse = await client.send(
|
|
81696
81701
|
new s32.PutObjectCommand({
|
|
81697
81702
|
Bucket: outputTarget.s3.bucketName,
|
|
81698
81703
|
Key: key,
|
|
@@ -81711,7 +81716,8 @@ async function curateOne({
|
|
|
81711
81716
|
const uploadUrl = `s3://${outputTarget.s3.bucketName}/${key}`;
|
|
81712
81717
|
clonedMapResults.outputUpload = {
|
|
81713
81718
|
url: uploadUrl,
|
|
81714
|
-
status: 200
|
|
81719
|
+
status: 200,
|
|
81720
|
+
etag: putResponse.ETag ?? void 0
|
|
81715
81721
|
};
|
|
81716
81722
|
} catch (e4) {
|
|
81717
81723
|
console.error("S3 Upload error", e4);
|
|
@@ -87477,6 +87483,7 @@ var filesMapped = 0;
|
|
|
87477
87483
|
var workerCurrentFile = /* @__PURE__ */ new Map();
|
|
87478
87484
|
var lastWorkerProgressTime = 0;
|
|
87479
87485
|
var pendingReplacements = 0;
|
|
87486
|
+
var aborted = false;
|
|
87480
87487
|
var currentFileInfoIndex;
|
|
87481
87488
|
var filesToProcess = [];
|
|
87482
87489
|
var directoryScanFinished = false;
|
|
@@ -87499,12 +87506,32 @@ function setScanResumeCallback(cb) {
|
|
|
87499
87506
|
function markScanPaused() {
|
|
87500
87507
|
scanPaused = true;
|
|
87501
87508
|
}
|
|
87509
|
+
function terminateAllWorkers() {
|
|
87510
|
+
aborted = true;
|
|
87511
|
+
while (availableMappingWorkers.length) {
|
|
87512
|
+
availableMappingWorkers.pop().terminate();
|
|
87513
|
+
}
|
|
87514
|
+
for (const [worker] of workerCurrentFile) {
|
|
87515
|
+
try {
|
|
87516
|
+
worker.terminate();
|
|
87517
|
+
} catch {
|
|
87518
|
+
}
|
|
87519
|
+
}
|
|
87520
|
+
workerCurrentFile.clear();
|
|
87521
|
+
filesToProcess.length = 0;
|
|
87522
|
+
workersActive = 0;
|
|
87523
|
+
pendingReplacements = 0;
|
|
87524
|
+
directoryScanFinished = false;
|
|
87525
|
+
scanPaused = false;
|
|
87526
|
+
scanResumeCallback = null;
|
|
87527
|
+
}
|
|
87502
87528
|
async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, progressCb, workerCount) {
|
|
87503
87529
|
mappingWorkerOptions = {};
|
|
87504
87530
|
workersActive = 0;
|
|
87505
87531
|
mapResultsList = skipCollectingMappings ? void 0 : [];
|
|
87506
87532
|
filesMapped = 0;
|
|
87507
87533
|
pendingReplacements = 0;
|
|
87534
|
+
aborted = false;
|
|
87508
87535
|
workerCurrentFile.clear();
|
|
87509
87536
|
lastWorkerProgressTime = Date.now();
|
|
87510
87537
|
currentFileInfoIndex = fileInfoIndex;
|
|
@@ -87520,6 +87547,8 @@ async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, p
|
|
|
87520
87547
|
availableMappingWorkers.push(...workers);
|
|
87521
87548
|
}
|
|
87522
87549
|
async function dispatchMappingJobs() {
|
|
87550
|
+
if (aborted)
|
|
87551
|
+
return;
|
|
87523
87552
|
while (filesToProcess.length > 0 && availableMappingWorkers.length > 0) {
|
|
87524
87553
|
const { fileInfo, previousFileInfo } = filesToProcess.pop();
|
|
87525
87554
|
const mappingWorker = availableMappingWorkers.pop();
|
|
@@ -87582,6 +87611,8 @@ async function getHardwareConcurrency() {
|
|
|
87582
87611
|
return cpus().length;
|
|
87583
87612
|
}
|
|
87584
87613
|
function recoverCrashedWorker(mappingWorker, errorMessage) {
|
|
87614
|
+
if (aborted)
|
|
87615
|
+
return;
|
|
87585
87616
|
if (!workerCurrentFile.has(mappingWorker)) {
|
|
87586
87617
|
return;
|
|
87587
87618
|
}
|
|
@@ -87617,6 +87648,10 @@ function recoverCrashedWorker(mappingWorker, errorMessage) {
|
|
|
87617
87648
|
pendingReplacements += 1;
|
|
87618
87649
|
void createMappingWorker().then((worker) => {
|
|
87619
87650
|
pendingReplacements -= 1;
|
|
87651
|
+
if (aborted) {
|
|
87652
|
+
worker.terminate();
|
|
87653
|
+
return;
|
|
87654
|
+
}
|
|
87620
87655
|
availableMappingWorkers.push(worker);
|
|
87621
87656
|
dispatchMappingJobs();
|
|
87622
87657
|
}).catch((error2) => {
|
|
@@ -87646,6 +87681,8 @@ async function createMappingWorker() {
|
|
|
87646
87681
|
});
|
|
87647
87682
|
}
|
|
87648
87683
|
mappingWorker.addEventListener("message", (event) => {
|
|
87684
|
+
if (aborted)
|
|
87685
|
+
return;
|
|
87649
87686
|
if (event.data.response === "lookup") {
|
|
87650
87687
|
const outputPath = event.data.outputPath;
|
|
87651
87688
|
const entry = currentFileInfoIndex?.[OUTPUT_FILE_PREFIX + outputPath];
|
|
@@ -87864,7 +87901,14 @@ function queueUrlsForMapping(organizeOptions) {
|
|
|
87864
87901
|
setDirectoryScanFinished(true);
|
|
87865
87902
|
}
|
|
87866
87903
|
async function curateMany(organizeOptions, onProgress) {
|
|
87904
|
+
if (organizeOptions.signal?.aborted) {
|
|
87905
|
+
return Promise.reject(
|
|
87906
|
+
new DOMException("The operation was aborted.", "AbortError")
|
|
87907
|
+
);
|
|
87908
|
+
}
|
|
87867
87909
|
return new Promise(async (resolve, reject) => {
|
|
87910
|
+
let settled = false;
|
|
87911
|
+
const signal = organizeOptions.signal;
|
|
87868
87912
|
const STALL_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
87869
87913
|
const stallWatchdog = setInterval(() => {
|
|
87870
87914
|
if (getWorkersActive() > 0 && Date.now() - getLastWorkerProgressTime() > STALL_TIMEOUT_MS) {
|
|
@@ -87883,15 +87927,42 @@ async function curateMany(organizeOptions, onProgress) {
|
|
|
87883
87927
|
}, 6e4);
|
|
87884
87928
|
const progressCallback2 = (msg) => {
|
|
87885
87929
|
onProgress?.(msg);
|
|
87886
|
-
if (msg.response === "done") {
|
|
87930
|
+
if (msg.response === "done" && !settled) {
|
|
87931
|
+
settled = true;
|
|
87887
87932
|
clearInterval(stallWatchdog);
|
|
87933
|
+
signal?.removeEventListener("abort", onAbort);
|
|
87888
87934
|
resolve(msg);
|
|
87889
87935
|
}
|
|
87890
87936
|
};
|
|
87891
87937
|
const rejectCallback = (reason) => {
|
|
87938
|
+
if (settled)
|
|
87939
|
+
return;
|
|
87940
|
+
settled = true;
|
|
87892
87941
|
clearInterval(stallWatchdog);
|
|
87942
|
+
signal?.removeEventListener("abort", onAbort);
|
|
87893
87943
|
reject(reason);
|
|
87894
87944
|
};
|
|
87945
|
+
let fileListWorker;
|
|
87946
|
+
const onAbort = () => {
|
|
87947
|
+
try {
|
|
87948
|
+
fileListWorker?.terminate();
|
|
87949
|
+
} catch {
|
|
87950
|
+
}
|
|
87951
|
+
terminateAllWorkers();
|
|
87952
|
+
rejectCallback(
|
|
87953
|
+
new DOMException("The operation was aborted.", "AbortError")
|
|
87954
|
+
);
|
|
87955
|
+
};
|
|
87956
|
+
if (signal) {
|
|
87957
|
+
if (signal.aborted) {
|
|
87958
|
+
clearInterval(stallWatchdog);
|
|
87959
|
+
rejectCallback(
|
|
87960
|
+
new DOMException("The operation was aborted.", "AbortError")
|
|
87961
|
+
);
|
|
87962
|
+
return;
|
|
87963
|
+
}
|
|
87964
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
87965
|
+
}
|
|
87895
87966
|
try {
|
|
87896
87967
|
await initializeMappingWorkers(
|
|
87897
87968
|
organizeOptions.skipCollectingMappings,
|
|
@@ -87903,7 +87974,7 @@ async function curateMany(organizeOptions, onProgress) {
|
|
|
87903
87974
|
await collectMappingOptions(organizeOptions)
|
|
87904
87975
|
);
|
|
87905
87976
|
if (organizeOptions.inputType === "directory" || organizeOptions.inputType === "path" || organizeOptions.inputType === "s3") {
|
|
87906
|
-
|
|
87977
|
+
fileListWorker = await initializeFileListWorker(rejectCallback);
|
|
87907
87978
|
setScanResumeCallback(() => {
|
|
87908
87979
|
fileListWorker.postMessage({ request: "resume" });
|
|
87909
87980
|
});
|
|
@@ -87960,7 +88031,7 @@ async function curateMany(organizeOptions, onProgress) {
|
|
|
87960
88031
|
}
|
|
87961
88032
|
dispatchMappingJobs();
|
|
87962
88033
|
} catch (error2) {
|
|
87963
|
-
|
|
88034
|
+
rejectCallback(error2);
|
|
87964
88035
|
}
|
|
87965
88036
|
});
|
|
87966
88037
|
}
|
|
@@ -12055,6 +12055,7 @@ var filesMapped = 0;
|
|
|
12055
12055
|
var workerCurrentFile = /* @__PURE__ */ new Map();
|
|
12056
12056
|
var lastWorkerProgressTime = 0;
|
|
12057
12057
|
var pendingReplacements = 0;
|
|
12058
|
+
var aborted = false;
|
|
12058
12059
|
var currentFileInfoIndex;
|
|
12059
12060
|
var filesToProcess = [];
|
|
12060
12061
|
var directoryScanFinished = false;
|
|
@@ -12077,12 +12078,35 @@ function setScanResumeCallback(cb) {
|
|
|
12077
12078
|
function markScanPaused() {
|
|
12078
12079
|
scanPaused = true;
|
|
12079
12080
|
}
|
|
12081
|
+
function terminateAllWorkers() {
|
|
12082
|
+
aborted = true;
|
|
12083
|
+
while (availableMappingWorkers.length) {
|
|
12084
|
+
availableMappingWorkers.pop().terminate();
|
|
12085
|
+
}
|
|
12086
|
+
for (const [worker] of workerCurrentFile) {
|
|
12087
|
+
try {
|
|
12088
|
+
worker.terminate();
|
|
12089
|
+
} catch {
|
|
12090
|
+
}
|
|
12091
|
+
}
|
|
12092
|
+
workerCurrentFile.clear();
|
|
12093
|
+
filesToProcess.length = 0;
|
|
12094
|
+
workersActive = 0;
|
|
12095
|
+
pendingReplacements = 0;
|
|
12096
|
+
directoryScanFinished = false;
|
|
12097
|
+
scanPaused = false;
|
|
12098
|
+
scanResumeCallback = null;
|
|
12099
|
+
}
|
|
12100
|
+
function isAborted() {
|
|
12101
|
+
return aborted;
|
|
12102
|
+
}
|
|
12080
12103
|
async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, progressCb, workerCount) {
|
|
12081
12104
|
mappingWorkerOptions = {};
|
|
12082
12105
|
workersActive = 0;
|
|
12083
12106
|
mapResultsList = skipCollectingMappings ? void 0 : [];
|
|
12084
12107
|
filesMapped = 0;
|
|
12085
12108
|
pendingReplacements = 0;
|
|
12109
|
+
aborted = false;
|
|
12086
12110
|
workerCurrentFile.clear();
|
|
12087
12111
|
lastWorkerProgressTime = Date.now();
|
|
12088
12112
|
currentFileInfoIndex = fileInfoIndex;
|
|
@@ -12098,6 +12122,8 @@ async function initializeMappingWorkers(skipCollectingMappings, fileInfoIndex, p
|
|
|
12098
12122
|
availableMappingWorkers.push(...workers);
|
|
12099
12123
|
}
|
|
12100
12124
|
async function dispatchMappingJobs() {
|
|
12125
|
+
if (aborted)
|
|
12126
|
+
return;
|
|
12101
12127
|
while (filesToProcess.length > 0 && availableMappingWorkers.length > 0) {
|
|
12102
12128
|
const { fileInfo, previousFileInfo } = filesToProcess.pop();
|
|
12103
12129
|
const mappingWorker = availableMappingWorkers.pop();
|
|
@@ -12160,6 +12186,8 @@ async function getHardwareConcurrency() {
|
|
|
12160
12186
|
return cpus().length;
|
|
12161
12187
|
}
|
|
12162
12188
|
function recoverCrashedWorker(mappingWorker, errorMessage) {
|
|
12189
|
+
if (aborted)
|
|
12190
|
+
return;
|
|
12163
12191
|
if (!workerCurrentFile.has(mappingWorker)) {
|
|
12164
12192
|
return;
|
|
12165
12193
|
}
|
|
@@ -12195,6 +12223,10 @@ function recoverCrashedWorker(mappingWorker, errorMessage) {
|
|
|
12195
12223
|
pendingReplacements += 1;
|
|
12196
12224
|
void createMappingWorker().then((worker) => {
|
|
12197
12225
|
pendingReplacements -= 1;
|
|
12226
|
+
if (aborted) {
|
|
12227
|
+
worker.terminate();
|
|
12228
|
+
return;
|
|
12229
|
+
}
|
|
12198
12230
|
availableMappingWorkers.push(worker);
|
|
12199
12231
|
dispatchMappingJobs();
|
|
12200
12232
|
}).catch((error) => {
|
|
@@ -12224,6 +12256,8 @@ async function createMappingWorker() {
|
|
|
12224
12256
|
});
|
|
12225
12257
|
}
|
|
12226
12258
|
mappingWorker.addEventListener("message", (event) => {
|
|
12259
|
+
if (aborted)
|
|
12260
|
+
return;
|
|
12227
12261
|
if (event.data.response === "lookup") {
|
|
12228
12262
|
const outputPath = event.data.outputPath;
|
|
12229
12263
|
const entry = currentFileInfoIndex?.[OUTPUT_FILE_PREFIX + outputPath];
|
|
@@ -12299,9 +12333,11 @@ export {
|
|
|
12299
12333
|
getWorkerCurrentFile,
|
|
12300
12334
|
getWorkersActive,
|
|
12301
12335
|
initializeMappingWorkers,
|
|
12336
|
+
isAborted,
|
|
12302
12337
|
markScanPaused,
|
|
12303
12338
|
scanAnomalies,
|
|
12304
12339
|
setDirectoryScanFinished,
|
|
12305
12340
|
setMappingWorkerOptions,
|
|
12306
|
-
setScanResumeCallback
|
|
12341
|
+
setScanResumeCallback,
|
|
12342
|
+
terminateAllWorkers
|
|
12307
12343
|
};
|
|
@@ -39,6 +39,18 @@ export declare function setScanResumeCallback(cb: (() => void) | null): void;
|
|
|
39
39
|
* index.ts when the queue exceeds the high-water mark.
|
|
40
40
|
*/
|
|
41
41
|
export declare function markScanPaused(): void;
|
|
42
|
+
/**
|
|
43
|
+
* Hard-terminate all workers (idle and active) and reset pool state.
|
|
44
|
+
* Called when curateMany is aborted via AbortSignal. Equivalent to a
|
|
45
|
+
* tab reload — partially written files are handled by hash checks on
|
|
46
|
+
* the next run.
|
|
47
|
+
*/
|
|
48
|
+
export declare function terminateAllWorkers(): void;
|
|
49
|
+
/**
|
|
50
|
+
* Whether the current run has been aborted. Used by worker message handlers
|
|
51
|
+
* to bail out on messages arriving after teardown.
|
|
52
|
+
*/
|
|
53
|
+
export declare function isAborted(): boolean;
|
|
42
54
|
/**
|
|
43
55
|
* Initialize the mapping worker pool. Call once per curateMany invocation.
|
|
44
56
|
*/
|
package/dist/types/types.d.ts
CHANGED
|
@@ -33,6 +33,7 @@ export type OrganizeOptions = {
|
|
|
33
33
|
fileInfoIndex?: TFileInfoIndex;
|
|
34
34
|
excludedPathGlobs?: string[];
|
|
35
35
|
workerCount?: number;
|
|
36
|
+
signal?: AbortSignal;
|
|
36
37
|
} & ({
|
|
37
38
|
inputType: 'directory';
|
|
38
39
|
inputDirectory: FileSystemDirectoryHandle;
|
|
@@ -141,6 +142,7 @@ export type TMapResults = {
|
|
|
141
142
|
outputUpload?: {
|
|
142
143
|
url: string;
|
|
143
144
|
status: number;
|
|
145
|
+
etag?: string;
|
|
144
146
|
};
|
|
145
147
|
mappingRequired?: boolean;
|
|
146
148
|
curationTime?: number;
|