dicom-synth 1.8.0 → 1.9.0

package/bin/dicom-synth-generate.mjs

@@ -2,11 +2,14 @@
 import { readFileSync, writeFileSync } from 'node:fs'
 import { resolve } from 'node:path'
 import {
-  generateCollectionFromSpec,
+  MAX_PIXEL_BYTES,
+  MAX_STREAM_BYTES,
+  previewCollection,
   resolveParametricSpec,
   validateDatasetSpec,
   withResolvedSeed,
   writeCollectionFromSpec,
+  writeLargeImageFile,
 } from '../dist/esm/index.js'
 
 function usage() {
@@ -14,13 +17,31 @@ function usage() {
     'Usage: dicom-synth-generate --schema <path> [--out <dir>] [--emit-spec <path>] [--dry-run]\n' +
       '       dicom-synth-generate --schema-inline <json> [--out <dir>] [--emit-spec <path>] [--dry-run]\n' +
       '       dicom-synth-generate --parametric <path> [--out <dir>] [--emit-spec <path>] [--dry-run]\n' +
+      '       dicom-synth-generate --large-file <path> --size <600mb|50gb> [--modality CT|PT|MR|CR]\n' +
       '\n' +
       'A seed is drawn when the spec omits one; pass --emit-spec to capture the\n' +
-      'effective spec (seed included) so the dataset can be re-created.',
+      'effective spec (seed included) so the dataset can be re-created.\n' +
+      '\n' +
+      '--large-file streams a single DICOM (512 MB–50 GB) to disk; files above\n' +
+      '~4 GB use encapsulated fragments (valid container, zero pixel content).',
   )
   process.exit(1)
 }
 
+const SIZE_UNITS = { b: 1, kb: 1024, mb: 1024 * 1024, gb: 1024 * 1024 * 1024 }
+
+function parseSizeToBytes(s) {
+  // Unit is required so a bare number (e.g. "600", meant as MB) is not silently
+  // read as 600 bytes.
+  const m = /^([0-9]*\.?[0-9]+)\s*(b|kb|mb|gb)$/i.exec(s.trim())
+  if (!m) {
+    throw new Error(
+      `invalid --size "${s}" — include a unit, e.g. 600mb or 50gb`,
+    )
+  }
+  return Math.round(Number.parseFloat(m[1]) * SIZE_UNITS[m[2].toLowerCase()])
+}
+
 const args = process.argv.slice(2)
 
 if (args.length === 0) usage()
@@ -30,6 +51,9 @@ let specSource
 let outDir = resolve('./fixtures/generated')
 let emitSpecPath
 let dryRun = false
+let largeFilePath
+let largeFileSize
+let largeFileModality
 
 function readSpecArg(flag, value) {
   if (rawSpec !== undefined) {
@@ -70,12 +94,73 @@ for (let i = 0; i < args.length; i++) {
     emitSpecPath = resolve(args[++i])
   } else if (args[i] === '--dry-run') {
     dryRun = true
+  } else if (args[i] === '--large-file' && args[i + 1]) {
+    largeFilePath = resolve(args[++i])
+  } else if (args[i] === '--size' && args[i + 1]) {
+    largeFileSize = args[++i]
+  } else if (args[i] === '--modality' && args[i + 1]) {
+    largeFileModality = args[++i]
   } else {
     console.error(`Unknown argument: ${args[i]}`)
     usage()
   }
 }
 
+if (largeFilePath !== undefined) {
+  if (rawSpec !== undefined) {
+    console.error('Error: --large-file cannot be combined with a spec mode')
+    process.exit(1)
+  }
+  if (largeFileSize === undefined) {
+    console.error('Error: --large-file requires --size')
+    usage()
+  }
+  if (
+    largeFileModality !== undefined &&
+    !['CT', 'PT', 'MR', 'CR'].includes(largeFileModality)
+  ) {
+    console.error(`Error: invalid --modality "${largeFileModality}"`)
+    process.exit(1)
+  }
+  let targetBytes
+  try {
+    targetBytes = parseSizeToBytes(largeFileSize)
+  } catch (err) {
+    console.error(`Error: ${err.message}`)
+    process.exit(1)
+  }
+  // Validate bounds here so --dry-run reflects what a real run would accept.
+  if (targetBytes <= MAX_PIXEL_BYTES) {
+    console.error(
+      'Error: --size must exceed the 512 MB in-memory limit — use a spec with targetSizeKb for smaller files',
+    )
+    process.exit(1)
+  }
+  if (targetBytes > MAX_STREAM_BYTES) {
+    console.error('Error: --size exceeds the 50 GB limit')
+    process.exit(1)
+  }
+  if (dryRun) {
+    console.log(
+      `Dry run: would write ~${(targetBytes / 1024 / 1024).toFixed(1)} MB to ${largeFilePath} — nothing written`,
+    )
+    process.exit(0)
+  }
+  try {
+    const result = writeLargeImageFile(
+      { targetBytes, modality: largeFileModality },
+      largeFilePath,
+    )
+    console.log(
+      `Wrote ${(result.bytesWritten / 1024 / 1024).toFixed(1)} MB ${result.encoding} DICOM to ${result.path}`,
+    )
+  } catch (err) {
+    console.error(`Error writing large file: ${err.message}`)
+    process.exit(1)
+  }
+  process.exit(0)
+}
+
 if (rawSpec === undefined) usage()
 
 function printTypeCounts(counts) {
@@ -120,13 +205,13 @@ if (dryRun) {
   let fileCount = 0
   let byteCount = 0
   try {
-    for await (const file of generateCollectionFromSpec(spec)) {
+    for await (const file of previewCollection(spec)) {
       console.log(
-        `${file.relativePath}  (${file.type}, ${(file.buffer.length / 1024).toFixed(1)} KB)`,
+        `${file.relativePath}  (${file.type}, ${(file.approxBytes / 1024).toFixed(1)} KB)`,
       )
       counts[file.type] = (counts[file.type] ?? 0) + 1
       fileCount++
-      byteCount += file.buffer.length
+      byteCount += file.approxBytes
     }
   } catch (err) {
     console.error(`Error generating collection: ${err.message}`)

package/dist/esm/collection/writer.js

@@ -1,8 +1,12 @@
 // src/collection/writer.ts
 import { randomBytes as randomBytes2 } from "node:crypto";
-import { mkdirSync as mkdirSync2 } from "node:fs";
+import { mkdirSync as mkdirSync3 } from "node:fs";
 import { writeFile } from "node:fs/promises";
-import { dirname, resolve as resolve2 } from "node:path";
+import { dirname, resolve as resolve3 } from "node:path";
+
+// src/schema/limits.ts
+var MAX_PIXEL_BYTES = 512 * 1024 * 1024;
+var MAX_STREAM_BYTES = 50 * 1024 * 1024 * 1024;
 
 // src/loadDcmjs.ts
 import dcmjsDefaultImport, * as dcmjsNamespace from "dcmjs";
@@ -46,7 +50,6 @@ function buildDicomdirBuffer() {
 }
 
 // src/schema/validate.ts
-var MAX_PIXEL_BYTES = 512 * 1024 * 1024;
 var HEX_TAG_RE = /^[0-9a-fA-F]{8}$/;
 
 // src/syntheticFixtures/generator.ts
@@ -220,9 +223,17 @@ function buildBufferForSpec(spec, uid) {
       return buildNonDicomBuffer(spec.content);
     case "dicomdir":
       return buildDicomdirBuffer();
+    case "large-image":
+      throw new Error(
+        "large-image cannot be generated in memory \u2014 use writeCollectionFromSpec (disk-only streaming)"
+      );
   }
 }
 
+// src/syntheticFixtures/streamWrite.ts
+import { closeSync, mkdirSync as mkdirSync2, openSync, writeSync } from "node:fs";
+import { resolve as resolve2 } from "node:path";
+
 // src/syntheticFixtures/uid.ts
 import { randomBytes } from "node:crypto";
 var ROLE_CODE = { study: 1, series: 2, sop: 3 };
@@ -288,6 +299,153 @@ function makeUidGenerator(seed) {
   };
 }
 
+// src/syntheticFixtures/streamWrite.ts
+var OW_MAX_BYTES = 4294967294;
+var FRAGMENT_MAX_BYTES = 4294967294;
+var DEFAULT_CHUNK_BYTES = 8 * 1024 * 1024;
+var RLE_TRANSFER_SYNTAX_UID = "1.2.840.10008.1.2.5";
+var PIXEL_DATA_OW_HEADER = Buffer.from([224, 127, 16, 0, 79, 87]);
+var PIXEL_DATA_OB_HEADER = Buffer.from([224, 127, 16, 0, 79, 66]);
+var ITEM_TAG_GROUP = 65534;
+var ITEM_TAG_ELEMENT = 57344;
+var SEQUENCE_DELIMITER_ELEMENT = 57565;
+function itemTag(element, byteLength = 0) {
+  const b = Buffer.alloc(8);
+  b.writeUInt16LE(ITEM_TAG_GROUP, 0);
+  b.writeUInt16LE(element, 2);
+  b.writeUInt32LE(byteLength, 4);
+  return b;
+}
+function nativeDimensions(pixelBytes) {
+  const cells = Math.max(1, Math.floor(pixelBytes / 2));
+  const rows = Math.min(65535, Math.max(1, Math.round(Math.sqrt(cells))));
+  const columns = Math.min(65535, Math.max(1, Math.floor(cells / rows)));
+  return { rows, columns, bytes: rows * columns * 2 };
+}
+function streamZeros(fd, totalBytes, zeroChunk) {
+  let remaining = totalBytes;
+  while (remaining > 0) {
+    const n = Math.min(zeroChunk.length, remaining);
+    writeSync(fd, zeroChunk, 0, n);
+    remaining -= n;
+  }
+}
+function writeNative(outPath, params, dims, zeroChunk) {
+  const { modality, uid, tags } = params;
+  const meta = buildMeta(uid, modality);
+  const dataset = buildBaseImageDataset(uid, modality);
+  applyTagOverrides(dataset, tags);
+  dataset.Rows = dims.rows;
+  dataset.Columns = dims.columns;
+  const minimalPixelBytes = dataset.PixelData.byteLength;
+  const probe = serializeDict(meta, dataset);
+  const pixelElement = Buffer.alloc(12);
+  PIXEL_DATA_OW_HEADER.copy(pixelElement);
+  pixelElement.writeUInt32LE(minimalPixelBytes, 8);
+  const owIdx = probe.indexOf(pixelElement);
+  if (owIdx < 0) {
+    throw new Error("failed to locate native PixelData element header");
+  }
+  const dataStart = owIdx + 12;
+  const header = Buffer.from(probe.subarray(0, dataStart));
+  header.writeUInt32LE(dims.bytes, owIdx + 8);
+  const trailing = probe.subarray(dataStart + minimalPixelBytes);
+  const fd = openSync(outPath, "w");
+  try {
+    writeSync(fd, header);
+    streamZeros(fd, dims.bytes, zeroChunk);
+    if (trailing.length > 0) writeSync(fd, trailing);
+  } finally {
+    closeSync(fd);
+  }
+  return header.length + dims.bytes + trailing.length;
+}
+function writeEncapsulated(outPath, params, pixelBytes, fragmentBytes, zeroChunk) {
+  const { modality, uid, tags } = params;
+  const meta = {
+    ...buildMeta(uid, modality),
+    TransferSyntaxUID: RLE_TRANSFER_SYNTAX_UID
+  };
+  const dataset = buildBaseImageDataset(uid, modality);
+  applyTagOverrides(dataset, tags);
+  dataset.PixelData = [new Uint8Array([]).buffer, new Uint8Array([0, 0]).buffer];
+  dataset._vrMap = { PixelData: "OB" };
+  const probe = serializeDict(meta, dataset);
+  const idx = probe.indexOf(PIXEL_DATA_OB_HEADER);
+  if (idx < 0 || probe.readUInt32LE(idx + 8) !== 4294967295) {
+    throw new Error("failed to locate encapsulated PixelData element header");
+  }
+  const prefix = Buffer.from(probe.subarray(0, idx + 12));
+  const fd = openSync(outPath, "w");
+  let bytesWritten = 0;
+  try {
+    writeSync(fd, prefix);
+    bytesWritten += prefix.length;
+    const bot = itemTag(ITEM_TAG_ELEMENT, 0);
+    writeSync(fd, bot);
+    bytesWritten += bot.length;
+    let remaining = pixelBytes;
+    while (remaining > 0) {
+      const fragLen = Math.min(fragmentBytes, remaining);
+      const fh = itemTag(ITEM_TAG_ELEMENT, fragLen);
+      writeSync(fd, fh);
+      streamZeros(fd, fragLen, zeroChunk);
+      bytesWritten += fh.length + fragLen;
+      remaining -= fragLen;
+    }
+    const delimiter = itemTag(SEQUENCE_DELIMITER_ELEMENT);
+    writeSync(fd, delimiter);
+    bytesWritten += delimiter.length;
+  } finally {
+    closeSync(fd);
+  }
+  return bytesWritten;
+}
+function streamLargeImage(outPath, options) {
+  const {
+    targetBytes,
+    chunkBytes = DEFAULT_CHUNK_BYTES,
+    owMaxBytes = OW_MAX_BYTES,
+    fragmentBytes = FRAGMENT_MAX_BYTES
+  } = options;
+  const modality = options.modality ?? "CT";
+  const uid = options.uid ?? makeUidGenerator(options.seed)(0);
+  const identity = { modality, uid, tags: options.tags };
+  mkdirSync2(resolve2(outPath, ".."), { recursive: true });
+  const zeroChunk = Buffer.alloc(Math.max(1, chunkBytes));
+  const probe = serializeDict(
+    buildMeta(uid, modality),
+    buildBaseImageDataset(uid, modality)
+  );
+  const nativeHeaderLen = probe.length - 2;
+  const requestedPixel = Math.max(2, targetBytes - nativeHeaderLen);
+  if (requestedPixel <= owMaxBytes) {
+    const dims = nativeDimensions(requestedPixel);
+    const bytesWritten2 = writeNative(outPath, identity, dims, zeroChunk);
+    return {
+      path: outPath,
+      bytesWritten: bytesWritten2,
+      encoding: "native",
+      pixelBytes: dims.bytes
+    };
+  }
+  const estimateFrags = Math.max(
+    1,
+    Math.ceil((targetBytes - nativeHeaderLen - 16) / fragmentBytes)
+  );
+  const overhead = nativeHeaderLen + 8 + estimateFrags * 8 + 8;
+  let pixelBytes = Math.max(2, targetBytes - overhead);
+  pixelBytes -= pixelBytes % 2;
+  const bytesWritten = writeEncapsulated(
+    outPath,
+    identity,
+    pixelBytes,
+    fragmentBytes - fragmentBytes % 2,
+    zeroChunk
+  );
+  return { path: outPath, bytesWritten, encoding: "encapsulated", pixelBytes };
+}
+
 // src/syntheticFixtures/violations.ts
 var dcmjsAny = dcmjs;
 function parseBuffer(buffer) {
@@ -401,11 +559,13 @@ function studyFileCount(studies) {
     0
   );
 }
+function resolveUid(seed, index, override) {
+  return { ...makeUidGenerator(seed)(index), ...override };
+}
 async function generateFile(spec, options) {
   const index = options?.index ?? 0;
   const padWidth = options?.padWidth ?? 3;
-  const uidGen = makeUidGenerator(options?.seed);
-  const uid = { ...uidGen(index), ...options?.uid };
+  const uid = resolveUid(options?.seed, index, options?.uid);
   let buffer = buildBufferForSpec(spec, uid);
   const violations = "violations" in spec ? spec.violations : void 0;
   if (violations?.length) {
@@ -473,25 +633,42 @@ function hierarchicalName(studyOrdinal, seriesIndex, instanceNumber) {
 function withResolvedSeed(spec) {
   return spec.seed === void 0 ? { ...spec, seed: randomBytes2(4).readUInt32BE(0) } : spec;
 }
-async function* generateCollectionFromSpec(spec) {
+function computeNames(type, index, padWidth, grouped, quirks) {
+  let names;
+  if (grouped) {
+    names = hierarchicalName(
+      grouped.studyOrdinal,
+      grouped.seriesIndex,
+      grouped.instanceNumber
+    );
+  } else {
+    const name = filename(type, index, padWidth);
+    names = { filename: name, relativePath: name };
+  }
+  return quirks.length === 0 ? names : applyPathQuirks(names.relativePath, quirks);
+}
+function* planCollection(spec) {
   const flatEntries = spec.entries ?? [];
   const studies = spec.studies ?? [];
   const totalFiles = entryCount(flatEntries) + studyFileCount(studies);
   const padWidth = String(totalFiles).length;
   const hierarchical = spec.layout === "hierarchical";
   const quirks = spec.pathQuirks ?? [];
-  const decorate = (file) => quirks.length === 0 ? file : { ...file, ...applyPathQuirks(file.relativePath, quirks) };
   let globalIndex = 0;
   for (const entry of flatEntries) {
     const { count = 1, ...fileSpec } = entry;
     for (let i = 0; i < count; i++) {
-      yield decorate(
-        await generateFile(fileSpec, {
-          index: globalIndex,
-          seed: spec.seed,
-          padWidth
-        })
-      );
+      yield {
+        fileSpec,
+        index: globalIndex,
+        ...computeNames(
+          fileSpec.type,
+          globalIndex,
+          padWidth,
+          void 0,
+          quirks
+        )
+      };
       globalIndex++;
     }
   }
@@ -514,25 +691,18 @@ async function* generateCollectionFromSpec(spec) {
               ...series.tags,
               ...fileSpec.tags
             };
-            const file = await generateFile(
-              { ...fileSpec, tags },
-              {
-                index: globalIndex,
-                seed: spec.seed,
+            yield {
+              fileSpec: { ...fileSpec, tags },
+              index: globalIndex,
+              uidOverride: { study: studyUid, series: seriesUid },
+              ...computeNames(
+                fileSpec.type,
+                globalIndex,
                 padWidth,
-                uid: { study: studyUid, series: seriesUid }
-              }
-            );
-            yield decorate(
-              hierarchical ? {
-                ...file,
-                ...hierarchicalName(
-                  studyOrdinal,
-                  seriesIndex,
-                  instanceNumber
-                )
-              } : file
-            );
+                hierarchical ? { studyOrdinal, seriesIndex, instanceNumber } : void 0,
+                quirks
+              )
+            };
             globalIndex++;
           }
         }
@@ -541,26 +711,84 @@ async function* generateCollectionFromSpec(spec) {
     }
   }
 }
+async function materialisePlan(plan, seed) {
+  const file = await generateFile(plan.fileSpec, {
+    index: plan.index,
+    seed,
+    uid: plan.uidOverride
+  });
+  return { ...file, filename: plan.filename, relativePath: plan.relativePath };
+}
+async function* generateCollectionFromSpec(spec) {
+  for (const plan of planCollection(spec)) {
+    yield await materialisePlan(plan, spec.seed);
+  }
+}
+async function* previewCollection(spec) {
+  for (const plan of planCollection(spec)) {
+    if (plan.fileSpec.type === "large-image") {
+      yield {
+        relativePath: plan.relativePath,
+        type: "large-image",
+        index: plan.index,
+        approxBytes: plan.fileSpec.targetBytes
+      };
+    } else {
+      const file = await materialisePlan(plan, spec.seed);
+      yield {
+        relativePath: plan.relativePath,
+        type: plan.fileSpec.type,
+        index: plan.index,
+        approxBytes: file.buffer.length
+      };
+    }
+  }
+}
 async function writeCollectionFromSpec(spec, outDir) {
-  const root = resolve2(outDir);
-  mkdirSync2(root, { recursive: true });
+  const root = resolve3(outDir);
+  mkdirSync3(root, { recursive: true });
   const manifest = [];
   const createdDirs = /* @__PURE__ */ new Set([root]);
-  for await (const file of generateCollectionFromSpec(spec)) {
-    const filePath = resolve2(root, file.relativePath);
+  const ensureDir = (filePath) => {
     const dir = dirname(filePath);
     if (!createdDirs.has(dir)) {
-      mkdirSync2(dir, { recursive: true });
+      mkdirSync3(dir, { recursive: true });
       createdDirs.add(dir);
     }
-    await writeFile(filePath, file.buffer);
-    manifest.push({ path: filePath, type: file.type, index: file.index });
+  };
+  for (const plan of planCollection(spec)) {
+    const filePath = resolve3(root, plan.relativePath);
+    ensureDir(filePath);
+    if (plan.fileSpec.type === "large-image") {
+      const large = plan.fileSpec;
+      if (large.targetBytes > MAX_STREAM_BYTES) {
+        throw new Error(
+          `large-image targetBytes (${large.targetBytes}) exceeds the 50 GB limit`
+        );
+      }
+      const uid = resolveUid(spec.seed, plan.index, plan.uidOverride);
+      streamLargeImage(filePath, {
+        targetBytes: large.targetBytes,
+        modality: large.modality,
+        uid,
+        tags: large.tags
+      });
+    } else {
+      const file = await materialisePlan(plan, spec.seed);
+      await writeFile(filePath, file.buffer);
+    }
+    manifest.push({
+      path: filePath,
+      type: plan.fileSpec.type,
+      index: plan.index
+    });
   }
   return manifest;
 }
 export {
   generateCollectionFromSpec,
   generateFile,
+  previewCollection,
   withResolvedSeed,
   writeCollectionFromSpec
 };