@patrickdk77/archive-defuse 0.1.1 → 0.1.3

package/dist/pack.js

@@ -4,6 +4,7 @@ exports.PackError = void 0;
 exports.createArchive = createArchive;
 exports.createArchiveStream = createArchiveStream;
 exports.createArchiveBytes = createArchiveBytes;
+exports.createArchiveWriter = createArchiveWriter;
 exports.compressTo = compressTo;
 exports.compressStream = compressStream;
 exports.compress = compress;
@@ -119,6 +120,12 @@ function settle(entry, options, now) {
     const kind = entry.kind ?? (raw.endsWith('/') ? 'directory' : entry.linkTarget !== undefined ? 'symlink' : 'file');
     if (kind === 'symlink' && entry.linkTarget === undefined)
         throw new PackError(`Symbolic link ${JSON.stringify(raw)} needs a linkTarget`);
+    if (entry.size !== undefined) {
+        if (kind !== 'file')
+            throw new PackError(`Entry ${JSON.stringify(raw)} is a ${kind}, and only a file takes a size`);
+        if (!Number.isSafeInteger(entry.size) || entry.size < 0)
+            throw new PackError(`Entry ${JSON.stringify(raw)} has a size of ${String(entry.size)}, which is not a whole number of bytes`);
+    }
     const name = kind === 'directory' ? (raw.endsWith('/') ? raw : `${raw}/`) : raw.replace(/\/+$/, '');
     for (const detail of (0, rules_1.nameDetails)(name, true)) {
         const why = REFUSED[detail];
@@ -135,21 +142,51 @@ function settle(entry, options, now) {
     }
     return { name, kind, entry, mode: (entry.mode ?? (kind === 'directory' ? 0o755 : kind === 'symlink' ? 0o777 : 0o644)) & 0o7777, mtime: entry.mtime ?? options.mtime ?? now };
 }
-/** The content of a settled entry: a file's data, a link's target, or nothing for a folder. */
+/** `chunks`, failing as soon as they pass `size` bytes, or when they end short of it. */
+async function* sized(chunks, size, name) {
+    let total = 0;
+    for await (const c of chunks) {
+        total += c.length;
+        if (total > size)
+            throw new Error(`Entry ${JSON.stringify(name)} has more than the ${size} bytes its size gives`);
+        yield c;
+    }
+    if (total !== size)
+        throw new Error(`Entry ${JSON.stringify(name)} has ${total} bytes, not the ${size} its size gives`);
+}
+/**
+ * The content of a settled entry: a file's data, a link's target, or nothing for a folder. A file's size is known when
+ * the entry gives one or its content is bytes or text. Content given as bytes or text of another size than the entry
+ * gives throws a PackError here, before the entry is written; a stream of another size fails as it is read.
+ */
 async function contentOf(s) {
     if (s.kind === 'directory')
-        return chunksOf(undefined);
-    if (s.kind === 'symlink')
-        return chunksOf(s.entry.linkTarget);
-    const d = s.entry.data;
-    return chunksOf(typeof d === 'function' ? await d() : d);
+        return { chunks: chunksOf(undefined), size: 0 };
+    if (s.kind === 'symlink') {
+        const target = toBytes(s.entry.linkTarget ?? '');
+        return { chunks: chunksOf(target), size: target.length };
+    }
+    const given = s.entry.data;
+    const raw = typeof given === 'function' ? await given() : given;
+    const data = typeof raw === 'string' ? toBytes(raw) : raw;
+    const own = data === undefined ? 0 : data instanceof Uint8Array ? data.length : undefined;
+    const size = s.entry.size;
+    if (size === undefined)
+        return { chunks: chunksOf(data), size: own };
+    if (own !== undefined && own !== size)
+        throw new PackError(`Entry ${JSON.stringify(s.name)} holds ${own} bytes, not the ${size} its size gives`);
+    return { chunks: sized(chunksOf(data), size, s.name), size };
 }
 /** Reads an entry's content into memory or a temporary file, counting its CRC-32. */
 async function spillOf(s, temp, threshold, check) {
+    return spillChunks((await contentOf(s)).chunks, temp, threshold, check);
+}
+/** Reads chunks into memory or a temporary file, counting their CRC-32. */
+async function spillChunks(chunks, temp, threshold, check) {
     const spill = new io_1.SpillSink(temp, threshold);
     let crc = 0;
     try {
-        for await (const c of checked(await contentOf(s), check)) {
+        for await (const c of checked(chunks, check)) {
             crc = (0, crc32_1.crc32)(c, crc);
             await spill.write(c);
         }
@@ -236,7 +273,8 @@ async function writeZipEntries(entries, sink, options, temp, check) {
  * second, hidden entry gets past a reader of the central directory.
  */
 async function streamZipEntry(out, s, info, level, check) {
-    const it = checked(await contentOf(s), check)[Symbol.asyncIterator]();
+    const content = await contentOf(s);
+    const it = checked(content.chunks, check)[Symbol.asyncIterator]();
     let first;
     try {
         first = await it.next();
@@ -277,7 +315,9 @@ async function streamZipEntry(out, s, info, level, check) {
         }
     };
     Object.assign(info, { method: 8, versionNeeded: 20 });
-    await out.streamEntry(info, (0, codecs_1.deflateRaw)(raw(), level), () => ({ crc, size }));
+    // zlib's deflateBound for any settings: deflate never grows data by more than this.
+    const zip64 = content.size !== undefined && content.size + Math.ceil(content.size / 8) + Math.ceil(content.size / 64) + 5 >= 0xffffffff;
+    await out.streamEntry(info, (0, codecs_1.deflateRaw)(raw(), level), () => ({ crc, size }), zip64);
 }
 // ---------------------------------------------------------------------------------------------------------------
 // tar
@@ -287,9 +327,25 @@ async function* tarChunks(entries, options, temp, check) {
     let written = 0;
     const pad = (n) => Buffer.alloc((tar_1.BLOCK - (n % tar_1.BLOCK)) % tar_1.BLOCK);
     for await (const s of entries) {
-        const { spill } = s.kind === 'file' ? await spillOf(s, temp, threshold, check) : { spill: undefined };
+        let spill;
+        let direct;
+        let first;
+        let size = 0;
         try {
-            const size = spill?.length ?? 0;
+            if (s.kind === 'file') {
+                const content = await contentOf(s);
+                if (content.size === undefined) {
+                    ({ spill } = await spillChunks(content.chunks, temp, threshold, check));
+                    size = spill.length;
+                }
+                else {
+                    // The header can go first and the data straight after it. The first chunk is read before the header, so a
+                    // stream that fails at once fails before its entry is begun, and a stream is open to close should the work stop.
+                    size = content.size;
+                    direct = checked(content.chunks, check)[Symbol.asyncIterator]();
+                    first = await direct.next();
+                }
+            }
             const link = s.kind === 'symlink' ? (s.entry.linkTarget ?? '') : '';
             const mtime = Math.floor(s.mtime.getTime() / 1000);
             // What the ustar header cannot hold goes in a pax header before it.
@@ -344,8 +400,13 @@ async function* tarChunks(entries, options, temp, check) {
             });
             yield h;
             written += h.length;
-            if (spill) {
+            if (spill)
                 yield* checked(spill.read(), check);
+            if (direct && first) {
+                for (let r = first; !r.done; r = await direct.next())
+                    yield r.value;
+            }
+            if (spill || direct) {
                 const p = pad(size);
                 if (p.length)
                     yield p;
@@ -353,6 +414,7 @@ async function* tarChunks(entries, options, temp, check) {
             }
         }
         finally {
+            await direct?.return?.(undefined);
             await spill?.dispose();
         }
     }
@@ -476,6 +538,15 @@ function compressChunks(format, input, options, temp, now, check) {
 // The calls
 const ARCHIVE_FORMATS = new Set(['zip', 'tar', 'tar.gz', 'tar.bz2', 'tar.zst', 'tar.xz', '7z']);
 const TAR_COMPRESSION = { 'tar.gz': 'gzip', 'tar.bz2': 'bzip2', 'tar.zst': 'zstd', 'tar.xz': 'xz' };
+/** Throws for a format `createArchive` does not write, and for `streaming` with any format but ZIP. */
+function checkFormat(format, options) {
+    if (!ARCHIVE_FORMATS.has(format))
+        throw new PackError(`Unknown archive format ${String(format)}`);
+    // tar writes a file of known size as it arrives without the option, and needs the size of any other first. 7z writes
+    // its header from every entry after the data.
+    if (options.streaming && format !== 'zip')
+        throw new PackError(`streaming writes ZIP archives only, not ${format}`);
+}
 /** Copies chunks to the sink, then closes it. On failure it aborts the sink, so a partial output never looks whole. */
 async function deliver(produce, sink) {
     try {
@@ -498,11 +569,7 @@ async function deliver(produce, sink) {
  * sink is aborted instead of closed and the error is thrown.
  */
 async function createArchive(format, entries, sink, options = {}) {
-    if (!ARCHIVE_FORMATS.has(format))
-        throw new PackError(`Unknown archive format ${String(format)}`);
-    // tar needs each entry's size before its data, and 7z writes its header from every entry after the data.
-    if (options.streaming && format !== 'zip')
-        throw new PackError(`streaming writes ZIP archives only, not ${format}`);
+    checkFormat(format, options);
     const now = new Date();
     const temp = new io_1.TempDir(options.tempDir);
     const check = () => options.signal?.throwIfAborted();
@@ -563,6 +630,95 @@ async function createArchiveBytes(format, entries, options = {}) {
     await createArchive(format, entries, sink, options);
     return sink.result();
 }
+/**
+ * `createArchiveStream` for a caller that finds its entries as it goes. An entry `createArchive` refuses, or data that
+ * fails, fails the archive: `finish` rejects with the error and `stream` is destroyed with it.
+ */
+function createArchiveWriter(format, options = {}) {
+    checkFormat(format, options);
+    const queue = [];
+    let head = 0;
+    let closed;
+    let failure;
+    let wake;
+    const drop = () => {
+        queue.length = 0;
+        head = 0;
+    };
+    const stop = new AbortController();
+    const outer = options.signal;
+    const forward = () => stop.abort(outer?.reason);
+    if (outer?.aborted)
+        forward();
+    else
+        outer?.addEventListener('abort', forward, { once: true });
+    // The loop waits here for the next entry. The wait ends when the work stops, so a reader that goes away or an abort
+    // does not leave it waiting for an entry that never comes.
+    const nextEntry = (signal) => new Promise((resolve, reject) => {
+        const stopped = () => {
+            wake = undefined;
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', stopped, { once: true });
+        wake = () => {
+            signal.removeEventListener('abort', stopped);
+            wake = undefined;
+            resolve();
+        };
+    });
+    async function* entries(signal) {
+        for (;;) {
+            if (head < queue.length) {
+                const e = queue[head];
+                queue[head++] = undefined;
+                if (head === queue.length)
+                    drop();
+                if (e)
+                    yield e;
+                continue;
+            }
+            if (closed === 'finished')
+                return;
+            signal.throwIfAborted();
+            await nextEntry(signal);
+        }
+    }
+    let done = Promise.resolve();
+    const stream = readableOf((sink, signal) => {
+        done = createArchive(format, entries(signal), sink, { ...options, signal });
+        return done;
+    }, stop.signal);
+    done.then(() => outer?.removeEventListener('abort', forward), error => {
+        failure ??= { error };
+        drop();
+        outer?.removeEventListener('abort', forward);
+    });
+    return {
+        stream,
+        add(entry) {
+            if (closed)
+                throw new PackError(`An entry was added after the archive was ${closed}`);
+            if (failure)
+                throw new PackError('An entry was added after the archive failed', { cause: failure.error });
+            queue.push(entry);
+            wake?.();
+        },
+        finish() {
+            if (!closed) {
+                closed = 'finished';
+                wake?.();
+            }
+            return done;
+        },
+        abort(reason) {
+            if (closed === 'aborted')
+                return;
+            closed = 'aborted';
+            drop();
+            stop.abort(reason);
+        },
+    };
+}
 /** Compresses `data` as one stream of `format` to `sink`, then closes it. */
 async function compressTo(format, data, sink, options = {}) {
     const temp = new io_1.TempDir(options.tempDir);

package/dist/zip-writer.d.ts

@@ -37,13 +37,17 @@ export declare class ZipOutput {
     /**
      * Writes one entry whose sizes are known only once its data has gone out: a local header with the data descriptor
      * flag and no CRC or sizes, then `chunks`, the compressed data, then a data descriptor with the CRC and the sizes.
-     * `totals` gives the CRC and size of the uncompressed data once `chunks` has ended. An entry of 4 GiB or more gets
-     * eight-byte sizes in its descriptor and a ZIP64 field in the central directory, as archiver writes one.
+     * `totals` gives the CRC and size of the uncompressed data once `chunks` has ended.
+     *
+     * With `zip64`, the local header has a ZIP64 field, which tells a reader the descriptor's sizes are eight bytes. Its
+     * sizes are 0xFFFFFFFF and the field's are zero, as Python's zipfile writes a streamed entry. Without it, an entry of
+     * 4 GiB or more still gets eight-byte sizes in its descriptor, as archiver writes one. Either way, the central
+     * directory has a ZIP64 field only when a size or the offset needs one.
      */
     streamEntry(info: Written, chunks: AsyncIterable<Uint8Array>, totals: () => {
         crc: number;
         size: number;
-    }): Promise<void>;
+    }, zip64?: boolean): Promise<void>;
     /** Writes the central directory and the end records, with the archive's comment. */
     end(comment: Uint8Array): Promise<void>;
 }

package/dist/zip-writer.js

@@ -89,26 +89,33 @@ class ZipOutput {
     /**
      * Writes one entry whose sizes are known only once its data has gone out: a local header with the data descriptor
      * flag and no CRC or sizes, then `chunks`, the compressed data, then a data descriptor with the CRC and the sizes.
-     * `totals` gives the CRC and size of the uncompressed data once `chunks` has ended. An entry of 4 GiB or more gets
-     * eight-byte sizes in its descriptor and a ZIP64 field in the central directory, as archiver writes one.
+     * `totals` gives the CRC and size of the uncompressed data once `chunks` has ended.
+     *
+     * With `zip64`, the local header has a ZIP64 field, which tells a reader the descriptor's sizes are eight bytes. Its
+     * sizes are 0xFFFFFFFF and the field's are zero, as Python's zipfile writes a streamed entry. Without it, an entry of
+     * 4 GiB or more still gets eight-byte sizes in its descriptor, as archiver writes one. Either way, the central
+     * directory has a ZIP64 field only when a size or the offset needs one.
      */
-    async streamEntry(info, chunks, totals) {
+    async streamEntry(info, chunks, totals, zip64 = false) {
         const flags = info.flags | 0x0008;
+        const versionNeeded = zip64 ? Math.max(45, info.versionNeeded) : info.versionNeeded;
         const localOffset = this.offset;
         const local = new Builder()
             .u32(0x04034b50)
-            .u16(info.versionNeeded)
+            .u16(versionNeeded)
             .u16(flags)
             .u16(info.method)
             .u16(info.time)
             .u16(info.date)
             .u32(0)
-            .u32(0)
-            .u32(0)
+            .u32(zip64 ? MAX32 : 0)
+            .u32(zip64 ? MAX32 : 0)
             .u16(info.name.length)
-            .u16(info.extras.length)
-            .bytes(info.name)
-            .bytes(info.extras);
+            .u16(info.extras.length + (zip64 ? 20 : 0))
+            .bytes(info.name);
+        if (zip64)
+            local.u16(0x0001).u16(16).u64(0).u64(0);
+        local.bytes(info.extras);
         await this.put(local.build());
         let compressedSize = 0;
         for await (const c of chunks) {
@@ -117,12 +124,12 @@ class ZipOutput {
         }
         const { crc, size } = totals();
         const descriptor = new Builder().u32(0x08074b50).u32(crc);
-        if (size >= MAX32 || compressedSize >= MAX32)
+        if (zip64 || size >= MAX32 || compressedSize >= MAX32)
             descriptor.u64(compressedSize).u64(size);
         else
             descriptor.u32(compressedSize).u32(size);
         await this.put(descriptor.build());
-        this.central.push(centralRecord({ ...info, flags, crc, size, compressedSize }, localOffset));
+        this.central.push(centralRecord({ ...info, versionNeeded, flags, crc, size, compressedSize }, localOffset));
     }
     /** Writes the central directory and the end records, with the archive's comment. */
     async end(comment) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@patrickdk77/archive-defuse",
-  "version": "0.1.1",
+  "version": "0.1.3",
   "description": "Sanitize zip, tar, gzip, bzip2, zstd, Brotli, cab, 7z, xz and lzma archives. Detects path traversal, links out of the tree, device files, duplicate names, overlapping entries, hidden and prepended data, encrypted entries and compression bombs, hands every entry to file plugins, scores the risk, and writes a defused archive in the same format with no runtime dependencies.",
   "license": "MIT AND Apache-2.0",
   "author": "patrickdk77",