@saltcorn/admin-models 1.7.0-alpha.0 → 1.7.0-beta.0

package/dist/models/zip-writer.js

@@ -0,0 +1,321 @@
+/**
+ * A streaming ZIP writer.
+ *
+ * Entries are compressed and written out as they are added, so the archive is
+ * never held in memory: a backup of a 50 GB file store needs no more memory
+ * than a few buffers, whatever its size. Deflation runs on libuv's thread
+ * pool (zlib's streaming API), and the whole writer is driven from the zip
+ * worker thread, so nothing here competes with the server's event loop.
+ *
+ * Sizes and CRCs are not known until an entry has been written, so entries
+ * carry a data descriptor (general purpose bit 3) instead. Entries larger
+ * than ZIP64_THRESHOLD are written in zip64 format, which lifts the 4 GiB
+ * limit on entry and archive size.
+ *
+ * @category saltcorn-admin-models
+ * @module models/zip-writer
+ */
+import { createDeflateRaw } from "zlib";
+import { createReadStream } from "fs";
+import { stat } from "fs/promises";
+import { randomBytes } from "crypto";
+import { Readable } from "stream";
+import { once } from "events";
+import { CENTRAL_HEADER_SIZE, CENTRAL_SIG, DESCRIPTOR_SIG, EOCD_SIG, EOCD_SIZE, FLAG_DESCRIPTOR, FLAG_ENCRYPTED, FLAG_UTF8, LOCAL_HEADER_SIZE, LOCAL_SIG, METHOD_DEFLATED, METHOD_STORED, UINT16_MAX, UINT32_MAX, ZIP64_EOCD_SIG, ZIP64_EOCD_SIZE, ZIP64_EXTRA_ID, ZIP64_LOCATOR_SIG, ZIP64_LOCATOR_SIZE, ZIP64_THRESHOLD, ZipCrypto, crc32, dosDateTime, writeUInt64, } from "./zip-format.js";
+export class ZipStreamWriter {
+    constructor(out, options = {}) {
+        this.offset = 0;
+        this.entries = [];
+        this.names = new Set();
+        this.out = out;
+        this.password = options.password || undefined;
+        this.level = options.level ?? 5;
+        this.zip64Threshold = options.zip64Threshold ?? ZIP64_THRESHOLD;
+        this.probeLimit = options.probeLimit ?? 32 * 1024 * 1024;
+    }
+    /** Write to the output, respecting its backpressure */
+    async put(buf) {
+        if (!this.out.write(buf))
+            await once(this.out, "drain");
+        this.offset += buf.length;
+    }
+    /**
+     * Zip entry names are always forward-slash separated and never absolute.
+     * A duplicate name would make the archive ambiguous, so it is an error.
+     */
+    entryName(name, isDirectory) {
+        let clean = name.replace(/\\/g, "/").replace(/^\/+/, "");
+        if (isDirectory && !clean.endsWith("/"))
+            clean += "/";
+        if (this.names.has(clean))
+            throw new Error(`Duplicate zip entry: ${clean}`);
+        this.names.add(clean);
+        return Buffer.from(clean, "utf8");
+    }
+    /**
+     * The local file header. When an entry is zip64 the header carries a
+     * zip64 extra field with placeholder sizes, which is what tells a reader
+     * that the trailing data descriptor has 8-byte sizes.
+     */
+    localHeader(entry) {
+        const extraSize = entry.zip64 ? 20 : 0;
+        const buf = Buffer.alloc(LOCAL_HEADER_SIZE + entry.name.length + extraSize);
+        buf.writeUInt32LE(LOCAL_SIG, 0);
+        buf.writeUInt16LE(entry.zip64 ? 45 : 20, 4);
+        buf.writeUInt16LE(entry.flags, 6);
+        buf.writeUInt16LE(entry.method, 8);
+        buf.writeUInt16LE(entry.time, 10);
+        buf.writeUInt16LE(entry.date, 12);
+        // crc and sizes follow in the data descriptor, except for entries
+        // written whole (directories), which have nothing to describe
+        buf.writeUInt32LE(entry.crc, 14);
+        buf.writeUInt32LE(entry.zip64 ? UINT32_MAX : entry.compressedSize, 18);
+        buf.writeUInt32LE(entry.zip64 ? UINT32_MAX : entry.uncompressedSize, 22);
+        buf.writeUInt16LE(entry.name.length, 26);
+        buf.writeUInt16LE(extraSize, 28);
+        entry.name.copy(buf, LOCAL_HEADER_SIZE);
+        if (entry.zip64) {
+            const at = LOCAL_HEADER_SIZE + entry.name.length;
+            buf.writeUInt16LE(ZIP64_EXTRA_ID, at);
+            buf.writeUInt16LE(16, at + 2);
+            writeUInt64(buf, entry.uncompressedSize, at + 4);
+            writeUInt64(buf, entry.compressedSize, at + 12);
+        }
+        return buf;
+    }
+    /** The data descriptor written after an entry's data */
+    dataDescriptor(entry) {
+        const buf = Buffer.alloc(entry.zip64 ? 24 : 16);
+        buf.writeUInt32LE(DESCRIPTOR_SIG, 0);
+        buf.writeUInt32LE(entry.crc, 4);
+        if (entry.zip64) {
+            writeUInt64(buf, entry.compressedSize, 8);
+            writeUInt64(buf, entry.uncompressedSize, 16);
+        }
+        else {
+            buf.writeUInt32LE(entry.compressedSize, 8);
+            buf.writeUInt32LE(entry.uncompressedSize, 12);
+        }
+        return buf;
+    }
+    /**
+     * Add an entry whose contents come from a stream.
+     *
+     * Whether an entry is zip64 has to be settled before any of its data is
+     * written, so a source of unknown length is read ahead of up to
+     * `probeLimit` bytes to find out which it is.
+     *
+     * @param options.sizeHint how large the source is known to be, if it is
+     */
+    async addStream(name, source, options = {}) {
+        const iterator = source[Symbol.asyncIterator]();
+        const probed = [];
+        let zip64;
+        if (options.sizeHint !== undefined)
+            zip64 = options.sizeHint >= this.zip64Threshold;
+        else {
+            let seen = 0;
+            let ended = false;
+            while (seen <= this.probeLimit) {
+                const next = await iterator.next();
+                if (next.done) {
+                    ended = true;
+                    break;
+                }
+                probed.push(next.value);
+                seen += next.value.length;
+            }
+            zip64 = !ended || seen >= this.zip64Threshold;
+        }
+        const { time, date } = dosDateTime(options.mtime ?? new Date());
+        const entry = {
+            name: this.entryName(name, false),
+            flags: FLAG_DESCRIPTOR | FLAG_UTF8 | (this.password ? FLAG_ENCRYPTED : 0),
+            method: METHOD_DEFLATED,
+            time,
+            date,
+            crc: 0,
+            compressedSize: 0,
+            uncompressedSize: 0,
+            offset: this.offset,
+            mode: options.mode ?? 0o644,
+            isDirectory: false,
+            zip64,
+        };
+        await this.put(this.localHeader(entry));
+        const crypto = this.password ? new ZipCrypto(this.password) : null;
+        if (crypto) {
+            // the 12 byte encryption header. Its last byte is a check byte the
+            // reader compares against - the high byte of the modification time,
+            // because with a data descriptor the CRC is not known yet
+            const salt = randomBytes(12);
+            salt[11] = (time >>> 8) & 0xff;
+            await this.put(crypto.encrypt(salt));
+            entry.compressedSize += 12;
+        }
+        const deflate = createDeflateRaw({ level: this.level });
+        let feedError;
+        const feed = (async () => {
+            const consume = async (buf) => {
+                entry.crc = crc32(buf, entry.crc);
+                entry.uncompressedSize += buf.length;
+                if (!deflate.write(buf))
+                    await once(deflate, "drain");
+            };
+            // whatever the format probe already read, then the rest of the source
+            for (const buf of probed)
+                await consume(buf);
+            while (true) {
+                const next = await iterator.next();
+                if (next.done)
+                    break;
+                await consume(next.value);
+            }
+            deflate.end();
+        })().catch((e) => {
+            feedError = e;
+            // unblock the consumer below, which would otherwise wait for an end
+            // that is never coming
+            deflate.destroy(e);
+        });
+        try {
+            for await (const chunk of deflate) {
+                const data = crypto
+                    ? crypto.encrypt(chunk)
+                    : chunk;
+                entry.compressedSize += data.length;
+                await this.put(data);
+            }
+        }
+        finally {
+            await feed;
+        }
+        if (feedError)
+            throw feedError;
+        if (!entry.zip64 &&
+            (entry.compressedSize > UINT32_MAX || entry.uncompressedSize > UINT32_MAX))
+            throw new Error(`Zip entry ${name} grew past 4GB while it was being written`);
+        await this.put(this.dataDescriptor(entry));
+        this.entries.push(entry);
+    }
+    /** Add an entry from content held in memory */
+    async addBuffer(name, data, options = {}) {
+        await this.addStream(name, Readable.from([data]), {
+            ...options,
+            sizeHint: data.length,
+        });
+    }
+    /** Add an entry read from a file on disk. The file is streamed. */
+    async addFile(name, localPath) {
+        const stats = await stat(localPath);
+        await this.addStream(name, createReadStream(localPath), {
+            sizeHint: stats.size,
+            mode: stats.mode & 0o7777,
+            mtime: stats.mtime,
+        });
+    }
+    /** Add a directory entry */
+    async addDirectory(name, options = {}) {
+        const { time, date } = dosDateTime(options.mtime ?? new Date());
+        const entry = {
+            name: this.entryName(name, true),
+            flags: FLAG_UTF8,
+            method: METHOD_STORED,
+            time,
+            date,
+            crc: 0,
+            compressedSize: 0,
+            uncompressedSize: 0,
+            offset: this.offset,
+            mode: options.mode ?? 0o755,
+            isDirectory: true,
+            zip64: false,
+        };
+        await this.put(this.localHeader(entry));
+        this.entries.push(entry);
+    }
+    /** The central directory record of one entry */
+    centralHeader(entry) {
+        // only the values that do not fit go into the zip64 extra field, in the
+        // order the format prescribes
+        const zip64Fields = [];
+        if (entry.uncompressedSize > UINT32_MAX)
+            zip64Fields.push(entry.uncompressedSize);
+        if (entry.compressedSize > UINT32_MAX)
+            zip64Fields.push(entry.compressedSize);
+        if (entry.offset > UINT32_MAX)
+            zip64Fields.push(entry.offset);
+        const extraSize = zip64Fields.length ? 4 + zip64Fields.length * 8 : 0;
+        const buf = Buffer.alloc(CENTRAL_HEADER_SIZE + entry.name.length + extraSize);
+        buf.writeUInt32LE(CENTRAL_SIG, 0);
+        buf.writeUInt16LE((3 << 8) | 45, 4); // made by unix
+        buf.writeUInt16LE(extraSize ? 45 : 20, 6);
+        buf.writeUInt16LE(entry.flags, 8);
+        buf.writeUInt16LE(entry.method, 10);
+        buf.writeUInt16LE(entry.time, 12);
+        buf.writeUInt16LE(entry.date, 14);
+        buf.writeUInt32LE(entry.crc, 16);
+        buf.writeUInt32LE(Math.min(entry.compressedSize, UINT32_MAX), 20);
+        buf.writeUInt32LE(Math.min(entry.uncompressedSize, UINT32_MAX), 24);
+        buf.writeUInt16LE(entry.name.length, 28);
+        buf.writeUInt16LE(extraSize, 30);
+        buf.writeUInt16LE(0, 32); // comment length
+        buf.writeUInt16LE(0, 34); // disk number
+        buf.writeUInt16LE(0, 36); // internal attributes
+        buf.writeUInt32LE(((entry.mode & 0xffff) << 16) | (entry.isDirectory ? 0x10 : 0), 38);
+        buf.writeUInt32LE(Math.min(entry.offset, UINT32_MAX), 42);
+        entry.name.copy(buf, CENTRAL_HEADER_SIZE);
+        if (extraSize) {
+            const at = CENTRAL_HEADER_SIZE + entry.name.length;
+            buf.writeUInt16LE(ZIP64_EXTRA_ID, at);
+            buf.writeUInt16LE(extraSize - 4, at + 2);
+            zip64Fields.forEach((v, i) => writeUInt64(buf, v, at + 4 + i * 8));
+        }
+        return buf;
+    }
+    /**
+     * Write the central directory and the end of central directory records.
+     * The writer cannot be used afterwards.
+     */
+    async finalize() {
+        const centralStart = this.offset;
+        for (const entry of this.entries)
+            await this.put(this.centralHeader(entry));
+        const centralSize = this.offset - centralStart;
+        const needsZip64 = this.entries.length > UINT16_MAX ||
+            centralStart > UINT32_MAX ||
+            centralSize > UINT32_MAX;
+        if (needsZip64) {
+            const zip64Eocd = Buffer.alloc(ZIP64_EOCD_SIZE);
+            zip64Eocd.writeUInt32LE(ZIP64_EOCD_SIG, 0);
+            writeUInt64(zip64Eocd, ZIP64_EOCD_SIZE - 12, 4);
+            zip64Eocd.writeUInt16LE((3 << 8) | 45, 12);
+            zip64Eocd.writeUInt16LE(45, 14);
+            zip64Eocd.writeUInt32LE(0, 16); // this disk
+            zip64Eocd.writeUInt32LE(0, 20); // disk with central directory
+            writeUInt64(zip64Eocd, this.entries.length, 24);
+            writeUInt64(zip64Eocd, this.entries.length, 32);
+            writeUInt64(zip64Eocd, centralSize, 40);
+            writeUInt64(zip64Eocd, centralStart, 48);
+            const eocdOffset = this.offset;
+            await this.put(zip64Eocd);
+            const locator = Buffer.alloc(ZIP64_LOCATOR_SIZE);
+            locator.writeUInt32LE(ZIP64_LOCATOR_SIG, 0);
+            locator.writeUInt32LE(0, 4);
+            writeUInt64(locator, eocdOffset, 8);
+            locator.writeUInt32LE(1, 16);
+            await this.put(locator);
+        }
+        const eocd = Buffer.alloc(EOCD_SIZE);
+        eocd.writeUInt32LE(EOCD_SIG, 0);
+        eocd.writeUInt16LE(0, 4);
+        eocd.writeUInt16LE(0, 6);
+        eocd.writeUInt16LE(Math.min(this.entries.length, UINT16_MAX), 8);
+        eocd.writeUInt16LE(Math.min(this.entries.length, UINT16_MAX), 10);
+        eocd.writeUInt32LE(Math.min(centralSize, UINT32_MAX), 12);
+        eocd.writeUInt32LE(Math.min(centralStart, UINT32_MAX), 16);
+        eocd.writeUInt16LE(0, 20); // comment length
+        await this.put(eocd);
+    }
+}
+//# sourceMappingURL=zip-writer.js.map

package/dist/models/zip-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip-writer.js","sourceRoot":"","sources":["../../models/zip-writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,gBAAgB,EAAE,MAAM,MAAM,CAAC;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,IAAI,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AACnC,OAAO,EAAE,WAAW,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAY,MAAM,QAAQ,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC;AAC9B,OAAO,EACL,mBAAmB,EACnB,WAAW,EACX,cAAc,EACd,QAAQ,EACR,SAAS,EACT,eAAe,EACf,cAAc,EACd,SAAS,EACT,iBAAiB,EACjB,SAAS,EACT,eAAe,EACf,aAAa,EACb,UAAU,EACV,UAAU,EACV,cAAc,EACd,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,kBAAkB,EAClB,eAAe,EACf,SAAS,EACT,KAAK,EACL,WAAW,EACX,WAAW,GACZ,MAAM,iBAAiB,CAAC;AAgCzB,MAAM,OAAO,eAAe;IAU1B,YAAY,GAAa,EAAE,UAA4B,EAAE;QAJjD,WAAM,GAAW,CAAC,CAAC;QACnB,YAAO,GAAmB,EAAE,CAAC;QAC7B,UAAK,GAAgB,IAAI,GAAG,EAAE,CAAC;QAGrC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,eAAe,CAAC;QAChE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3D,CAAC;IAED,uDAAuD;IAC/C,KAAK,CAAC,GAAG,CAAC,GAAW;QAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACxD,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACK,SAAS,CAAC,IAAY,EAAE,WAAoB;QAClD,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzD,IAAI,WAAW,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,KAAK,IAAI,GAAG,CAAC;QACtD,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,KAAK,EAAE,CAAC,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACtB,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,KAAmB;QACrC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;QAC5E,GAAG,CAAC,aAAa,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAChC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC5C,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACnC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,kEAAkE;QAClE,8DAA8D;QAC9D,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACvE,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QACzE,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzC,GAAG,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,iBAAiB,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,GAAG,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YACjD,GAAG,CAAC,aAAa,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YACtC,GAAG,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YAC9B,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,gBAAgB,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YACjD,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,cAAc,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,wDAAwD;IAChD,cAAc,CAAC,KAAmB;QACxC,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChD,GAAG,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACrC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAC1C,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAC3C,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,MAAgB,EAChB,UAA8D,EAAE;QAEhE,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC;QAChD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,KAAc,CAAC;QACnB,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;YAChC,KAAK,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,cAAc,CAAC;aAC7C,CAAC;YACJ,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,KAAK,GAAG,KAAK,CAAC;YAClB,OAAO,IAAI,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,KAAK,GAAG,IAAI,CAAC;oBACb,MAAM;gBACR,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAe,CAAC,CAAC;gBAClC,IAAI,IAAK,IAAI,CAAC,KAAgB,CAAC,MAAM,CAAC;YACxC,CAAC;YACD,KAAK,GAAG,CAAC,KAAK,IAAI,IAAI,IAAI,IAAI,CAAC,cAAc,CAAC;QAChD,CAAC;QAED,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,KAAK,GAAiB;YAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;YACjC,KAAK,EAAE,eAAe,GAAG,SAAS,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;YACzE,MAAM,EAAE,eAAe;YACvB,IAAI;YACJ,IAAI;YACJ,GAAG,EAAE,CAAC;YACN,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK;YAC3B,WAAW,EAAE,KAAK;YAClB,KAAK;SACN,CAAC;QACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,IAAI,MAAM,EAAE,CAAC;YACX,mEAAmE;YACnE,oEAAoE;YACpE,0DAA0D;YAC1D,MAAM,IAAI,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC;YAC7B,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC;YAC/B,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;YACrC,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,GAAG,gBAAgB,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACxD,IAAI,SAAc,CAAC;QACnB,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,EAAE;YACvB,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAAE,EAAE;gBACpC,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClC,KAAK,CAAC,gBAAgB,IAAI,GAAG,CAAC,MAAM,CAAC;gBACrC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACxD,CAAC,CAAC;YACF,sEAAsE;YACtE,KAAK,MAAM,GAAG,IAAI,MAAM;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YAC7C,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,IAAI,CAAC,IAAI;oBAAE,MAAM;gBACrB,MAAM,OAAO,CAAC,IAAI,CAAC,KAAe,CAAC,CAAC;YACtC,CAAC;YACD,OAAO,CAAC,GAAG,EAAE,CAAC;QAChB,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACf,SAAS,GAAG,CAAC,CAAC;YACd,oEAAoE;YACpE,uBAAuB;YACvB,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAClC,MAAM,IAAI,GAAG,MAAM;oBACjB,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAe,CAAC;oBACjC,CAAC,CAAE,KAAgB,CAAC;gBACtB,KAAK,CAAC,cAAc,IAAI,IAAI,CAAC,MAAM,CAAC;gBACpC,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC;QACb,CAAC;QACD,IAAI,SAAS;YAAE,MAAM,SAAS,CAAC;QAE/B,IACE,CAAC,KAAK,CAAC,KAAK;YACZ,CAAC,KAAK,CAAC,cAAc,GAAG,UAAU,IAAI,KAAK,CAAC,gBAAgB,GAAG,UAAU,CAAC;YAE1E,MAAM,IAAI,KAAK,CACb,aAAa,IAAI,2CAA2C,CAC7D,CAAC;QAEJ,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,+CAA+C;IAC/C,KAAK,CAAC,SAAS,CACb,IAAY,EACZ,IAAY,EACZ,UAA2C,EAAE;QAE7C,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE;YAChD,GAAG,OAAO;YACV,QAAQ,EAAE,IAAI,CAAC,MAAM;SACtB,CAAC,CAAC;IACL,CAAC;IAED,mEAAmE;IACnE,KAAK,CAAC,OAAO,CAAC,IAAY,EAAE,SAAiB;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,gBAAgB,CAAC,SAAS,CAAC,EAAE;YACtD,QAAQ,EAAE,KAAK,CAAC,IAAI;YACpB,IAAI,EAAE,KAAK,CAAC,IAAI,GAAG,MAAM;YACzB,KAAK,EAAE,KAAK,CAAC,KAAK;SACnB,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,YAAY,CAChB,IAAY,EACZ,UAA2C,EAAE;QAE7C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,EAAE,CAAC,CAAC;QAChE,MAAM,KAAK,GAAiB;YAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC;YAChC,KAAK,EAAE,SAAS;YAChB,MAAM,EAAE,aAAa;YACrB,IAAI;YACJ,IAAI;YACJ,GAAG,EAAE,CAAC;YACN,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,CAAC;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK;YAC3B,WAAW,EAAE,IAAI;YACjB,KAAK,EAAE,KAAK;SACb,CAAC;QACF,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;QACxC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAED,gDAAgD;IACxC,aAAa,CAAC,KAAmB;QACvC,wEAAwE;QACxE,8BAA8B;QAC9B,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,IAAI,KAAK,CAAC,gBAAgB,GAAG,UAAU;YACrC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,cAAc,GAAG,UAAU;YACnC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,MAAM,GAAG,UAAU;YAAE,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAEtE,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CACtB,mBAAmB,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,SAAS,CACpD,CAAC;QACF,GAAG,CAAC,aAAa,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe;QACpD,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC1C,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACjC,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QACpE,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACzC,GAAG,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACjC,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB;QAC3C,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc;QACxC,GAAG,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,sBAAsB;QAChD,GAAG,CAAC,aAAa,CACf,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAC9D,EAAE,CACH,CAAC;QACF,GAAG,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;QAC1C,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,EAAE,GAAG,mBAAmB,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;YACnD,GAAG,CAAC,aAAa,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;YACtC,GAAG,CAAC,aAAa,CAAC,SAAS,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;YACzC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACrE,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,QAAQ;QACZ,MAAM,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;QACjC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;QAC5E,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC;QAE/C,MAAM,UAAU,GACd,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,UAAU;YAChC,YAAY,GAAG,UAAU;YACzB,WAAW,GAAG,UAAU,CAAC;QAE3B,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAChD,SAAS,CAAC,aAAa,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;YAC3C,WAAW,CAAC,SAAS,EAAE,eAAe,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;YAChD,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3C,SAAS,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YAChC,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY;YAC5C,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,8BAA8B;YAC9D,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAChD,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAChD,WAAW,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;YACxC,WAAW,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,CAAC,CAAC;YACzC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC;YAC/B,MAAM,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAE1B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;YACjD,OAAO,CAAC,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;YAC5C,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC5B,WAAW,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;YACpC,OAAO,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC7B,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACrC,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;QACjE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,IAAI,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB;QAC5C,MAAM,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;CACF"}

package/dist/models/zip.d.ts

@@ -0,0 +1,101 @@
+import { Readable } from "stream";
+/** Request sent from the main thread to the zip worker. */
+export type ZipRequest = {
+    id: number;
+    cmd: "beginFile";
+    path: string;
+    password?: string;
+} | {
+    id: number;
+    cmd: "beginStream";
+    password?: string;
+} | {
+    id: number;
+    cmd: "addBuffer";
+    entryName: string;
+    data: Uint8Array;
+} | {
+    id: number;
+    cmd: "addLocalFile";
+    entryName: string;
+    localPath: string;
+} | {
+    id: number;
+    cmd: "addDirectory";
+    entryName: string;
+} | {
+    id: number;
+    cmd: "beginEntry";
+    entryName: string;
+} | {
+    id: number;
+    cmd: "entryChunk";
+    data: Uint8Array;
+} | {
+    id: number;
+    cmd: "endEntry";
+} | {
+    id: number;
+    cmd: "finish";
+};
+/** Reply sent from the zip worker back to the main thread. */
+export type ZipResponse = {
+    id: number;
+    ok: true;
+    result?: any;
+} | {
+    id: number;
+    ok: false;
+    error: string;
+};
+/**
+ * Builds a zip file, one entry at a time, on a worker thread.
+ *
+ * `finish` completes the archive and shuts the worker down; the builder
+ * cannot be used afterwards.
+ */
+export declare class ZipBuilder {
+    private worker;
+    private closed;
+    /** set when the archive is streamed to the caller rather than to a file */
+    readonly stream?: Readable;
+    private constructor();
+    private static begin;
+    /**
+     * Build an archive at a path on disk. It is written where it belongs -
+     * writing it elsewhere and moving it would fail across filesystems.
+     *
+     * @param password if non-empty, entries are encrypted with the traditional
+     *   zip password scheme, as the system `zip -P` did
+     */
+    static toFile(path: string, password?: string): Promise<ZipBuilder>;
+    /**
+     * Build an archive into a readable stream, for destinations written over
+     * the network (S3, SFTP) or straight to a browser, so that no copy is made
+     * on the local disk. Read `stream` while adding entries: the archive is
+     * produced as it is consumed.
+     */
+    static toStream(password?: string): Promise<ZipBuilder>;
+    /** Add an entry from content held in memory */
+    addBuffer(entryName: string, data: Buffer): Promise<void>;
+    /** Add an entry from a file on disk. The worker streams it. */
+    addLocalFile(entryName: string, localPath: string): Promise<void>;
+    /**
+     * Add an entry from a stream of content produced in the main thread, for
+     * content too large to want as one buffer - a table dump, say. Each chunk
+     * is acknowledged before the next is sent, so the producer runs no faster
+     * than the archive is written.
+     */
+    addStream(entryName: string, source: Readable): Promise<void>;
+    /** Add a directory entry. A trailing slash is added if missing. */
+    addDirectory(entryName: string): Promise<void>;
+    /** Complete the archive and close the worker */
+    finish(): Promise<void>;
+    /**
+     * Abandon a half-built archive and close the worker. Any consumer of
+     * `stream` is failed too, so a partial archive is never mistaken for a
+     * complete one.
+     */
+    discard(cause?: any): Promise<void>;
+}
+//# sourceMappingURL=zip.d.ts.map

package/dist/models/zip.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip.d.ts","sourceRoot":"","sources":["../../models/zip.ts"],"names":[],"mappings":"AAmBA,OAAO,EAAe,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAE/C,2DAA2D;AAC3D,MAAM,MAAM,UAAU,GAClB;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACjE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,aAAa,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,GACrD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACrE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACzE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,cAAc,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACtD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,YAAY,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAE,GACnD;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,UAAU,CAAA;CAAE,GAC/B;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,QAAQ,CAAA;CAAE,CAAC;AAElC,8DAA8D;AAC9D,MAAM,MAAM,WAAW,GACnB;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,IAAI,CAAC;IAAC,MAAM,CAAC,EAAE,GAAG,CAAA;CAAE,GACtC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAgH7C;;;;;GAKG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,MAAM,CAAkB;IAChC,2EAA2E;IAC3E,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC;IAE3B,OAAO;mBAKc,KAAK;IAgB1B;;;;;;OAMG;WACU,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAMzE;;;;;OAKG;WACU,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAQ7D,+CAA+C;IACzC,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/D,+DAA+D;IACzD,YAAY,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvE;;;;;OAKG;IACG,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAOnE,mEAAmE;IAC7D,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIpD,gDAAgD;IAC1C,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAW7B;;;;OAIG;IACG,OAAO,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;CAQ1C"}

package/dist/models/zip.js

@@ -0,0 +1,211 @@
+/**
+ * Building a zip file on a worker thread.
+ *
+ * Zipping is CPU-bound, so doing it on the main thread would block the event
+ * loop for the whole duration of a backup, which is why this used to shell
+ * out to the system `zip` executable. A worker thread gives the same
+ * non-blocking behaviour without depending on anything being installed on the
+ * host. (Reading is a different matter - see zip-extract.ts.)
+ *
+ * The archive is streamed, never assembled in memory: entries are compressed
+ * and written out as they are added (see zip-writer.ts), so a backup of a
+ * file store far larger than RAM works. Content generated in the main thread
+ * (pack.json, table dumps, config) is handed over a buffer at a time, while
+ * files already on disk are only named - the worker streams them itself.
+ *
+ * @category saltcorn-admin-models
+ * @module models/zip
+ */
+import { Worker } from "worker_threads";
+import { PassThrough } from "stream";
+const workerURL = new URL("./zip-worker.js", import.meta.url);
+/**
+ * A worker thread running zip-worker.js, with a promise-per-message
+ * request/response protocol on top of it.
+ */
+class ZipWorker {
+    constructor() {
+        this.seq = 0;
+        this.pending = new Map();
+        this.worker = new Worker(workerURL);
+        this.worker.on("message", (msg) => {
+            if (msg?.type === "chunk")
+                return this.takeChunk(msg.data);
+            if (msg?.type === "chunkEnd")
+                return this.sink?.end();
+            const waiter = this.pending.get(msg.id);
+            if (!waiter)
+                return;
+            this.pending.delete(msg.id);
+            if (msg.ok)
+                waiter.resolve(msg.result);
+            else
+                waiter.reject(new Error(msg.error));
+        });
+        this.worker.on("error", (e) => this.fail(e));
+        this.worker.on("exit", (code) => {
+            if (this.pending.size)
+                this.fail(new Error(`zip worker stopped with exit code ${code}`));
+        });
+    }
+    /**
+     * Take a chunk of the archive and acknowledge it once the consumer has
+     * room for more. The worker sends the next chunk only after the
+     * acknowledgement, so a slow upload slows the whole pipeline down rather
+     * than filling memory.
+     */
+    takeChunk(data) {
+        const sink = this.sink;
+        if (!sink || sink.destroyed)
+            return this.abandon(new Error("Zip output stream was closed"));
+        const ack = () => this.worker.postMessage({ cmd: "chunkAck" });
+        if (sink.write(Buffer.from(data.buffer, data.byteOffset, data.byteLength)))
+            ack();
+        else
+            sink.once("drain", ack);
+    }
+    fail(e) {
+        this.dead = e;
+        for (const waiter of this.pending.values())
+            waiter.reject(e);
+        this.pending.clear();
+        this.sink?.destroy(e);
+    }
+    /**
+     * The output went away - a failed upload, a browser that closed the
+     * connection. Stop the worker rather than leaving it blocked forever on an
+     * acknowledgement that is never coming.
+     */
+    abandon(e) {
+        if (this.dead)
+            return;
+        this.fail(e);
+        this.worker.terminate();
+    }
+    /**
+     * Send a request and wait for its reply.
+     *
+     * Content buffers are copied, not transferred. Transferring would save the
+     * copy, but the buffer belongs to whoever passed it in: a producer that
+     * fills and reuses one buffer - which a Readable is free to do - would find
+     * it detached under it, and would go on writing zero-length chunks with no
+     * error anywhere. The copy is a memcpy against a deflate; it does not show.
+     */
+    send(req) {
+        if (this.dead)
+            return Promise.reject(this.dead);
+        const id = ++this.seq;
+        const message = { ...req, id };
+        return new Promise((resolve, reject) => {
+            this.pending.set(id, { resolve, reject });
+            this.worker.postMessage(message);
+        });
+    }
+    /** Start collecting the archive bytes the worker sends us */
+    openSink() {
+        const sink = new PassThrough();
+        // a consumer that fails takes the sink down with it; keep a listener of
+        // our own so that never becomes an unhandled error event, and stop
+        // producing bytes nobody is going to read
+        sink.on("error", (e) => this.abandon(e));
+        this.sink = sink;
+        return sink;
+    }
+    async close() {
+        await this.worker.terminate();
+    }
+}
+/**
+ * Builds a zip file, one entry at a time, on a worker thread.
+ *
+ * `finish` completes the archive and shuts the worker down; the builder
+ * cannot be used afterwards.
+ */
+export class ZipBuilder {
+    constructor(worker, stream) {
+        this.closed = false;
+        this.worker = worker;
+        this.stream = stream;
+    }
+    static async begin(start, stream) {
+        const worker = new ZipWorker();
+        const readable = stream?.(worker);
+        const builder = new ZipBuilder(worker, readable);
+        try {
+            await start(worker);
+        }
+        catch (e) {
+            await builder.discard(e);
+            throw e;
+        }
+        return builder;
+    }
+    /**
+     * Build an archive at a path on disk. It is written where it belongs -
+     * writing it elsewhere and moving it would fail across filesystems.
+     *
+     * @param password if non-empty, entries are encrypted with the traditional
+     *   zip password scheme, as the system `zip -P` did
+     */
+    static async toFile(path, password) {
+        return await ZipBuilder.begin((worker) => worker.send({ cmd: "beginFile", path, password: password || undefined }));
+    }
+    /**
+     * Build an archive into a readable stream, for destinations written over
+     * the network (S3, SFTP) or straight to a browser, so that no copy is made
+     * on the local disk. Read `stream` while adding entries: the archive is
+     * produced as it is consumed.
+     */
+    static async toStream(password) {
+        return await ZipBuilder.begin((worker) => worker.send({ cmd: "beginStream", password: password || undefined }), (worker) => worker.openSink());
+    }
+    /** Add an entry from content held in memory */
+    async addBuffer(entryName, data) {
+        await this.worker.send({ cmd: "addBuffer", entryName, data });
+    }
+    /** Add an entry from a file on disk. The worker streams it. */
+    async addLocalFile(entryName, localPath) {
+        await this.worker.send({ cmd: "addLocalFile", entryName, localPath });
+    }
+    /**
+     * Add an entry from a stream of content produced in the main thread, for
+     * content too large to want as one buffer - a table dump, say. Each chunk
+     * is acknowledged before the next is sent, so the producer runs no faster
+     * than the archive is written.
+     */
+    async addStream(entryName, source) {
+        await this.worker.send({ cmd: "beginEntry", entryName });
+        for await (const chunk of source)
+            await this.worker.send({ cmd: "entryChunk", data: chunk });
+        await this.worker.send({ cmd: "endEntry" });
+    }
+    /** Add a directory entry. A trailing slash is added if missing. */
+    async addDirectory(entryName) {
+        await this.worker.send({ cmd: "addDirectory", entryName });
+    }
+    /** Complete the archive and close the worker */
+    async finish() {
+        try {
+            await this.worker.send({ cmd: "finish" });
+        }
+        catch (e) {
+            await this.discard(e);
+            throw e;
+        }
+        this.closed = true;
+        await this.worker.close();
+    }
+    /**
+     * Abandon a half-built archive and close the worker. Any consumer of
+     * `stream` is failed too, so a partial archive is never mistaken for a
+     * complete one.
+     */
+    async discard(cause) {
+        if (this.closed)
+            return;
+        this.closed = true;
+        this.stream?.destroy(cause instanceof Error ? cause : new Error("Zip archive abandoned"));
+        await this.worker.close();
+    }
+}
+//# sourceMappingURL=zip.js.map

package/dist/models/zip.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zip.js","sourceRoot":"","sources":["../../models/zip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AACH,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAY,MAAM,QAAQ,CAAC;AAyB/C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAE9D;;;GAGG;AACH,MAAM,SAAS;IAWb;QATQ,QAAG,GAAW,CAAC,CAAC;QAChB,YAAO,GAGX,IAAI,GAAG,EAAE,CAAC;QAMZ,IAAI,CAAC,MAAM,GAAG,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;YACrC,IAAI,GAAG,EAAE,IAAI,KAAK,OAAO;gBAAE,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3D,IAAI,GAAG,EAAE,IAAI,KAAK,UAAU;gBAAE,OAAO,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC;YACtD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxC,IAAI,CAAC,MAAM;gBAAE,OAAO;YACpB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,GAAG,CAAC,EAAE;gBAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;;gBAClC,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE;YACtC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI;gBACnB,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,qCAAqC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACK,SAAS,CAAC,IAAgB;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS;YACzB,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;QAC/D,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YACxE,GAAG,EAAE,CAAC;;YACH,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IAC/B,CAAC;IAEO,IAAI,CAAC,CAAQ;QACnB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,OAAO,CAAC,CAAQ;QACtB,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO;QACtB,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACb,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED;;;;;;;;OAQG;IACH,IAAI,CAAC,GAAe;QAClB,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;QACtB,MAAM,OAAO,GAAG,EAAE,GAAG,GAAG,EAAE,EAAE,EAAgB,CAAC;QAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,6DAA6D;IAC7D,QAAQ;QACN,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,wEAAwE;QACxE,mEAAmE;QACnE,0CAA0C;QAC1C,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,UAAU;IAMrB,YAAoB,MAAiB,EAAE,MAAiB;QAJhD,WAAM,GAAY,KAAK,CAAC;QAK9B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAEO,MAAM,CAAC,KAAK,CAAC,KAAK,CACxB,KAA2C,EAC3C,MAAwC;QAExC,MAAM,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;QACtB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,CAAC,CAAC;QACV,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,QAAiB;QACjD,OAAO,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CACvC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC,CACzE,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAiB;QACrC,OAAO,MAAM,UAAU,CAAC,KAAK,CAC3B,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,IAAI,SAAS,EAAE,CAAC,EACtE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;IACJ,CAAC;IAED,+CAA+C;IAC/C,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,IAAY;QAC7C,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,CAAC;IAED,+DAA+D;IAC/D,KAAK,CAAC,YAAY,CAAC,SAAiB,EAAE,SAAiB;QACrD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;IACxE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,SAAS,CAAC,SAAiB,EAAE,MAAgB;QACjD,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC;QACzD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM;YAC9B,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,YAAY,EAAE,IAAI,EAAE,KAAe,EAAE,CAAC,CAAC;QACvE,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,mEAAmE;IACnE,KAAK,CAAC,YAAY,CAAC,SAAiB;QAClC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,CAAC;QACV,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO,CAAC,KAAW;QACvB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,MAAkC,EAAE,OAAO,CAC/C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CACpE,CAAC;QACF,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC5B,CAAC;CACF"}