@patrickdk77/archive-defuse 0.0.0-stage → 0.1.0

package/dist/single.js

@@ -0,0 +1,453 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isBzip2 = exports.isGzip = exports.BrotliArchive = exports.ZstdArchive = exports.Bzip2Archive = exports.GzipArchive = void 0;
+exports.looksLikeBrotli = looksLikeBrotli;
+// The formats that compress one file: gzip, bzip2, zstd and Brotli. Each is an archive with one entry, whose content
+// the engine hands to the file plugins like any other entry's. A gzip file can name its content, the others cannot.
+const bzip2_1 = require("./bzip2");
+const codecs_1 = require("./codecs");
+const crc32_1 = require("./crc32");
+const errors_1 = require("./errors");
+const io_1 = require("./io");
+const model_1 = require("./model");
+const sniff_1 = require("./sniff");
+const types_1 = require("./types");
+const zstd_1 = require("./zstd");
+class Single {
+    reader;
+    notes;
+    single = true;
+    lateNotes = [];
+    entries;
+    comment;
+    /** Set once the entry has been read and the stream turned out to be one stream with nothing after it. */
+    raw;
+    constructor(reader, notes, name = '') {
+        this.reader = reader;
+        this.notes = notes;
+        this.entries = [{ index: 0, name, kind: 'file', size: 0, compressedSize: reader.size, notes: [] }];
+    }
+    async unlock() {
+        return false;
+    }
+    hasMetadata() {
+        return false;
+    }
+    /** True when encode() can store data uncompressed. */
+    canStore = false;
+    /** Notes what follows the last stream, and what it reads as. */
+    async trailing(at) {
+        const size = this.reader.size;
+        if (at >= size)
+            return;
+        this.lateNotes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.TrailingBytes, data: { bytes: size - at } });
+        const type = (0, sniff_1.sniffType)(await this.reader.read(at, Math.min(1024, size - at)));
+        if (type !== undefined)
+            this.lateNotes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.Polyglot, data: { type, where: 'after the compressed data' } });
+    }
+    /**
+     * Writes the stream: the compressed bytes as they were when the content stayed as it was and was one stream, or the
+     * content compressed again. Compressed data that would pass the run's ratio limit is stored instead where the
+     * format can store, so the output reads under the same limit.
+     */
+    async writeStream(plan, sink, context) {
+        const k = plan.kept[0];
+        if (k && !k.replacement && !k.recompress && this.raw) {
+            for await (const c of this.reader.chunks(this.raw.start, this.raw.length)) {
+                context.meter().time();
+                await sink.write(c);
+            }
+            return;
+        }
+        const size = k ? this.sizeOf(k) : 0;
+        const content = () => (k ? (0, model_1.contentOf)(this, k, context) : (0, model_1.nothing)());
+        const spill = new io_1.SpillSink(context.temp, context.threshold);
+        try {
+            for await (const c of this.encode(content(), plan, false))
+                await spill.write(c);
+            await spill.close();
+            if (this.canStore && size > 1024 * 1024 && size > spill.length * plan.ratioLimit) {
+                await spill.dispose();
+                for await (const c of this.encode(content(), plan, true))
+                    await sink.write(c);
+                return;
+            }
+            const source = spill.source();
+            try {
+                const length = await this.encodedLength(source, spill.length);
+                for (let at = 0; at < length; at += 262144) {
+                    context.meter().time();
+                    await sink.write(await source.read(at, Math.min(262144, length - at)));
+                }
+            }
+            finally {
+                await source.close?.();
+            }
+        }
+        finally {
+            await spill.dispose();
+        }
+    }
+    /** How much of what encode() wrote to keep. */
+    async encodedLength(_source, length) {
+        return length;
+    }
+    sizeOf(k) {
+        return k.replacement ? k.replacement.length : (k.entry.decoded ?? (0, model_1.sizeOf)(k));
+    }
+    async write(plan, sink, context) {
+        await this.writeStream(plan, sink, context);
+        if (plan.keepTrailing && this.trailingStart !== undefined)
+            for await (const c of this.reader.chunks(this.trailingStart, this.reader.size - this.trailingStart))
+                await sink.write(c);
+    }
+    /** Where the data after the last stream starts, once the entry has been read. */
+    trailingStart;
+}
+/** Reads a gzip member header at `at`. */
+async function gzipHeader(reader, at, notes) {
+    const h = await reader.read(at, 10);
+    if (h.length < 10)
+        throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Truncated, 'The file ends inside a gzip header', { what: 'gzip header' });
+    if (h[0] !== 0x1f || h[1] !== 0x8b)
+        throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Unparseable, 'Not a gzip member', { reason: 'no gzip signature' });
+    if (h[2] !== 8)
+        throw new errors_1.StopError(types_1.ArchiveCategory.Compression, types_1.ArchiveDetail.UnsupportedMethod, 'A gzip method other than deflate', { method: `gzip method ${h[2]}` });
+    const flags = h[3];
+    const view = new DataView(h.buffer, h.byteOffset, h.byteLength);
+    const header = { flags, mtime: view.getUint32(4, true), os: h[9], dataStart: 0 };
+    if (flags & 0xe0)
+        notes?.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MalformedHeader, data: { reason: 'the gzip header sets reserved flags, which gzip refuses' } });
+    let p = at + 10;
+    const field = async () => {
+        // A NUL ends the name and the comment. Neither may run past 64 KiB.
+        const parts = [];
+        for (let len = 0;;) {
+            const b = await reader.read(p, 4096);
+            if (b.length === 0)
+                throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Truncated, 'The file ends inside a gzip header', { what: 'gzip header' });
+            const z = b.indexOf(0);
+            const piece = z < 0 ? b : b.subarray(0, z);
+            parts.push(piece);
+            len += piece.length;
+            p += piece.length;
+            if (z >= 0) {
+                p++;
+                return Buffer.concat(parts);
+            }
+            if (len > 65536)
+                throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Unparseable, 'A gzip header field without an end', { reason: 'a gzip name or comment without an end' });
+        }
+    };
+    if (flags & 0x04) {
+        const x = await reader.read(p, 2);
+        if (x.length < 2)
+            throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Truncated, 'The file ends inside a gzip header', { what: 'gzip header' });
+        const xlen = x[0] | (x[1] << 8);
+        notes?.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.ExtraField, data: { field: 'a gzip extra field' } });
+        p += 2 + xlen;
+    }
+    if (flags & 0x08)
+        header.name = Buffer.from(await field()).toString('latin1');
+    if (flags & 0x10)
+        header.comment = Buffer.from(await field()).toString('latin1');
+    if (flags & 0x02) {
+        const stored = await reader.read(p, 2);
+        if (stored.length < 2)
+            throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Truncated, 'The file ends inside a gzip header', { what: 'gzip header' });
+        const crc = (0, crc32_1.crc32)(await reader.read(at, p - at)) & 0xffff;
+        if ((stored[0] | (stored[1] << 8)) !== crc)
+            notes?.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MalformedHeader, data: { reason: 'the gzip header CRC is wrong, which gzip refuses' } });
+        p += 2;
+    }
+    if (p > reader.size)
+        throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Truncated, 'The file ends inside a gzip header', { what: 'gzip header' });
+    header.dataStart = p;
+    return header;
+}
+class GzipArchive extends Single {
+    header;
+    format = 'gzip';
+    constructor(reader, notes, header) {
+        super(reader, notes, header.name ?? '');
+        this.header = header;
+        if (header.comment !== undefined)
+            this.comment = header.comment;
+    }
+    hasMetadata() {
+        return this.header.mtime !== 0 || this.header.os !== 255 || this.comment !== undefined;
+    }
+    static async open(source) {
+        const reader = new io_1.Reader(source, await source.size());
+        const notes = [];
+        const header = await gzipHeader(reader, 0, notes);
+        return new GzipArchive(reader, notes, header);
+    }
+    async *read(e, meter) {
+        const size = this.reader.size;
+        let at = 0;
+        let members = 0;
+        let total = 0;
+        let used = 0;
+        this.lateNotes.length = 0;
+        for (;;) {
+            members++;
+            const h = members === 1 ? this.header : await gzipHeader(this.reader, at, undefined);
+            const consumed = { bytes: 0 };
+            let crc = 0;
+            let length = 0;
+            for await (const c of (0, codecs_1.inflateRaw)(this.reader.chunks(h.dataStart, size - h.dataStart), consumed)) {
+                length += c.length;
+                total += c.length;
+                meter.add(total, used + consumed.bytes);
+                crc = (0, crc32_1.crc32)(c, crc);
+                yield c;
+            }
+            used += consumed.bytes;
+            const trailerAt = h.dataStart + consumed.bytes;
+            const t = await this.reader.read(trailerAt, 8);
+            if (t.length < 8)
+                throw new errors_1.DataError('the file ends inside a gzip trailer', true);
+            const view = new DataView(t.buffer, t.byteOffset, 8);
+            if (view.getUint32(0, true) !== crc)
+                throw new errors_1.DataError('a gzip member does not match its CRC');
+            if (view.getUint32(4, true) !== length % 2 ** 32)
+                throw new errors_1.DataError('a gzip member does not match its size');
+            at = trailerAt + 8;
+            if (members === 1)
+                this.raw = { start: 0, length: at };
+            const next = await this.reader.read(at, 2);
+            if (next.length < 2 || next[0] !== 0x1f || next[1] !== 0x8b)
+                break;
+        }
+        if (members > 1) {
+            this.raw = undefined;
+            this.lateNotes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MultipleStreams, data: { count: members } });
+        }
+        this.trailingStart = at;
+        await this.trailing(at);
+        e.decoded = total;
+        e.size = total;
+    }
+    /** A member header with the name a rule fixed, and the metadata unless it is stripped. */
+    headerBytes(plan) {
+        const k = plan.kept[0];
+        const keepMeta = !plan.stripMetadata;
+        const name = k?.name || undefined;
+        const comment = keepMeta ? this.comment : undefined;
+        const out = Buffer.alloc(10);
+        out.set([0x1f, 0x8b, 8, (name !== undefined ? 0x08 : 0) | (comment !== undefined ? 0x10 : 0)]);
+        out.writeUInt32LE(keepMeta ? this.header.mtime : 0, 4);
+        out[8] = 0;
+        out[9] = keepMeta ? this.header.os : 255;
+        const field = (v) => Buffer.concat([Buffer.from(v.replace(/\0/g, '_'), 'latin1'), Buffer.alloc(1)]);
+        return Buffer.concat([out, ...(name !== undefined ? [field(name)] : []), ...(comment !== undefined ? [field(comment)] : [])]);
+    }
+    canStore = true;
+    async *encode(chunks, plan, store) {
+        yield this.headerBytes(plan);
+        let crc = 0;
+        let length = 0;
+        async function* tally() {
+            for await (const c of chunks) {
+                crc = (0, crc32_1.crc32)(c, crc);
+                length += c.length;
+                yield c;
+            }
+        }
+        yield* (0, codecs_1.deflateRaw)(tally(), store ? 0 : 6);
+        const trailer = Buffer.alloc(8);
+        trailer.writeUInt32LE(crc >>> 0, 0);
+        trailer.writeUInt32LE(length % 2 ** 32, 4);
+        yield trailer;
+    }
+    /** The header is written again, so the name a rule fixed and the metadata stripped apply also when the data is copied. */
+    async writeStream(plan, sink, context) {
+        const k = plan.kept[0];
+        const raw = this.raw;
+        if (!k || k.replacement || k.recompress || !raw)
+            return super.writeStream(plan, sink, context);
+        await sink.write(this.headerBytes(plan));
+        for await (const c of this.reader.chunks(this.header.dataStart, raw.length - this.header.dataStart)) {
+            context.meter().time();
+            await sink.write(c);
+        }
+    }
+}
+exports.GzipArchive = GzipArchive;
+// ---------------------------------------------------------------------------------------------------------------
+// bzip2
+class Bzip2Archive extends Single {
+    format = 'bzip2';
+    level;
+    constructor(reader, level) {
+        super(reader, []);
+        this.level = level;
+    }
+    static async open(source) {
+        const reader = new io_1.Reader(source, await source.size());
+        const h = await reader.read(0, 4);
+        if (h.length < 4 || h[0] !== 0x42 || h[1] !== 0x5a || h[2] !== 0x68 || h[3] < 0x31 || h[3] > 0x39)
+            throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, types_1.ArchiveDetail.Unparseable, 'Not a bzip2 stream', { reason: 'no bzip2 signature' });
+        return new Bzip2Archive(reader, h[3] - 0x30);
+    }
+    async *read(e, meter) {
+        const info = { streams: [], trailing: 0, trailingZeros: true };
+        const consumed = { bytes: 0 };
+        let total = 0;
+        this.lateNotes.length = 0;
+        for await (const c of (0, bzip2_1.bunzip2)(this.reader.chunks(0, this.reader.size), info, true, consumed)) {
+            total += c.length;
+            meter.add(total, consumed.bytes);
+            yield c;
+        }
+        const end = info.streams[info.streams.length - 1]?.end ?? 0;
+        this.raw = info.streams.length === 1 ? { start: 0, length: end } : undefined;
+        if (info.streams.length > 1)
+            this.lateNotes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MultipleStreams, data: { count: info.streams.length } });
+        this.trailingStart = end;
+        await this.trailing(end);
+        e.decoded = total;
+        e.size = total;
+    }
+    encode(chunks) {
+        return (0, bzip2_1.bzip2)(chunks, this.level);
+    }
+}
+exports.Bzip2Archive = Bzip2Archive;
+// ---------------------------------------------------------------------------------------------------------------
+// zstd
+class ZstdArchive extends Single {
+    layout;
+    format = 'zstd';
+    constructor(reader, notes, layout) {
+        super(reader, notes);
+        this.layout = layout;
+    }
+    static async open(source, deadline) {
+        const reader = new io_1.Reader(source, await source.size());
+        let layout;
+        try {
+            layout = await (0, zstd_1.zstdLayout)((o, n) => reader.read(o, n), 0, reader.size, false, () => (0, errors_1.checkTime)(deadline));
+        }
+        catch (err) {
+            if (err instanceof errors_1.UnsupportedError)
+                throw new errors_1.StopError(types_1.ArchiveCategory.Compression, types_1.ArchiveDetail.UnsupportedMethod, err.message, { method: 'zstd', reason: err.message });
+            if (err instanceof errors_1.DataError)
+                throw new errors_1.StopError(types_1.ArchiveCategory.Corrupted, err.truncated ? types_1.ArchiveDetail.Truncated : types_1.ArchiveDetail.Unparseable, err.message, err.truncated ? { what: 'zstd frame' } : { reason: err.message });
+            throw err;
+        }
+        if (!codecs_1.zstdDecompressor)
+            throw new errors_1.StopError(types_1.ArchiveCategory.Compression, types_1.ArchiveDetail.Unavailable, 'This Node has no zstd', { method: 'zstd' });
+        const notes = [];
+        const frames = layout.frames.filter(f => !f.skippable);
+        const skipped = layout.frames.filter(f => f.skippable);
+        if (skipped.length)
+            notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.HiddenData, data: { bytes: skipped.reduce((s, f) => s + f.end - f.start, 0), reason: 'skippable zstd frames' } });
+        if (frames.length > 1)
+            notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MultipleStreams, data: { count: frames.length } });
+        const a = new ZstdArchive(reader, notes, layout);
+        const last = layout.frames[layout.frames.length - 1];
+        a.trailingStart = last?.end ?? 0;
+        if (frames.length === 1 && !skipped.length)
+            a.raw = { start: frames[0].start, length: frames[0].end - frames[0].start };
+        const declared = frames.every(f => f.contentSize !== undefined) ? frames.reduce((s, f) => s + (f.contentSize ?? 0), 0) : undefined;
+        if (declared !== undefined)
+            a.entries[0].size = declared;
+        return a;
+    }
+    async *read(e, meter) {
+        const make = codecs_1.zstdDecompressor;
+        if (!make)
+            throw new errors_1.StopError(types_1.ArchiveCategory.Compression, types_1.ArchiveDetail.Unavailable, 'This Node has no zstd', { method: 'zstd' });
+        this.lateNotes.length = 0;
+        let total = 0;
+        let base = 0;
+        for (const f of this.layout.frames) {
+            if (f.skippable)
+                continue;
+            const consumed = { bytes: 0 };
+            let out = 0;
+            for await (const c of (0, codecs_1.through)(make(), this.reader.chunks(f.start, f.end - f.start), consumed)) {
+                out += c.length;
+                total += c.length;
+                if (f.contentSize !== undefined && out > f.contentSize)
+                    throw new errors_1.DataError('a zstd frame decodes past its declared size');
+                meter.add(total, base + consumed.bytes);
+                yield c;
+            }
+            if (consumed.bytes !== f.end - f.start)
+                throw new errors_1.DataError('a zstd frame ends before its blocks do', true);
+            if (f.contentSize !== undefined && out !== f.contentSize)
+                throw new errors_1.DataError('a zstd frame decodes to less than its declared size', true);
+            base += f.end - f.start;
+        }
+        await this.trailing(this.trailingStart ?? this.reader.size);
+        e.decoded = total;
+        e.size = total;
+    }
+    encode(chunks) {
+        return (0, zstd_1.zstdEncode)(chunks);
+    }
+    /** Node's zstd stream can end with a second, empty frame after the one that holds the data, which is left out. */
+    async encodedLength(source, length) {
+        const layout = await (0, zstd_1.zstdLayout)((o, n) => source.read(o, n), 0, length, false, () => undefined);
+        const last = layout.frames[layout.frames.length - 1];
+        return layout.frames.length > 1 && last.contentSize === 0 ? last.start : length;
+    }
+}
+exports.ZstdArchive = ZstdArchive;
+// ---------------------------------------------------------------------------------------------------------------
+// Brotli
+class BrotliArchive extends Single {
+    format = 'brotli';
+    static async open(source) {
+        return new BrotliArchive(new io_1.Reader(source, await source.size()), []);
+    }
+    async *read(e, meter) {
+        this.lateNotes.length = 0;
+        const consumed = { bytes: 0 };
+        let total = 0;
+        for await (const c of (0, codecs_1.brotliDecode)(this.reader.chunks(0, this.reader.size), consumed)) {
+            total += c.length;
+            meter.add(total, consumed.bytes);
+            yield c;
+        }
+        this.raw = { start: 0, length: consumed.bytes };
+        this.trailingStart = consumed.bytes;
+        await this.trailing(consumed.bytes);
+        e.decoded = total;
+        e.size = total;
+    }
+    encode(chunks) {
+        return (0, codecs_1.brotliEncode)(chunks);
+    }
+}
+exports.BrotliArchive = BrotliArchive;
+/**
+ * True when the content decodes as one Brotli stream, which has no signature: the stream must end where the file
+ * does, or decode to more than 32 MiB without an error. A prefix that merely decodes is not enough, since an
+ * uncompressed meta-block can take megabytes of any bytes before the next header shows whether they are Brotli.
+ */
+async function looksLikeBrotli(source) {
+    const size = await source.size();
+    if (size === 0)
+        return false;
+    const consumed = { bytes: 0 };
+    let out = 0;
+    try {
+        for await (const c of (0, codecs_1.brotliDecode)(new io_1.Reader(source, size).chunks(0, size), consumed)) {
+            out += c.length;
+            if (out > 32 * 1024 * 1024)
+                return true;
+        }
+    }
+    catch {
+        return false;
+    }
+    return consumed.bytes === size;
+}
+const isGzip = (b) => b.length >= 3 && b[0] === 0x1f && b[1] === 0x8b && b[2] === 8;
+exports.isGzip = isGzip;
+const isBzip2 = (b) => b.length >= 4 && b[0] === 0x42 && b[1] === 0x5a && b[2] === 0x68 && b[3] >= 0x31 && b[3] <= 0x39;
+exports.isBzip2 = isBzip2;

package/dist/sniff.d.ts

@@ -0,0 +1,20 @@
+/**
+ * True when `block` is a tar header: its checksum field matches the sum of its bytes, counted unsigned or, as some old
+ * writers did, signed, with the field itself counted as spaces. An all-zero block is the end of an archive, not a header.
+ */
+export declare function isTarHeader(block: Uint8Array): boolean;
+/** Identifies a file type from its first bytes. Returns a MIME type or undefined. */
+export declare function sniffType(head: Uint8Array): string | undefined;
+/** Maps MIME types that share a container format, and common aliases of one type, onto one family. */
+export declare function family(mime: string): string;
+/**
+ * A file name's extension, lowercased. Windows drops trailing dots and spaces when it saves a file, so the extension
+ * of "page.html." is html.
+ */
+export declare function extensionOf(name?: string): string | undefined;
+export declare function typeFromName(name?: string): string | undefined;
+/**
+ * True when any two of the declared type, the name's extension and the sniffed content contradict each other, or
+ * when the declared type or the name claims a type sniffType recognizes and the content matched no signature.
+ */
+export declare function typesDisagree(declared: string | undefined, name: string | undefined, sniffed: string | undefined): boolean;