@patrickdk77/archive-defuse 0.0.0-stage → 0.1.1

package/dist/lzma-formats.js

@@ -0,0 +1,373 @@
+"use strict";
+// The formats that came from lzma-defuse when it merged into this package: 7z, xz and .lzma. The readers and writers
+// are lzma-defuse's; these classes put them behind the Archive interface the engine drives, so 7z entries go through
+// the same rules and plugins as ZIP and tar entries.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LzmaArchive = exports.XzArchive = exports.SevenZipArchive = void 0;
+const alone_1 = require("./alone");
+const checks_1 = require("./checks");
+const errors_1 = require("./errors");
+const io_1 = require("./io");
+const lzma_sniff_1 = require("./lzma-sniff");
+const lzma_writers_1 = require("./lzma-writers");
+const model_1 = require("./model");
+const seven_1 = require("./seven");
+const sniff_1 = require("./sniff");
+const types_1 = require("./types");
+const xz_1 = require("./xz");
+/** Notes of one kind as one note with a count, so a file of a million xz streams does not make a million findings. */
+function grouped(notes) {
+    const groups = new Map();
+    for (const n of notes) {
+        const k = `${n.category}/${n.detail}`;
+        const g = groups.get(k) ?? [];
+        g.push(n);
+        groups.set(k, g);
+    }
+    return [...groups.values()].map(g => (g.length > 1 ? { ...g[0], data: { ...g[0].data, count: g.length } } : g[0]));
+}
+/**
+ * Notes a ZIP end record that a ZIP reader would find from the end of the file, and a PDF header in the first 1024
+ * bytes, where a PDF reader takes one. `trailingAt` is where data after the format's own structures starts.
+ */
+async function otherReaders(reader, start, trailingAt, notes) {
+    const size = reader.size;
+    const last = await reader.read(Math.max(0, size - 65_557), Math.min(size, 65_557));
+    for (let i = last.length - 22; i >= 0; i--) {
+        if (last[i] === 0x50 && last[i + 1] === 0x4b && last[i + 2] === 0x05 && last[i + 3] === 0x06 && i + 22 + (last[i + 20] | (last[i + 21] << 8)) === last.length) {
+            const at = size - last.length + i;
+            notes.push({
+                category: types_1.ArchiveCategory.Structure,
+                detail: types_1.ArchiveDetail.Polyglot,
+                data: { type: 'application/zip', where: trailingAt !== undefined && at >= trailingAt ? 'after the archive' : 'at the end of the file' },
+            });
+            break;
+        }
+    }
+    const first = await reader.read(start, 1024);
+    if (Buffer.from(first.buffer, first.byteOffset, first.byteLength).toString('latin1').includes('%PDF-')) {
+        notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.Polyglot, data: { type: 'application/pdf', where: 'in the first 1024 bytes' } });
+    }
+}
+/** Notes data after the format's own structures, and what it reads as. */
+async function trailing(reader, at, notes) {
+    if (at === undefined || at >= reader.size)
+        return;
+    notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.TrailingBytes, data: { bytes: reader.size - at } });
+    const type = (0, sniff_1.sniffType)(await reader.read(at, Math.min(1024, reader.size - at)));
+    if (type !== undefined)
+        notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.Polyglot, data: { type, where: 'after the archive' } });
+}
+/** Copies a range of the source to the sink. */
+async function copyRange(reader, sink, from, to, time) {
+    for await (const c of reader.chunks(from, to - from)) {
+        time();
+        await sink.write(c);
+    }
+}
+const KINDS = { file: 'file', dir: 'directory', symlink: 'symlink', device: 'device', fifo: 'fifo', socket: 'socket' };
+/** No system takes a link target longer than PATH_MAX, 4096 bytes on Linux. */
+const MAX_LINK = 4096;
+/** `s` with each lone surrogate, which no file system name holds, read as U+FFFD, as a UTF-16 decoder reads it. */
+function wellFormed(s) {
+    let out = '';
+    let from = 0;
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        if (c < 0xd800 || c > 0xdfff)
+            continue;
+        const next = s.charCodeAt(i + 1);
+        if (c <= 0xdbff && next >= 0xdc00 && next <= 0xdfff) {
+            i++;
+            continue;
+        }
+        out += `${s.slice(from, i)}\ufffd`;
+        from = i + 1;
+    }
+    return from === 0 ? s : out + s.slice(from);
+}
+class SevenZipArchive {
+    reader;
+    seven;
+    notes;
+    passwords;
+    format = '7z';
+    single = false;
+    nameless;
+    lateNotes = [];
+    entries;
+    contents;
+    constructor(reader, seven, notes, passwords, env) {
+        this.reader = reader;
+        this.seven = seven;
+        this.notes = notes;
+        this.passwords = passwords;
+        this.contents = new seven_1.SevenReader(env, seven);
+        this.nameless = seven.entries.length > 0 && seven.entries.every(s => s.info.name === undefined);
+        this.entries = seven.entries.map(s => {
+            const kind = KINDS[s.kind];
+            const raw = s.info.name ?? '';
+            const name = wellFormed(raw);
+            const entryNotes = name === raw ? [] : [{ category: types_1.ArchiveCategory.Name, detail: types_1.ArchiveDetail.Encoding, data: { name } }];
+            const folder = s.folder === undefined ? undefined : seven.streams?.folders[s.folder];
+            const encrypted = folder?.coders.some(c => c.method === '06f10701') === true;
+            const full = (0, seven_1.unixMode)(s.info.attrib);
+            return {
+                index: s.index,
+                name: kind === 'directory' && name !== '' && !name.endsWith('/') ? `${name}/` : name,
+                kind,
+                mode: full === undefined ? undefined : full & 0o7777,
+                size: s.size,
+                compressedSize: 0,
+                encryption: encrypted ? { kind: 'aes', bits: 256 } : undefined,
+                notes: entryNotes,
+                anti: s.info.anti === true,
+                seven: s,
+            };
+        });
+    }
+    static async open(source, limits, deadline, passwords) {
+        const size = await source.size();
+        const reader = new io_1.Reader(source, size);
+        const found = (0, lzma_sniff_1.detectLzmaFormat)(await reader.read(0, Math.min(size, lzma_sniff_1.SEVENZIP_SEARCH)));
+        if (found?.format !== '7z')
+            throw new errors_1.DataError('no 7z signature header');
+        const env = { source, reader, size, limits, totals: { decoded: 0, compressed: 0 }, deadline, passwords };
+        const seven = await (0, seven_1.openSevenZip)(env, found.offset);
+        const notes = grouped(seven.notes);
+        if (seven.headerEncrypted)
+            notes.push({ category: types_1.ArchiveCategory.Encrypted, detail: types_1.ArchiveDetail.Header });
+        if (found.offset > 0) {
+            notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.LeadingBytes, data: { bytes: found.offset } });
+            const type = (0, sniff_1.sniffType)(await reader.read(0, Math.min(found.offset, 1024)));
+            if (type !== undefined)
+                notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.Polyglot, data: { type, where: 'before the archive' } });
+        }
+        await trailing(reader, seven.trailingAt, notes);
+        await otherReaders(reader, found.offset, seven.trailingAt, notes);
+        return new SevenZipArchive(reader, seven, notes, passwords, env);
+    }
+    hasMetadata() {
+        return this.seven.entries.some(e => e.info.ctime || e.info.atime || e.info.mtime);
+    }
+    async unlock(e, password) {
+        return this.contents.unlock(e.seven, password);
+    }
+    async *read(e, meter, password) {
+        let total = 0;
+        for await (const c of this.contents.content(e.seven, password)) {
+            total += c.length;
+            meter.add(total);
+            yield c;
+        }
+    }
+    /** A symbolic link's target, which a 7z archive keeps as the link's content. */
+    async linkTarget(e, meter) {
+        // A Windows reparse point holds a reparse buffer, not a path, so where it leads cannot be read as a path.
+        if (e.size > MAX_LINK || (0, seven_1.unixMode)(e.seven.info.attrib) === undefined)
+            return undefined;
+        let password;
+        if (e.encryption) {
+            for (const p of this.passwords) {
+                if (await this.unlock(e, p)) {
+                    password = p;
+                    break;
+                }
+            }
+            if (password === undefined)
+                return undefined;
+        }
+        const parts = [];
+        for await (const c of this.read(e, meter, password))
+            parts.push(c);
+        return Buffer.concat(parts).toString('utf8');
+    }
+    async write(plan, sink, context) {
+        const time = () => context.meter().time();
+        if (plan.keepLeading && this.seven.offset > 0)
+            await copyRange(this.reader, sink, 0, this.seven.offset, time);
+        const out = plan.kept.map(k => this.out(k, plan, context));
+        await (0, lzma_writers_1.writeSevenZip)(out, sink, context.temp, context.threshold, time, plan.ratioLimit);
+        await this.contents.close();
+        const end = this.seven.trailingAt;
+        if (plan.keepTrailing && end !== undefined)
+            await copyRange(this.reader, sink, end, this.reader.size, time);
+    }
+    /** One kept entry as the writer takes it, with its name and permissions as the rules left them. */
+    out(k, plan, context) {
+        const info = k.entry.seven.info;
+        const dir = k.entry.kind === 'directory';
+        let attrib = info.attrib;
+        const full = (0, seven_1.unixMode)(attrib);
+        if (full !== undefined && attrib !== undefined) {
+            const mode = ((full & ~0o7777) | ((k.mode ?? full) & 0o7777)) & 0xffff;
+            attrib = ((attrib & 0xffff) | (mode << 16)) >>> 0;
+        }
+        const size = (0, model_1.sizeOf)(k);
+        const content = dir || size === 0 ? undefined : { size, chunks: () => (0, model_1.contentOf)(this, k, context) };
+        const name = dir && k.name.endsWith('/') ? k.name.slice(0, -1) : k.name;
+        const o = { name: info.name === undefined ? undefined : name, dir, attrib, content };
+        if (!plan.stripMetadata) {
+            o.ctime = info.ctime;
+            o.atime = info.atime;
+            o.mtime = info.mtime;
+        }
+        return o;
+    }
+}
+exports.SevenZipArchive = SevenZipArchive;
+// ---------------------------------------------------------------------------------------------------------------
+// xz and .lzma
+/** The engine's meter in the shape the xz and .lzma readers count through: decoded bytes against compressed ones. */
+function counter(meter) {
+    let decoded = 0;
+    let compressed = 0;
+    return {
+        get decoded() {
+            return decoded;
+        },
+        get compressed() {
+            return compressed;
+        },
+        tick: () => meter.time(),
+        packed: n => {
+            compressed += n;
+        },
+        add: n => {
+            decoded += n;
+            meter.add(decoded, compressed);
+        },
+    };
+}
+/** A file that compresses one file: its one entry has no name, and its content goes to the plugins like any other. */
+class LzmaSingle {
+    reader;
+    limits;
+    single = true;
+    notes = [];
+    lateNotes = [];
+    entries;
+    /** Set once the entry has been read: where data after the stream starts, when the file goes on. */
+    trailingAt;
+    /** Set once the entry has been read and nothing about the stream needs writing again: it can be copied as it is. */
+    copyable = false;
+    constructor(reader, limits) {
+        this.reader = reader;
+        this.limits = limits;
+        this.entries = [{ index: 0, name: '', kind: 'file', size: 0, compressedSize: reader.size, notes: [] }];
+    }
+    async unlock() {
+        return false;
+    }
+    hasMetadata() {
+        return false;
+    }
+    /** True when encode() can store data uncompressed. */
+    canStore = false;
+    /** Notes what the read found besides the content, and whether the stream can be copied as it is. */
+    async finishRead(e, total, notes, oneStream, trailingAt) {
+        e.decoded = total;
+        e.size = total;
+        this.trailingAt = trailingAt;
+        this.lateNotes.length = 0;
+        this.lateNotes.push(...grouped(notes));
+        await trailing(this.reader, trailingAt, this.lateNotes);
+        await otherReaders(this.reader, 0, trailingAt, this.lateNotes);
+        this.copyable = oneStream && this.lateNotes.length === 0;
+    }
+    async write(plan, sink, context) {
+        const time = () => context.meter().time();
+        const k = plan.kept[0];
+        const end = this.trailingAt ?? this.reader.size;
+        if (k && !k.replacement && !k.recompress && this.copyable)
+            await copyRange(this.reader, sink, 0, end, time);
+        else
+            await this.writeContent(k, sink, context, time, plan.ratioLimit);
+        if (plan.keepTrailing && this.trailingAt !== undefined)
+            await copyRange(this.reader, sink, this.trailingAt, this.reader.size, time);
+    }
+    async writeContent(k, sink, context, time, ratioLimit) {
+        if (!k)
+            return this.encode(undefined, sink, time, false);
+        // The writer reads the content by offset, so content that is not a plugin's copy is decoded again into memory or
+        // a temporary file first.
+        const decoded = k.replacement ? undefined : new io_1.SpillSink(context.temp, context.threshold);
+        const encoded = new io_1.SpillSink(context.temp, context.threshold);
+        try {
+            if (decoded) {
+                for await (const c of (0, model_1.contentOf)(this, k, context))
+                    await decoded.write(c);
+                await decoded.close();
+            }
+            const content = (decoded ?? k.replacement)?.source();
+            if (!content)
+                throw new Error('No content to write');
+            try {
+                await this.encode(content, encoded, time, false);
+                await encoded.close();
+                const size = await content.size();
+                // Content compressed past the ratio limit is stored where the format can store, so the output reads under the
+                // same limit.
+                if (this.canStore && (0, lzma_writers_1.passesRatio)(size, encoded.length, ratioLimit))
+                    await this.encode(content, sink, time, true);
+                else
+                    for await (const c of encoded.read())
+                        await sink.write(c);
+            }
+            finally {
+                await content.close?.();
+            }
+        }
+        finally {
+            await encoded.dispose();
+            await decoded?.dispose();
+        }
+    }
+}
+class XzArchive extends LzmaSingle {
+    format = 'xz';
+    /** The check the rewrite writes: the one every stream used when this package can compute it, else CRC64. */
+    checkId = checks_1.CHECK_CRC64;
+    static async open(source, limits) {
+        return new XzArchive(new io_1.Reader(source, await source.size()), limits);
+    }
+    async *read(e, meter) {
+        const info = { streams: 0, checks: [], notes: [] };
+        let total = 0;
+        for await (const c of (0, xz_1.readXz)({ reader: this.reader, size: this.reader.size, limits: this.limits, meter: counter(meter) }, info)) {
+            total += c.length;
+            yield c;
+        }
+        const notes = [...info.notes];
+        if (info.streams > 1)
+            notes.push({ category: types_1.ArchiveCategory.Structure, detail: types_1.ArchiveDetail.MultipleStreams, data: { count: info.streams } });
+        const checks = new Set(info.checks);
+        const only = checks.size === 1 ? [...checks][0] : undefined;
+        this.checkId = only !== undefined && lzma_writers_1.WRITABLE_CHECKS.has(only) ? only : checks_1.CHECK_CRC64;
+        await this.finishRead(e, total, notes, info.streams === 1, info.trailingAt);
+    }
+    canStore = true;
+    encode(content, sink, time, store) {
+        return (0, lzma_writers_1.writeXz)(content, this.checkId, sink, time, store);
+    }
+}
+exports.XzArchive = XzArchive;
+class LzmaArchive extends LzmaSingle {
+    format = 'lzma';
+    static async open(source, limits) {
+        return new LzmaArchive(new io_1.Reader(source, await source.size()), limits);
+    }
+    async *read(e, meter) {
+        const info = { notes: [] };
+        let total = 0;
+        for await (const c of (0, alone_1.readAlone)({ reader: this.reader, size: this.reader.size, limits: this.limits, meter: counter(meter) }, info)) {
+            total += c.length;
+            yield c;
+        }
+        await this.finishRead(e, total, info.notes, true, info.trailingAt);
+    }
+    encode(content, sink, time) {
+        return (0, lzma_writers_1.writeAlone)(content, sink, time);
+    }
+}
+exports.LzmaArchive = LzmaArchive;

package/dist/lzma-sniff.d.ts

@@ -0,0 +1,20 @@
+export declare const XZ_MAGIC: number[];
+export declare const SEVENZIP_MAGIC: number[];
+/**
+ * True when the 13 bytes at `at` are a header liblzma's .lzma decoder takes when it has to guess the format: valid
+ * properties with lc + lp at most 4, a dictionary of 2^n or 2^n + 2^(n-1) bytes, a size that is unknown or below 256
+ * GiB, and a first byte of 0 in the range coder data that follows.
+ */
+export declare function looksLikeLzma(b: Uint8Array, at?: number): boolean;
+/** True when a 7z signature header sits at `at`: the signature, version 0 and a start header whose CRC matches. */
+export declare function sevenZipHeaderAt(b: Uint8Array, at: number): boolean;
+/** How far into a file a 7z signature header is looked for, as self-extracting archives put a program first. */
+export declare const SEVENZIP_SEARCH: number;
+/**
+ * The format of a file from its first bytes: xz and 7z by their signatures at the start, then .lzma by its header.
+ * `head` should hold the first SEVENZIP_SEARCH bytes when a 7z archive may follow other data.
+ */
+export declare function detectLzmaFormat(head: Uint8Array): {
+    format: 'xz' | '7z' | 'lzma';
+    offset: number;
+} | undefined;

package/dist/lzma-sniff.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SEVENZIP_SEARCH = exports.SEVENZIP_MAGIC = exports.XZ_MAGIC = void 0;
+exports.looksLikeLzma = looksLikeLzma;
+exports.sevenZipHeaderAt = sevenZipHeaderAt;
+exports.detectLzmaFormat = detectLzmaFormat;
+// The signatures of xz, 7z and .lzma files. Copied from lzma-defuse src/sniff.ts when it merged into this package.
+const crc32_1 = require("./crc32");
+exports.XZ_MAGIC = [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00];
+exports.SEVENZIP_MAGIC = [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c];
+const startsWith = (b, at, bytes) => b.length >= at + bytes.length && bytes.every((x, i) => b[at + i] === x);
+/**
+ * True when the 13 bytes at `at` are a header liblzma's .lzma decoder takes when it has to guess the format: valid
+ * properties with lc + lp at most 4, a dictionary of 2^n or 2^n + 2^(n-1) bytes, a size that is unknown or below 256
+ * GiB, and a first byte of 0 in the range coder data that follows.
+ */
+function looksLikeLzma(b, at = 0) {
+    if (b.length < at + 14)
+        return false;
+    const props = b[at];
+    if (props >= 9 * 5 * 5)
+        return false;
+    const lc = props % 9;
+    const lp = Math.floor(props / 9) % 5;
+    if (lc + lp > 4)
+        return false;
+    const view = new DataView(b.buffer, b.byteOffset + at, 13);
+    const dict = view.getUint32(1, true);
+    if (dict !== 0xffffffff) {
+        // 2^n or 2^n + 2^(n-1): the highest set bit, plus at most the one below it.
+        let d = dict - 1;
+        d |= d >>> 2;
+        d |= d >>> 3;
+        d |= d >>> 4;
+        d |= d >>> 8;
+        d |= d >>> 16;
+        if ((d >>> 0) + 1 !== dict && dict !== 0)
+            return false;
+    }
+    const low = view.getUint32(5, true);
+    const high = view.getUint32(9, true);
+    if (!(low === 0xffffffff && high === 0xffffffff) && high >= 1 << 6)
+        return false;
+    return b[at + 13] === 0x00;
+}
+/** True when a 7z signature header sits at `at`: the signature, version 0 and a start header whose CRC matches. */
+function sevenZipHeaderAt(b, at) {
+    if (!startsWith(b, at, exports.SEVENZIP_MAGIC) || b.length < at + 32 || b[at + 6] !== 0)
+        return false;
+    const stored = new DataView(b.buffer, b.byteOffset + at + 8, 4).getUint32(0, true);
+    return (0, crc32_1.crc32)(b.subarray(at + 12, at + 32)) === stored;
+}
+/** How far into a file a 7z signature header is looked for, as self-extracting archives put a program first. */
+exports.SEVENZIP_SEARCH = 1024 * 1024;
+/**
+ * The format of a file from its first bytes: xz and 7z by their signatures at the start, then .lzma by its header.
+ * `head` should hold the first SEVENZIP_SEARCH bytes when a 7z archive may follow other data.
+ */
+function detectLzmaFormat(head) {
+    if (startsWith(head, 0, exports.XZ_MAGIC))
+        return { format: 'xz', offset: 0 };
+    if (startsWith(head, 0, exports.SEVENZIP_MAGIC))
+        return { format: '7z', offset: 0 };
+    if (looksLikeLzma(head))
+        return { format: 'lzma', offset: 0 };
+    // A 7z archive after other data, such as the program of a self-extracting archive.
+    const limit = Math.min(head.length, exports.SEVENZIP_SEARCH) - 32;
+    for (let at = head.indexOf(0x37, 1); at >= 0 && at <= limit; at = head.indexOf(0x37, at + 1)) {
+        if (sevenZipHeaderAt(head, at))
+            return { format: '7z', offset: at };
+    }
+    return undefined;
+}

package/dist/lzma-writers.d.ts

@@ -0,0 +1,35 @@
+import { type TempDir } from './io';
+import type { ByteSink, ByteSource } from './types';
+/** True when data compressed this far would pass the ratio limit, so the writer stores it instead. */
+export declare const passesRatio: (size: number, packed: number, ratioLimit: number) => boolean;
+/** Content to write: its size, known before it is read, and its bytes, read once. */
+export interface Content {
+    size: number;
+    chunks(): AsyncIterable<Uint8Array>;
+}
+/** The check IDs this package writes, the ones it can compute. */
+export declare const WRITABLE_CHECKS: Set<number>;
+/**
+ * Writes `content` as a single-stream xz file with one LZMA2 block, or no block when it is empty. The block's chunks
+ * come from Lzma2ChunkWriter. Chunks that would start within the first 1024 bytes of the file stay compressed.
+ */
+export declare function writeXz(content: ByteSource | undefined, checkId: number, sink: ByteSink, time: () => void, store?: boolean): Promise<void>;
+/** Writes `content` as an .lzma file with its size in the header and no end marker. */
+export declare function writeAlone(content: ByteSource | undefined, sink: ByteSink, time: () => void): Promise<void>;
+/** One entry of a 7z archive to write. */
+export interface SevenOut {
+    /** Absent for an entry of an archive that stores no names. */
+    name?: string;
+    dir: boolean;
+    attrib?: number;
+    ctime?: Uint8Array;
+    atime?: Uint8Array;
+    mtime?: Uint8Array;
+    /** The content of a file. Empty or absent for an empty file. */
+    content?: Content;
+}
+/**
+ * Writes a 7z archive: one LZMA folder per file with content, compressed by lzma1's encoder at its preset 1, a plain
+ * header with the names, empty files and directories, attributes and the times given, and CRCs of every file.
+ */
+export declare function writeSevenZip(entries: SevenOut[], sink: ByteSink, temp: TempDir, memoryThreshold: number, time: () => void, ratioLimit?: number): Promise<void>;