@patrickdk77/archive-defuse 0.1.1 → 0.1.3

package/dist/lzma1/range-encoder.js

@@ -1,41 +1,68 @@
 "use strict";
+// Copied from lzma1 0.6.1 (Apache-2.0, Copyright Filip Seman) and changed for archive-defuse. NOTICE in this folder lists the changes.
+/**
+ * Range encoder writing to its own growable buffer.
+ *
+ * Ported from XZ for Java (0BSD), `src/org/tukaani/xz/rangecoder/RangeEncoder.java`
+ * and `RangeEncoderToBuffer.java`, by Lasse Collin and Igor Pavlov.
+ *
+ * Integer adaptations (see the hazard checklist in `docs/verification.md`):
+ * - `low` needs 33 bits (32 bits plus a carry), so it is a regular number;
+ *   doubles represent integers up to 2^53 exactly. The carry test uses
+ *   division instead of `low >>> 32`.
+ * - `range` is an unsigned 32-bit value in a regular number: `<<` is
+ *   followed by `>>> 0`, and `(range & TOP_MASK) == 0` becomes an
+ *   unsigned `range < TOP_VALUE`.
+ * - Bitwise ops on values that may exceed 2^31 are kept off the signed
+ *   int32 path (`low`'s 24-bit wrap multiplies by 256 instead of
+ *   shifting).
+ */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.RangeEncoder = void 0;
+exports.createPriceTable = createPriceTable;
 exports.getBitPrice = getBitPrice;
 exports.getBitTreePrice = getBitTreePrice;
 exports.getReverseBitTreePrice = getReverseBitTreePrice;
 exports.getDirectBitsPrice = getDirectBitsPrice;
-// Copied from lzma1 (Apache-2.0, Copyright Filip Seman) and changed for lzma-defuse. NOTICE in this folder lists the changes.
 const range_coder_1 = require("./range-coder");
 const MOVE_REDUCING_BITS = 4;
 const BIT_PRICE_SHIFT_BITS = 4;
-/**
- * Price (approximate cost in 1/16 bits) of encoding a bit with a given
- * probability, indexed by `probability >>> MOVE_REDUCING_BITS`.
- */
-const PRICES = createPriceTable();
+// Bit prices in 1/16 bits: the price of a 0 bit, then the same prices
+// reversed, which are the prices of a 1 bit, at
+// (bit << 7) | (prob >>> MOVE_REDUCING_BITS). One lookup without the Java
+// code's bit flip was measured faster.
+const bitPrices = createBitPriceTable();
+function createBitPriceTable() {
+    const prices = createPriceTable();
+    const half = prices.length;
+    const table = new Uint32Array(2 * half);
+    for (let i = 0; i < half; ++i) {
+        table[i] = prices[i];
+        table[half + i] = prices[half - 1 - i];
+    }
+    return table;
+}
 function createPriceTable() {
-    const prices = new Uint32Array(range_coder_1.BIT_MODEL_TOTAL >>> MOVE_REDUCING_BITS);
+    const table = new Uint32Array(range_coder_1.BIT_MODEL_TOTAL >>> MOVE_REDUCING_BITS);
     for (let i = (1 << MOVE_REDUCING_BITS) / 2; i < range_coder_1.BIT_MODEL_TOTAL; i += 1 << MOVE_REDUCING_BITS) {
-        // Approximate -log2(i / BIT_MODEL_TOTAL) using repeated squaring.
         let w = i;
         let bitCount = 0;
         for (let j = 0; j < BIT_PRICE_SHIFT_BITS; ++j) {
             w *= w;
             bitCount <<= 1;
-            while (w >= 0x10000) {
+            while ((w & 0xffff0000) !== 0) {
                 w >>>= 1;
                 ++bitCount;
             }
         }
-        prices[i >> MOVE_REDUCING_BITS] = (range_coder_1.BIT_MODEL_TOTAL_BITS << BIT_PRICE_SHIFT_BITS) - 15 - bitCount;
+        table[i >> MOVE_REDUCING_BITS] = (range_coder_1.BIT_MODEL_TOTAL_BITS << BIT_PRICE_SHIFT_BITS) - 15 - bitCount;
     }
-    return prices;
+    return table;
 }
 function getBitPrice(prob, bit) {
-    return PRICES[(prob ^ (-bit & (range_coder_1.BIT_MODEL_TOTAL - 1))) >>> MOVE_REDUCING_BITS];
+    // NOTE: Unlike in encodeBit(), here bit must be 0 or 1.
+    return bitPrices[(bit << 7) | (prob >>> MOVE_REDUCING_BITS)];
 }
-/** Price of a `bits`-bit symbol coded MSB-first with the bit tree at `offset`. */
 function getBitTreePrice(probs, offset, bits, symbolIn) {
     let symbol = symbolIn;
     let price = 0;
@@ -47,42 +74,47 @@ function getBitTreePrice(probs, offset, bits, symbolIn) {
     } while (symbol !== 1);
     return price;
 }
-/** Price of a `bits`-bit symbol coded LSB-first with the bit tree at `offset`. */
 function getReverseBitTreePrice(probs, offset, bits, symbolIn) {
     let symbol = symbolIn;
     let price = 0;
     let index = 1;
-    for (let i = 0; i < bits; ++i) {
+    symbol |= 1 << bits;
+    do {
         const bit = symbol & 1;
         symbol >>>= 1;
         price += getBitPrice(probs[offset + index], bit);
         index = (index << 1) | bit;
-    }
+    } while (symbol !== 1);
     return price;
 }
-/** Price of bits coded with a fixed 50% probability. */
 function getDirectBitsPrice(count) {
     return count << BIT_PRICE_SHIFT_BITS;
 }
-/**
- * Range encoder writing to its own growable buffer.
- *
- * `low` needs 33 bits (32 bits plus a carry), so it is kept as a regular
- * number instead of an int32. Doubles represent integers up to 2^53 exactly.
- *
- * Performance: like `RangeDecoder`, the methods that encode several bits
- * keep `range` and `low` in local variables for the whole symbol.
- */
 class RangeEncoder {
     low = 0;
     range = 0xffffffff;
     cache = 0;
+    /**
+     * NOTE: int is OK for LZMA2 because a compressed chunk is not more
+     * than 64 KiB, but with LZMA1 there is no chunking so in theory
+     * cacheSize can grow very big. A JS number counts well past 2^32.
+     */
     cacheSize = 1;
+    // Instead of the fixed output array of RangeEncoderToBuffer, the
+    // buffer grows on demand and the written bytes are handed out with
+    // take()/finish(), because the compressed size isn't known in advance
+    // here (contract adaptation).
     buf;
     size = 0;
     constructor(initialCapacity) {
         this.buf = new Uint8Array(Math.max(64, initialCapacity));
     }
+    reset() {
+        this.low = 0;
+        this.range = 0xffffffff;
+        this.cache = 0x00;
+        this.cacheSize = 1;
+    }
     /** Appends bytes that are not range coded, e.g. a header. */
     writeBytes(bytes) {
         this.ensureCapacity(bytes.length);
@@ -97,16 +129,34 @@ class RangeEncoder {
     }
     /** Flushes all pending bytes and returns the rest of the output. */
     finish() {
-        let low = this.low;
         for (let i = 0; i < 5; ++i) {
-            low = this.shiftLow(low);
+            this.shiftLow();
         }
-        this.low = low;
         return this.size === this.buf.length ? this.buf : this.buf.slice(0, this.size);
     }
+    writeByte(b) {
+        this.ensureCapacity(1);
+        this.buf[this.size++] = b;
+    }
+    shiftLow() {
+        // (int)(low >>> 32), the pending carry: 0 or 1.
+        const lowHi = Math.floor(this.low / 0x100000000);
+        if (lowHi !== 0 || this.low < 0xff000000) {
+            let temp = this.cache;
+            do {
+                this.writeByte(temp + lowHi);
+                temp = 0xff;
+            } while (--this.cacheSize !== 0);
+            this.cache = (this.low >>> 24) & 0xff;
+        }
+        ++this.cacheSize;
+        // (low & 0x00FFFFFF) << 8, kept exact above 2^31 with * 256.
+        this.low = (this.low & 0x00ffffff) * 256;
+    }
     encodeBit(probs, index, bit) {
         const prob = probs[index];
         const bound = (this.range >>> range_coder_1.BIT_MODEL_TOTAL_BITS) * prob;
+        // NOTE: Any non-zero value for bit is taken as 1.
         if (bit === 0) {
             this.range = bound;
             probs[index] = prob + ((range_coder_1.BIT_MODEL_TOTAL - prob) >>> range_coder_1.MOVE_BITS);
@@ -118,145 +168,43 @@ class RangeEncoder {
         }
         if (this.range < range_coder_1.TOP_VALUE) {
             this.range = (this.range << 8) >>> 0;
-            this.low = this.shiftLow(this.low);
+            this.shiftLow();
         }
     }
-    /** Encodes a `bits`-bit symbol MSB-first with the bit tree at `offset`. */
     encodeBitTree(probs, offset, bits, symbol) {
-        let range = this.range;
-        let low = this.low;
         let index = 1;
-        for (let i = bits - 1; i >= 0; --i) {
-            const bit = (symbol >>> i) & 1;
-            const prob = probs[offset + index];
-            const bound = (range >>> range_coder_1.BIT_MODEL_TOTAL_BITS) * prob;
-            if (bit === 0) {
-                range = bound;
-                probs[offset + index] = prob + ((range_coder_1.BIT_MODEL_TOTAL - prob) >>> range_coder_1.MOVE_BITS);
-            }
-            else {
-                low += bound;
-                range -= bound;
-                probs[offset + index] = prob - (prob >>> range_coder_1.MOVE_BITS);
-            }
-            if (range < range_coder_1.TOP_VALUE) {
-                range = (range << 8) >>> 0;
-                low = this.shiftLow(low);
+        let mask = 1 << bits;
+        do {
+            mask >>>= 1;
+            const bit = symbol & mask;
+            this.encodeBit(probs, offset + index, bit);
+            index <<= 1;
+            if (bit !== 0) {
+                index |= 1;
             }
-            index = (index << 1) | bit;
-        }
-        this.range = range;
-        this.low = low;
+        } while (mask !== 1);
     }
-    /** Encodes a `bits`-bit symbol LSB-first with the bit tree at `offset`. */
     encodeReverseBitTree(probs, offset, bits, symbolIn) {
         let symbol = symbolIn;
-        let range = this.range;
-        let low = this.low;
         let index = 1;
-        for (let i = 0; i < bits; ++i) {
+        symbol |= 1 << bits;
+        do {
             const bit = symbol & 1;
             symbol >>>= 1;
-            const prob = probs[offset + index];
-            const bound = (range >>> range_coder_1.BIT_MODEL_TOTAL_BITS) * prob;
-            if (bit === 0) {
-                range = bound;
-                probs[offset + index] = prob + ((range_coder_1.BIT_MODEL_TOTAL - prob) >>> range_coder_1.MOVE_BITS);
-            }
-            else {
-                low += bound;
-                range -= bound;
-                probs[offset + index] = prob - (prob >>> range_coder_1.MOVE_BITS);
-            }
-            if (range < range_coder_1.TOP_VALUE) {
-                range = (range << 8) >>> 0;
-                low = this.shiftLow(low);
-            }
+            this.encodeBit(probs, offset + index, bit);
             index = (index << 1) | bit;
-        }
-        this.range = range;
-        this.low = low;
-    }
-    /** Encodes a literal byte with the 0x100-leaf bit tree at `offset`. */
-    encodeLiteral(probs, offset, symbol) {
-        this.encodeBitTree(probs, offset, 8, symbol);
+        } while (symbol !== 1);
     }
-    /**
-     * Encodes a literal after a match. The bits of `matchByte` (the byte at
-     * rep0) select separate probabilities until the first bit that differs.
-     */
-    encodeMatchedLiteral(probs, offset, symbolIn, matchByteIn) {
-        let symbol = symbolIn;
-        let matchByte = matchByteIn;
-        let range = this.range;
-        let low = this.low;
-        let matchOffset = 0x100;
-        symbol |= 0x100;
-        do {
-            matchByte <<= 1;
-            const matchBit = matchByte & matchOffset;
-            const index = offset + matchOffset + matchBit + (symbol >>> 8);
-            const bit = (symbol >>> 7) & 1;
-            const prob = probs[index];
-            const bound = (range >>> range_coder_1.BIT_MODEL_TOTAL_BITS) * prob;
-            if (bit === 0) {
-                range = bound;
-                probs[index] = prob + ((range_coder_1.BIT_MODEL_TOTAL - prob) >>> range_coder_1.MOVE_BITS);
-            }
-            else {
-                low += bound;
-                range -= bound;
-                probs[index] = prob - (prob >>> range_coder_1.MOVE_BITS);
-            }
-            if (range < range_coder_1.TOP_VALUE) {
-                range = (range << 8) >>> 0;
-                low = this.shiftLow(low);
-            }
-            symbol <<= 1;
-            matchOffset &= ~(matchByte ^ symbol);
-        } while (symbol < 0x10000);
-        this.range = range;
-        this.low = low;
-    }
-    /** Encodes the `count` low bits of `value` with a fixed 50% probability. */
     encodeDirectBits(value, countIn) {
         let count = countIn;
-        let range = this.range;
-        let low = this.low;
         do {
-            range >>>= 1;
-            if (((value >>> --count) & 1) !== 0) {
-                low += range;
-            }
-            if (range < range_coder_1.TOP_VALUE) {
-                range = (range << 8) >>> 0;
-                low = this.shiftLow(low);
+            this.range >>>= 1;
+            this.low += this.range & (0 - ((value >>> --count) & 1));
+            if (this.range < range_coder_1.TOP_VALUE) {
+                this.range = (this.range << 8) >>> 0;
+                this.shiftLow();
             }
         } while (count !== 0);
-        this.range = range;
-        this.low = low;
-    }
-    /**
-     * Moves the top byte of `low` to the output and returns the new `low`.
-     * A run of 0xFF bytes is held back (`cache` + `cacheSize`) until it is
-     * known whether a carry will propagate into it.
-     */
-    shiftLow(low) {
-        if (low < 0xff000000 || low >= 0x100000000) {
-            const carry = low >= 0x100000000 ? 1 : 0;
-            this.ensureCapacity(this.cacheSize);
-            const buf = this.buf;
-            let size = this.size;
-            buf[size++] = (this.cache + carry) & 0xff;
-            for (let i = 1; i < this.cacheSize; ++i) {
-                buf[size++] = (0xff + carry) & 0xff;
-            }
-            this.size = size;
-            this.cacheSize = 0;
-            this.cache = (low >>> 24) & 0xff;
-        }
-        ++this.cacheSize;
-        return (low & 0x00ffffff) * 256;
     }
     ensureCapacity(extra) {
         if (this.size + extra > this.buf.length) {

package/dist/pack.d.ts

@@ -13,6 +13,13 @@ export interface PackEntry {
      * a caller open a stream only when its turn comes.
      */
     data?: PackData | (() => PackData | Promise<PackData>);
+    /**
+     * A file's size in bytes, for content given as a stream whose length the caller knows, such as a file in a database
+     * that records it. tar writes such a file as its data arrives instead of reading it into memory or a temporary file
+     * first, and streaming ZIP gives it ZIP64 fields only when its size needs them. Content that comes to another size
+     * fails the archive.
+     */
+    size?: number;
     /** The target of a symbolic link. */
     linkTarget?: string;
     /** Permission bits. Defaults to 0644 for a file and 0755 for a folder. */
@@ -44,8 +51,9 @@ export interface PackOptions {
     /** Folder for temporary files. Defaults to `os.tmpdir()`. */
     tempDir?: string;
     /**
-     * Bytes of one entry held in memory before a temporary file takes over, default 8 MiB. ZIP, tar and 7z need an
-     * entry's size before its data, so an entry given as a stream is read into memory or a temporary file first.
+     * Bytes of one entry held in memory before a temporary file takes over, default 8 MiB. An entry given as a stream
+     * is read into memory or a temporary file before it is written when the format needs something of it first: its
+     * CRC and sizes for ZIP without `streaming` and for 7z, and its size for tar when the entry gives no `size`.
      */
     memoryThreshold?: number;
     /** Stops the work when aborted: the sink is aborted and the call throws the signal's reason. */
@@ -54,7 +62,10 @@ export interface PackOptions {
      * ZIP only: write each file as its data arrives, deflated on the way, with its CRC and sizes in a data descriptor
      * after the data. Nothing goes to a temporary file. An entry with `store: true` is held in memory, since a stored
      * entry keeps its sizes in its header. `ratioLimit` does not apply, since an entry is written before its ratio is
-     * known.
+     * known. A file whose size is known, from its `size` or because its content is bytes or text, gets ZIP64 fields in
+     * its local header only when deflating it could pass 4 GiB. A stream without a size gets none, since some readers
+     * refuse ZIP64 fields an entry does not need; should it reach 4 GiB, its data descriptor holds eight-byte sizes, as
+     * archiver writes one. tar writes a file of known size as it arrives without this option.
      */
     streaming?: boolean;
 }
@@ -71,6 +82,31 @@ export declare function createArchive(format: PackFormat, entries: Iterable<Pack
 export declare function createArchiveStream(format: PackFormat, entries: Iterable<PackEntry> | AsyncIterable<PackEntry>, options?: PackOptions): Readable;
 /** `createArchive` into memory. */
 export declare function createArchiveBytes(format: PackFormat, entries: Iterable<PackEntry> | AsyncIterable<PackEntry>, options?: PackOptions): Promise<Uint8Array>;
+/** An archive that entries are added to one at a time, as archiver's `append` and `finalize` work. */
+export interface ArchiveWriter {
+    /**
+     * The archive as it is written, to read or pipe while entries are added. Entries wait in memory until it is read.
+     * An error destroys it, so a reader never takes a partial archive for a whole one.
+     */
+    readonly stream: Readable;
+    /**
+     * Adds an entry after the ones added before it. Its data is read when its turn comes, and a function given as its
+     * data is called then. Throws a PackError once `finish` or `abort` has been called or the archive has failed.
+     */
+    add(entry: PackEntry): void;
+    /**
+     * Says no more entries come. Resolves once the end of the archive is written to `stream`, and rejects with the error
+     * the archive failed on. The end is written only as `stream` is read, so keep reading it while this waits.
+     */
+    finish(): Promise<void>;
+    /** Stops the archive: `stream` is destroyed with `reason`, and `finish` rejects with it. Does nothing once it has finished. */
+    abort(reason?: unknown): void;
+}
+/**
+ * `createArchiveStream` for a caller that finds its entries as it goes. An entry `createArchive` refuses, or data that
+ * fails, fails the archive: `finish` rejects with the error and `stream` is destroyed with it.
+ */
+export declare function createArchiveWriter(format: PackFormat, options?: PackOptions): ArchiveWriter;
 /** Compresses `data` as one stream of `format` to `sink`, then closes it. */
 export declare function compressTo(format: CompressFormat, data: PackData, sink: ByteSink, options?: PackOptions): Promise<void>;
 /** `compressTo` as a Node Readable. */