node-pkware 2.0.0 → 3.0.1

package/README.md

@@ -1,30 +1,36 @@
 # node-pkware
 
-Node JS implementation of StormLib's Pkware compression/decompression algorithm
+Node JS implementation of StormLib's Pkware compression/decompression algorithm, which is not the same as the zip pkware
+format that is commonly used today
 
-It was the de-facto compression for games from around Y2K, like Arx Fatalis
+It was the de-facto compression for games from around Y2K, like [Arx Fatalis](https://en.wikipedia.org/wiki/Arx_Fatalis)
 
 ## installation / update existing version
 
 `npm i -g node-pkware`
 
-minimum required node version: 8.5+
+minimum required node version: 18.0.0
 
-development and testing should be done in node 12.3+ because of the tests utilizing `Readable.from()` - source: https://stackoverflow.com/a/59638132/1806628
+## command line interface (CLI)
 
-tested in node version 14.9.0
+`explode [<filename>] [--offset=<offset>] [--drop-before-offset] [--output=<filename> [--verbose]]` - decompresses a file or a stream
 
-## command line interface
+`implode [<filename>] <compression type> <dictionary size> [--offset=<offset>] [--drop-before-offset] [--output=<filename> [--verbose]]` - compresses a file or a stream
 
-`implode <filename> --output=<filename> --ascii|-a|--binary|-b --small|-s|--medium|-m|--large|-l` - compresses file. If `--output` is omitted, then output will be placed next to input and names as `<filename>.compressed`. Optionally you can specify an offset from which the compressed data starts with the `--offset=<int|hex>`, which is useful for mixed files, such as the fts files of Arx Fatalis
+`<filename>`, `--output` or both can be omitted when the input is being piped from stdin or when the output is being piped into stdout
 
-`explode <filename> --output=<filename>` - decompresses file. If `--output` is omitted, then output will be placed next to input and names as `<filename>.decompressed`. Optionally you can specify an offset from which the compressed data starts with the `--offset=<int|hex>`, which is useful for mixed files, such as the fts files of Arx Fatalis
+The `--offset` can have a numeric value in either decimal or hexadecimal format which tells explode or implode to start decompression at a later point.
+This is useful for partially compressed files where the initial header part is uncompressed while the remaining part is compressed.
 
 The `--drop-before-offset` flag tells node-pkware to drop the portion before `--offset`, otherwise it will keep it untouched and attach it to the output file.
 
-**currently disabled, WIP** There is an `--auto-detect` flag, which will search for the first pkware header starting from the beginning of the file. If `--offset` is defined, then it will start searching from that point.
+The `--verbose` flag will display additional information while running the commands
 
-Calling either explode or implode with the `-v` or `--version` flag will display the package's version
+For implode `<compression type>` can either be `--ascii` or `--binary`
+
+For implode `<dictionary size>` can either be `--small`, `--medium` or `--large`
+
+Calling either explode or implode with only the `-v` or `--version` flag will display the package's version
 
 ## examples
 
@@ -34,14 +40,8 @@ Calling either explode or implode with the `-v` or `--version` flag will display
 
 `implode test/files/fast.fts.unpacked --output=C:/fast.fts --binary --large --offset=1816`
 
-`explode test/files/fast.fts --auto-detect --verbose --output=E:/fast.fts.unpacked`
-
-`explode test/files/fast.fts --auto-detect --verbose --output=E:/fast.fts.unpacked --offset=2000`
-
 ### piping also works
 
-**Don't use --verbose when piping, because verbose messages will be outputted to where the decompressed data is being outputted!**
-
 `cat c:/arx/level8.llf | explode > c:/arx/level8.llf.unpacked`
 
 `explode c:/arx/level8.llf > c:/arx/level8.llf.unpacked`
@@ -60,35 +60,37 @@ Calling either explode or implode with the `-v` or `--version` flag will display
 
 `explode(config: object): transform._transform` - decompresses stream
 
-Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method. The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
+`decompress(config: object): transform._transform` - alias for explode
+
+Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method.
+The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
 
 Takes an optional config object, which has the following properties:
 
 ```js
 {
   verbose: boolean, // whether the code should display extra debug messages on the console or not (default = false)
-  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)
+  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have positive performance impact (default 0)
   outputBufferSize: int // same as inputBufferSize, but for the outputBuffer (default 0)
 }
 ```
 
-`decompress(config: object): transform._transform` - alias for explode
-
 `implode(compressionType: int, dictionarySize: int, config: object): transform._transform` - compresses stream
 
+`compress(compressionType: int, dictionarySize: int, config: object): transform._transform` - alias for implode
+
 Takes an optional config object, which has the following properties:
 
 ```js
 {
   verbose: boolean, // whether the code should display extra debug messages on the console or not (default = false)
-  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)
+  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have positive performance impact (default 0)
   outputBufferSize: int // same as inputBufferSize, but for the outputBuffer (default 0)
 }
 ```
 
-Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method. The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
-
-`compress(compressionType: int, dictionarySize: int, config: object): transform._transform` - aliasa for implode
+Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method.
+The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
 
 `stream` - an object of helper functions for channeling streams to and from explode/implode
 
@@ -102,7 +104,7 @@ Returns a function, that you can use as a [transform.\_transform](https://nodejs
 
 ```
 1) [inputBuffer, emptyBuffer, false]
-2) [inputBuffer.slice(0, 40), inputBuffer.slice(40, 60), true]
+2) [inputBuffer.subarray(0, 40), inputBuffer.subarray(40, 60), true]
 3) [emptyBuffer, inputBuffer, true]
 4) [emptyBuffer, inputBuffer, true]
 ... and so on
@@ -110,11 +112,11 @@ Returns a function, that you can use as a [transform.\_transform](https://nodejs
 
 `stream.transformSplitBy(predicate: predicate, left: transform._transform, right: transform._transform): transform._transform` - higher order function for introducing conditional logic to transform.\_transform functions. This is used internally to handle offsets for explode()
 
-`stream.streamToBuffer(callback: function): writable._write` - data can be piped to the returned function from a stream and it will concatenate all chunks into a single buffer. Takes a callback function, which will receive the concatenated buffer as a parameter
+`stream.toBuffer(callback: function): writable._write` - data can be piped to the returned function from a stream and it will concatenate all chunks into a single buffer. Takes a callback function, which will receive the concatenated buffer as a parameter
 
-`constants.COMPRESSION_BINARY` and `constants.COMPRESSION_ASCII` - compression types for implode
+`constants.Compression.Binary` and `constants.Compression.Ascii` - compression types for implode
 
-`constants.DICTIONARY_SIZE_SMALL`, `constants.DICTIONARY_SIZE_MEDIUM` and `constants.DICTIONARY_SIZE_LARGE` - dictionary sizes for implode, determines how well the file get compressed. Small dictionary size means less memory to lookback in data for repetitions, meaning it will be less effective, the file stays larger, less compressed. On the other hand, large compression allows more lookback allowing more effective compression, thus generating smaller, more compressed files.
+`constants.DictionarySize.Small`, `constants.DictionarySize.Medium` and `constants.DictionarySize.Large` - dictionary sizes for implode, determines how well the file get compressed. Small dictionary size means less memory to lookback in data for repetitions, meaning it will be less effective, the file stays larger, less compressed. On the other hand, large compression allows more lookback allowing more effective compression, thus generating smaller, more compressed files. The original C library used less memory when the dictionary size was smaller, plus there might be files out there which only support smaller dictionary sizes
 
 `errors.InvalidDictionarySizeError` - thrown by implode when invalid dictionary size was specified or by explode when it encounters invalid data in the header section (the first 2 bytes of a compressed files)
 
@@ -128,8 +130,8 @@ Returns a function, that you can use as a [transform.\_transform](https://nodejs
 
 #### decompressing file with no offset into a file
 
-```javascript
-const fs = require('fs')
+```js
+const fs = require('node:fs')
 const { explode, stream } = require('node-pkware')
 const { through } = stream
 
@@ -140,15 +142,15 @@ fs.createReadStream(`path-to-compressed-file`)
 
 #### decompressing buffer with no offset into a buffer
 
-```javascript
-const { Readable } = require('stream')
+```js
+const { Readable } = require('node:stream')
 const { explode, stream } = require('node-pkware')
-const { through, streamToBuffer } = stream
+const { through, toBuffer } = stream
 
 Readable.from(buffer) // buffer is of type Buffer with compressed data
   .pipe(through(explode()))
   .pipe(
-    streamToBuffer((decompressedData) => {
+    toBuffer((decompressedData) => {
       // decompressedData holds the decompressed buffer
     }),
   )
@@ -156,8 +158,8 @@ Readable.from(buffer) // buffer is of type Buffer with compressed data
 
 #### decompressing file with offset into a file, keeping initial part intact
 
-```javascript
-const fs = require('fs')
+```js
+const fs = require('node:fs')
 const { explode, stream } = require('node-pkware')
 const { through, transformSplitBy, splitAt, transformIdentity } = stream
 
@@ -170,8 +172,8 @@ fs.createReadStream(`path-to-compressed-file`)
 
 #### decompressing file with offset into a file, discarding initial part
 
-```javascript
-const fs = require('fs')
+```js
+const fs = require('node:fs')
 const { explode, stream } = require('node-pkware')
 const { through, transformSplitBy, splitAt, transformEmpty } = stream
 
@@ -184,8 +186,8 @@ fs.createReadStream(`path-to-compressed-file`)
 
 ### Catching errors
 
-```javascript
-const fs = require('fs')
+```js
+const fs = require('node:fs')
 const { explode, stream } = require('node-pkware')
 const { through } = stream
 

package/dist/ExpandingBuffer.d.ts

@@ -0,0 +1,53 @@
+/// <reference types="node" />
+import { Buffer } from 'node:buffer';
+export declare class ExpandingBuffer {
+    #private;
+    constructor(numberOfBytes?: number);
+    size(): number;
+    isEmpty(): boolean;
+    heapSize(): number;
+    append(buffer: Buffer): void;
+    /**
+     * Watch out! The returned slice of Buffer points to the same Buffer in memory!
+     */
+    read(offset?: number, limit?: number): Buffer;
+    readByte(offset?: number): number;
+    /**
+     * Does hard delete
+     *
+     * Removes data from the start of the internal buffer (heap)
+     * by copying bytes to lower indices making sure the
+     * startIndex goes back to 0 afterwards
+     */
+    flushStart(numberOfBytes: number): void;
+    /**
+     * Does hard delete
+     *
+     * Removes data from the end of the internal buffer (heap)
+     * by moving the endIndex back
+     */
+    flushEnd(numberOfBytes: number): void;
+    /**
+     * Does soft delete
+     *
+     * Removes data from the start of the internal buffer (heap)
+     * by moving the startIndex forward
+     * When the heap gets empty it also resets the indices as a cleanup
+     */
+    dropStart(numberOfBytes: number): void;
+    /**
+     * Does soft delete
+     *
+     * removes data from the end of the internal buffer (heap)
+     * by moving the endIndex back
+     * When the heap gets empty it also resets the indices as a cleanup
+     */
+    dropEnd(numberOfBytes: number): void;
+    /**
+     * returns the internal buffer
+     */
+    getHeap(): Buffer;
+    clear(): void;
+    saveIndices(): void;
+    restoreIndices(): void;
+}

package/dist/ExpandingBuffer.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ExpandingBuffer = void 0;
+const node_buffer_1 = require("node:buffer");
+const functions_1 = require("./functions");
+class ExpandingBuffer {
+    #heap;
+    #startIndex = 0;
+    #endIndex = 0;
+    #backup = {
+        startIndex: 0,
+        endIndex: 0,
+    };
+    constructor(numberOfBytes = 0) {
+        this.#heap = node_buffer_1.Buffer.allocUnsafe(numberOfBytes);
+    }
+    #getActualData(offset = 0) {
+        return this.#heap.subarray(this.#startIndex + offset, this.#endIndex);
+    }
+    size() {
+        return this.#endIndex - this.#startIndex;
+    }
+    isEmpty() {
+        return this.size() === 0;
+    }
+    heapSize() {
+        return this.#heap.length;
+    }
+    append(buffer) {
+        if (this.#endIndex + buffer.length < this.heapSize()) {
+            buffer.copy(this.#heap, this.#endIndex);
+            this.#endIndex += buffer.length;
+        }
+        else {
+            this.#heap = node_buffer_1.Buffer.concat([this.#getActualData(), buffer]);
+            this.#startIndex = 0;
+            this.#endIndex = this.heapSize();
+        }
+    }
+    /**
+     * Watch out! The returned slice of Buffer points to the same Buffer in memory!
+     */
+    read(offset = 0, limit = this.size()) {
+        if (offset < 0 || limit < 1) {
+            return node_buffer_1.Buffer.from([]);
+        }
+        if (offset + limit < this.size()) {
+            return this.#heap.subarray(this.#startIndex + offset, this.#startIndex + limit + offset);
+        }
+        return this.#getActualData(offset);
+    }
+    readByte(offset = 0) {
+        return this.#heap[this.#startIndex + offset];
+    }
+    /**
+     * Does hard delete
+     *
+     * Removes data from the start of the internal buffer (heap)
+     * by copying bytes to lower indices making sure the
+     * startIndex goes back to 0 afterwards
+     */
+    flushStart(numberOfBytes) {
+        numberOfBytes = (0, functions_1.clamp)(0, this.heapSize(), numberOfBytes);
+        if (numberOfBytes > 0) {
+            if (numberOfBytes < this.heapSize()) {
+                this.#heap.copy(this.#heap, 0, this.#startIndex + numberOfBytes);
+            }
+            this.#endIndex -= this.#startIndex + numberOfBytes;
+            this.#startIndex = 0;
+        }
+    }
+    /**
+     * Does hard delete
+     *
+     * Removes data from the end of the internal buffer (heap)
+     * by moving the endIndex back
+     */
+    flushEnd(numberOfBytes) {
+        const clampedNumberOfBytes = (0, functions_1.clamp)(0, this.heapSize(), numberOfBytes);
+        if (clampedNumberOfBytes > 0) {
+            this.#endIndex -= clampedNumberOfBytes;
+        }
+    }
+    /**
+     * Does soft delete
+     *
+     * Removes data from the start of the internal buffer (heap)
+     * by moving the startIndex forward
+     * When the heap gets empty it also resets the indices as a cleanup
+     */
+    dropStart(numberOfBytes) {
+        if (numberOfBytes > 0) {
+            this.#startIndex += numberOfBytes;
+            if (this.#startIndex >= this.#endIndex) {
+                this.clear();
+            }
+        }
+    }
+    /**
+     * Does soft delete
+     *
+     * removes data from the end of the internal buffer (heap)
+     * by moving the endIndex back
+     * When the heap gets empty it also resets the indices as a cleanup
+     */
+    dropEnd(numberOfBytes) {
+        if (numberOfBytes > 0) {
+            this.#endIndex -= numberOfBytes;
+            if (this.#startIndex >= this.#endIndex) {
+                this.clear();
+            }
+        }
+    }
+    /**
+     * returns the internal buffer
+     */
+    getHeap() {
+        return this.#heap;
+    }
+    clear() {
+        this.#startIndex = 0;
+        this.#endIndex = 0;
+    }
+    saveIndices() {
+        this.#backup.startIndex = this.#startIndex;
+        this.#backup.endIndex = this.#endIndex;
+    }
+    restoreIndices() {
+        this.#startIndex = this.#backup.startIndex;
+        this.#endIndex = this.#backup.endIndex;
+    }
+}
+exports.ExpandingBuffer = ExpandingBuffer;
+//# sourceMappingURL=ExpandingBuffer.js.map

package/dist/ExpandingBuffer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ExpandingBuffer.js","sourceRoot":"","sources":["../src/ExpandingBuffer.ts"],"names":[],"mappings":";;;AAAA,6CAAoC;AACpC,2CAAmC;AAEnC,MAAa,eAAe;IAC1B,KAAK,CAAQ;IACb,WAAW,GAAW,CAAC,CAAA;IACvB,SAAS,GAAW,CAAC,CAAA;IACrB,OAAO,GAA6C;QAClD,UAAU,EAAE,CAAC;QACb,QAAQ,EAAE,CAAC;KACZ,CAAA;IAED,YAAY,gBAAwB,CAAC;QACnC,IAAI,CAAC,KAAK,GAAG,oBAAM,CAAC,WAAW,CAAC,aAAa,CAAC,CAAA;IAChD,CAAC;IAED,cAAc,CAAC,SAAiB,CAAC;QAC/B,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;IACvE,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,WAAW,CAAA;IAC1C,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;IAC1B,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;IAC1B,CAAC;IAED,MAAM,CAAC,MAAc;QACnB,IAAI,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAE;YACpD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,CAAA;YACvC,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,MAAM,CAAA;SAChC;aAAM;YACL,IAAI,CAAC,KAAK,GAAG,oBAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,cAAc,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;YAC3D,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAA;SACjC;IACH,CAAC;IAED;;OAEG;IACH,IAAI,CAAC,SAAiB,CAAC,EAAE,QAAgB,IAAI,CAAC,IAAI,EAAE;QAClD,IAAI,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE;YAC3B,OAAO,oBAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;SACvB;QAED,IAAI,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,EAAE;YAChC,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,GAAG,MAAM,EAAE,IAAI,CAAC,WAAW,GAAG,KAAK,GAAG,MAAM,CAAC,CAAA;SACzF;QAED,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC,CAAA;IACpC,CAAC;IAED,QAAQ,CAAC,SAAiB,CAAC;QACzB,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,CAAA;IAC9C,CAAC;IAED;;;;;;OAMG;IACH,UAAU,CAAC,aAAqB;QAC9B,aAAa,GAAG,IAAA,iBAAK,EAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,aAAa,CAAC,CAAA;QACxD,IAAI,aAAa,GAAG,CAAC,EAAE;YACrB,IAAI,aAAa,GAAG,IAAI,CAAC,QAAQ,EAAE,EAAE;gBACnC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,GAAG,aAAa,CAAC,CAAA;aACjE;YAED,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,WAAW,GAAG,aAAa,CAAA;YAClD,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;SACrB;IACH,CAAC;IAED;;;;;OAKG;IACH,QAAQ,CAAC,aAAqB;QAC5B,MAAM,oBAAoB,GAAG,IAAA,iBAAK,EAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,aAAa,CAAC,CAAA;QACrE,IAAI,oBAAoB,GAAG,CAAC,EAAE;YAC5B,IAAI,CAAC,SAAS,IAAI,oBAAoB,CAAA;SACvC;IACH,CAAC;IAED;;;;;;OAMG;IACH,SAAS,CAAC,aAAqB;QAC7B,IAAI,aAAa,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,WAAW,IAAI,aAAa,CAAA;YACjC,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE;gBACtC,IAAI,CAAC,KAAK,EAAE,CAAA;aACb;SACF;IACH,CAAC;IAED;;;;;;OAMG;IACH,OAAO,CAAC,aAAqB;QAC3B,IAAI,aAAa,GAAG,CAAC,EAAE;YACrB,IAAI,CAAC,SAAS,IAAI,aAAa,CAAA;YAC/B,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE;gBACtC,IAAI,CAAC,KAAK,EAAE,CAAA;aACb;SACF;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;IAED,KAAK;QACH,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;QACpB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAA;IACpB,CAAC;IAED,WAAW;QACT,IAAI,CAAC,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,WAAW,CAAA;QAC1C,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAA;IACxC,CAAC;IAED,cAAc;QACZ,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAA;QAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAA;IACxC,CAAC;CACF;AAhJD,0CAgJC"}

package/dist/Explode.d.ts

@@ -0,0 +1,8 @@
+import { Buffer } from 'node:buffer';
+import { Transform, TransformCallback } from 'node:stream';
+import { Config } from './types';
+export declare class Explode {
+    #private;
+    constructor(config?: Config);
+    getHandler(): (this: Transform, chunk: Buffer, encoding: BufferEncoding, callback: TransformCallback) => void;
+}