node-pkware 0.6.0 → 1.0.0--beta.8

package/README.md

@@ -1,61 +1,228 @@
-# node-pkware
-
-nodejs implementation of StormLib compression/decompression algorithm
-
-it was the de-facto compression for games from around Y2K, like Arx Fatalis
-
-## installation / update existing version
-
-`npm i -g node-pkware`
-
-recommended node version: 8.5+
-
-tested in node version 12.7.0
-
-## command line interface
-
-`implode <filename> --output=<filename> --ascii|--binary --level=1|2|3` - compresses file.
-if `--output` is omitted, then output will be placed next to input and names as `<filename>.compressed`.
-optionally you can specify an offset from which the compressed data starts with the `--offset=<int|hex>`,
-which is useful for mixed files, such as the fts files of Arx Fatalis
-
-`explode <filename> --output=<filename>` - decompresses file. if `--output` is omitted, then
-output will be placed next to input and names as `<filename>.decompressed`. optionally you can
-specify an offset from which the compressed data starts with the `--offset=<int|hex>`, which is useful
-for mixed files, such as the fts files of Arx Fatalis
-
-The `--drop-before-offset` flag tells node-pkware to drop the portion before `--offset`, otherwise
-it will keep it untouched and attach it to the output file.
-
-## examples
-
-`explode test/files/fast.fts --output=C:/fast.fts.decompressed --offset=1816 --keep-header`
-
-`explode test/files/fast.fts --output=C:/fast.fts.decompressed --offset=0x718 --keep-header`
-
-`implode test/files/fast.fts.unpacked --output=C:/fast.fts --binary --level=3 --offset=1816 --keep-header`
-
-### piping also works
-
-`cat c:/arx/level8.llf | explode > c:/arx/level8.llf.unpacked --keep-header`
-
-`explode c:/arx/level8.llf > c:/arx/level8.llf.unpacked --keep-header`
-
-`cat c:/arx/level8.llf | explode --output=c:/arx/level8.llf.unpacked --keep-header`
-
-
-`cat e:/piping/level8.llf.unpacked | implode --binary --level=3 > e:/piping/level8.llf.comp2 --keep-header`
-
-`implode e:/piping/level8.llf.unpacked --binary --level=3 > e:/piping/level8.llf.comp --keep-header`
-
-`cat e:/piping/level8.llf.unpacked | implode --binary --level=3 --output="e:/piping/level8.llf.comp2" --keep-header`
-
-## sources:
-
-* https://github.com/ladislav-zezula/StormLib/tree/master/src/pklib
-* https://github.com/ShieldBattery/implode-decoder
-
-### helpful links:
-
-* https://stackoverflow.com/questions/2094666/pointers-in-c-when-to-use-the-ampersand-and-the-asterisk
-* https://stackoverflow.com/a/49394095/1806628
+# node-pkware
+
+Node JS implementation of StormLib's Pkware compression/decompression algorithm
+
+It was the de-facto compression for games from around Y2K, like Arx Fatalis
+
+## installation / update existing version
+
+`npm i -g node-pkware`
+
+recommended node version: 8.5+
+
+development and testing should be done in node 12.3+ because of the tests utilizing `Readable.from()` - source: https://stackoverflow.com/a/59638132/1806628
+
+tested in node version 14.9.0
+
+## command line interface
+
+`implode <filename> --output=<filename> --ascii|-a|--binary|-b --small|-s|--medium|-m|--large|-l` - compresses file. If `--output` is omitted, then output will be placed next to input and names as `<filename>.compressed`. Optionally you can specify an offset from which the compressed data starts with the `--offset=<int|hex>`, which is useful for mixed files, such as the fts files of Arx Fatalis
+
+`explode <filename> --output=<filename>` - decompresses file. If `--output` is omitted, then output will be placed next to input and names as `<filename>.decompressed`. Optionally you can specify an offset from which the compressed data starts with the `--offset=<int|hex>`, which is useful for mixed files, such as the fts files of Arx Fatalis
+
+The `--drop-before-offset` flag tells node-pkware to drop the portion before `--offset`, otherwise it will keep it untouched and attach it to the output file.
+
+**currently disabled, WIP** There is an `--auto-detect` flag, which will search for the first pkware header starting from the beginning of the file. If `--offset` is defined, then it will start searching from that point.
+
+Calling either explode or implode with the `-v` or `--version` flag will display the package's version
+
+## examples
+
+`explode test/files/fast.fts --output=C:/fast.fts.decompressed --offset=1816`
+
+`explode test/files/fast.fts --output=C:/fast.fts.decompressed --offset=0x718`
+
+`implode test/files/fast.fts.unpacked --output=C:/fast.fts --binary --large --offset=1816`
+
+`explode test/files/fast.fts --auto-detect --debug --output=E:/fast.fts.unpacked`
+
+`explode test/files/fast.fts --auto-detect --debug --output=E:/fast.fts.unpacked --offset=2000`
+
+### piping also works
+
+**Don't use --debug when piping, because debug messages will be outputted to where the decompressed data is being outputted!**
+
+`cat c:/arx/level8.llf | explode > c:/arx/level8.llf.unpacked`
+
+`explode c:/arx/level8.llf > c:/arx/level8.llf.unpacked`
+
+`cat c:/arx/level8.llf | explode --output=c:/arx/level8.llf.unpacked`
+
+`cat e:/piping/level8.llf.unpacked | implode --binary --large > e:/piping/level8.llf`
+
+`implode e:/piping/level8.llf.unpacked --binary --large > e:/piping/level8.llf.comp`
+
+`cat e:/piping/level8.llf.unpacked | implode --binary --large --output="e:/piping/level8.llf"`
+
+## using as a library
+
+### API (named imports of node-pkware)
+
+`explode(config: object): transform._transform` - decompresses stream
+
+Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method. The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
+
+Takes an optional config object, which has the following properties:
+
+```js
+{
+  debug: boolean, // whether the code should display debug messages on the console or not (default = false)
+  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)
+  outputBufferSize: int // same as inputBufferSize, but for the outputBuffer (default 0)
+}
+```
+
+`decompress(config: object): transform._transform` - alias for explode
+
+`implode(compressionType: int, dictionarySize: int, config: object): transform._transform` - compresses stream
+
+Takes an optional config object, which has the following properties:
+
+```js
+{
+  debug: boolean, // whether the code should display debug messages on the console or not (default = false)
+  inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)
+  outputBufferSize: int // same as inputBufferSize, but for the outputBuffer (default 0)
+}
+```
+
+Returns a function, that you can use as a [transform.\_transform](https://nodejs.org/api/stream.html#stream_transform_transform_chunk_encoding_callback) method. The returned function has the `(chunk: Buffer, encoding: string, callback: function)` parameter signature.
+
+`compress(compressionType: int, dictionarySize: int, config: object): transform._transform` - aliasa for implode
+
+`stream` - an object of helper functions for channeling streams to and from explode/implode
+
+`stream.through(transformer: function): Transform` - a function, which takes a `transform._transform` type function and turns it into a Transform stream instance
+
+`stream.transformEmpty(chunk: Buffer, encoding: string, callback: function)` - a `transform._transform` type function, which for every input chunk will output an empty buffer
+
+`stream.transformIdentity(chunk: Buffer, encoding: string, callback: function)` - a `transform._transform` type function, which lets the input chunks through without any change
+
+`stream.splitAt(index: int): function` - creates a **"predicate"** function, that awaits Buffers, keeps an internal counter of the bytes from them and splits the appropriate buffer at the given index. Splitting is done by returning an array with `[left: Buffer, right: Buffer, isLeftDone: bool]`. If you want to split data at the 100th byte and you keep feeding 60 byte long buffers to the function returned by splitAt(100), then it will return arrays in the following manner:
+
+```
+1) [inputBuffer, emptyBuffer, false]
+2) [inputBuffer.slice(0, 40), inputBuffer.slice(40, 60), true]
+3) [emptyBuffer, inputBuffer, true]
+4) [emptyBuffer, inputBuffer, true]
+... and so on
+```
+
+`stream.transformSplitBy(predicate: predicate, left: transform._transform, right: transform._transform): transform._transform` - higher order function for introducing conditional logic to transform.\_transform functions. This is used internally to handle offsets for explode()
+
+`stream.streamToBuffer(callback: function): writable._write` - data can be piped to the returned function from a stream and it will concatenate all chunks into a single buffer. Takes a callback function, which will receive the concatenated buffer as a parameter
+
+`constants.COMPRESSION_BINARY` and `constants.COMPRESSION_ASCII` - compression types for implode
+
+`constants.DICTIONARY_SIZE_SMALL`, `constants.DICTIONARY_SIZE_MEDIUM` and `constants.DICTIONARY_SIZE_LARGE` - dictionary sizes for implode, determines how well the file get compressed. Small dictionary size means less memory to lookback in data for repetitions, meaning it will be less effective, the file stays larger, less compressed. On the other hand, large compression allows more lookback allowing more effective compression, thus generating smaller, more compressed files.
+
+`errors.InvalidDictionarySizeError` - thrown by implode when invalid dictionary size was specified or by explode when it encounters invalid data in the header section (the first 2 bytes of a compressed files)
+
+`errors.InvalidCompressionTypeError` - thrown by implode when invalid compression type was specified or by explode when it encounters invalid data in the header section (the first 2 bytes of a compressed files)
+
+`errors.InvalidDataError` - thrown by explode, when compressed data is less, than 5 bytes long. Pkware compressed files have 2 bytes header followed by at lest 2 bytes of data and an end literal.
+
+`errors.AbortedError` - thrown by explode when compressed data ends without reaching the end literal or in mid decompression
+
+### examples
+
+#### decompressing file with no offset into a file
+
+```javascript
+const fs = require('fs')
+const { explode, stream } = require('node-pkware')
+const { through } = stream
+
+fs.createReadStream(`path-to-compressed-file`)
+  .pipe(through(explode()))
+  .pipe(fs.createWriteStream(`path-to-write-decompressed-data`))
+```
+
+#### decompressing buffer with no offset into a buffer
+
+```javascript
+const { Readable } = require('stream')
+const { explode, stream } = require('node-pkware')
+const { through, streamToBuffer } = stream
+
+Readable.from(buffer) // buffer is of type Buffer with compressed data
+  .pipe(through(explode()))
+  .pipe(
+    streamToBuffer(decompressedData => {
+      // decompressedData holds the decompressed buffer
+    })
+  )
+```
+
+#### decompressing file with offset into a file, keeping initial part intact
+
+```javascript
+const fs = require('fs')
+const { explode, stream } = require('node-pkware')
+const { through, transformSplitBy, splitAt, transformIdentity } = stream
+
+const offset = 150 // 150 bytes of data will be skipped and explode will decompress the data that comes afterwards
+
+fs.createReadStream(`path-to-compressed-file`)
+  .pipe(through(transformSplitBy(splitAt(offset), transformIdentity(), explode())))
+  .pipe(fs.createWriteStream(`path-to-write-decompressed-data`))
+```
+
+#### decompressing file with offset into a file, discarding initial part
+
+```javascript
+const fs = require('fs')
+const { explode, stream } = require('node-pkware')
+const { through, transformSplitBy, splitAt, transformEmpty } = stream
+
+const offset = 150 // 150 bytes of data will be skipped and explode will decompress the data that comes afterwards
+
+fs.createReadStream(`path-to-compressed-file`)
+  .pipe(through(transformSplitBy(splitAt(offset), transformEmpty(), explode())))
+  .pipe(fs.createWriteStream(`path-to-write-decompressed-data`))
+```
+
+### Catching errors
+
+```javascript
+const fs = require('fs')
+const { explode, stream } = require('node-pkware')
+const { through } = stream
+
+fs.createReadStream(`path-to-compressed-file`)
+  .on('error', err => {
+    console.error('readstream error')
+  })
+  .pipe(
+    through(explode()).on('error', err => {
+      console.error('explode error')
+    })
+  )
+  .pipe(
+    fs.createWriteStream(`path-to-write-decompressed-data`).on('error', err => {
+      console.error('writestream error')
+    })
+  )
+```
+
+## Useful links
+
+### test files
+
+https://github.com/meszaros-lajos-gyorgy/pkware-test-files
+
+### sources
+
+- https://github.com/ladislav-zezula/StormLib/tree/master/src/pklib
+- https://github.com/ShieldBattery/implode-decoder
+- https://github.com/TheNitesWhoSay/lawine/blob/master/lawine/misc/implode.c - nice find, @alexpineda!
+- https://github.com/arx/ArxLibertatis/blob/master/src/io/Blast.cpp
+- https://github.com/arx/ArxLibertatis/blob/229d55b1c537c137ac50096221fa486df18ba0d2/src/io/Implode.cpp
+
+Implode was removed from Arx Libertatis at this commit: https://github.com/arx/ArxLibertatis/commit/2db9f0dd023fdd5d4da6f06c08a92d932e218187
+
+### helpful info
+
+- https://stackoverflow.com/questions/2094666/pointers-in-c-when-to-use-the-ampersand-and-the-asterisk
+- https://stackoverflow.com/a/49394095/1806628
+- https://nodejs.org/api/stream.html

package/bin/explode.js

@@ -0,0 +1,99 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const minimist = require('minimist-lite')
+const { getPackageVersion, parseNumberString, fileExists } = require('../src/helpers/functions.js')
+const { transformEmpty, transformIdentity, transformSplitBy, splitAt, through } = require('../src/helpers/stream.js')
+const { explode } = require('../src/explode.js')
+// const {
+//   COMPRESSION_BINARY,
+//   COMPRESSION_ASCII,
+//   DICTIONARY_SIZE_SMALL,
+//   DICTIONARY_SIZE_MEDIUM,
+//   DICTIONARY_SIZE_LARGE
+// } = require('../src/constants.js')
+
+const args = minimist(process.argv.slice(2), {
+  string: ['output', 'offset', 'input-buffer-size', 'output-buffer-size'],
+  boolean: ['version', 'drop-before-offset', 'debug' /*, 'auto-detect' */],
+  alias: {
+    v: 'version'
+  }
+})
+
+const decompress = (input, output, offset, /* autoDetect, */ keepHeader, config) => {
+  const leftHandler = keepHeader ? transformIdentity() : transformEmpty()
+  const rightHandler = explode(config)
+
+  let handler = rightHandler
+
+  // if (autoDetect) {
+  //   const everyPkwareHeader = [
+  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_SMALL]),
+  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_MEDIUM]),
+  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_LARGE]),
+  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_SMALL]),
+  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_MEDIUM]),
+  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_LARGE])
+  //   ]
+  //   handler = transformSplitBy(splitAtMatch(everyPkwareHeader, offset, config.debug), leftHandler, rightHandler)
+  // } else if (offset > 0) {
+  handler = transformSplitBy(splitAt(offset), leftHandler, rightHandler)
+  // }
+
+  return new Promise((resolve, reject) => {
+    input.pipe(through(handler).on('error', reject)).pipe(output).on('finish', resolve).on('error', reject)
+  })
+}
+
+;(async () => {
+  if (args.version) {
+    console.log(await getPackageVersion())
+    process.exit(0)
+  }
+
+  let input = args._[0] || args.input
+  let output = args.output
+
+  let hasErrors = false
+
+  if (input) {
+    if (await fileExists(input)) {
+      input = fs.createReadStream(input)
+    } else {
+      console.error('error: input file does not exist')
+      hasErrors = true
+    }
+  } else {
+    input = process.openStdin()
+  }
+
+  if (output) {
+    output = fs.createWriteStream(output)
+  } else {
+    output = process.stdout
+  }
+
+  if (hasErrors) {
+    process.exit(1)
+  }
+
+  const offset = parseNumberString(args.offset, 0)
+  // const autoDetect = args['auto-detect']
+
+  const keepHeader = !args['drop-before-offset']
+  const config = {
+    debug: args.debug,
+    inputBufferSize: parseNumberString(args['input-buffer-size'], 0x10000),
+    outputBufferSize: parseNumberString(args['output-buffer-size'], 0x40000)
+  }
+
+  decompress(input, output, offset, /* autoDetect, */ keepHeader, config)
+    .then(() => {
+      process.exit(0)
+    })
+    .catch(e => {
+      console.error(`error: ${e.message}`)
+      process.exit(1)
+    })
+})()

package/bin/implode.js

@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+
+const fs = require('fs')
+const minimist = require('minimist-lite')
+const {
+  COMPRESSION_BINARY,
+  COMPRESSION_ASCII,
+  DICTIONARY_SIZE_SMALL,
+  DICTIONARY_SIZE_MEDIUM,
+  DICTIONARY_SIZE_LARGE
+} = require('../src/constants.js')
+const { getPackageVersion, parseNumberString, fileExists } = require('../src/helpers/functions.js')
+const { implode } = require('../src/implode.js')
+const { transformEmpty, transformIdentity, transformSplitBy, splitAt, through } = require('../src/helpers/stream.js')
+
+const decompress = (input, output, offset, keepHeader, compressionType, dictionarySize, config) => {
+  const leftHandler = keepHeader ? transformIdentity() : transformEmpty()
+  const rightHandler = implode(compressionType, dictionarySize, config)
+
+  const handler = transformSplitBy(splitAt(offset), leftHandler, rightHandler)
+
+  return new Promise((resolve, reject) => {
+    input.pipe(through(handler).on('error', reject)).pipe(output).on('finish', resolve).on('error', reject)
+  })
+}
+
+const args = minimist(process.argv.slice(2), {
+  string: ['output', 'offset', 'input-buffer-size', 'output-buffer-size'],
+  boolean: ['version', 'binary', 'ascii', 'drop-before-offset', 'debug', 'small', 'medium', 'large'],
+  alias: {
+    a: 'ascii',
+    b: 'binary',
+    s: 'small',
+    m: 'medium',
+    l: 'large',
+    v: 'version'
+  }
+})
+
+;(async () => {
+  if (args.version) {
+    console.log(await getPackageVersion())
+    process.exit(0)
+  }
+
+  let input = args._[0] || args.input
+  let output = args.output
+
+  let hasErrors = false
+
+  if (input) {
+    if (await fileExists(input)) {
+      input = fs.createReadStream(input)
+    } else {
+      console.error('error: given file does not exist')
+      hasErrors = true
+    }
+  } else {
+    input = process.openStdin()
+  }
+
+  if (args.ascii && args.binary) {
+    console.error('error: multiple compression types specified, can only work with one of --ascii and --binary')
+    hasErrors = true
+  } else if (!args.ascii && !args.binary) {
+    console.error('error: compression type missing, expected either --ascii or --binary')
+    hasErrors = true
+  }
+
+  const sizes = [args.small, args.medium, args.large].filter(x => x === true)
+  if (sizes.length > 1) {
+    console.error('error: multiple size types specified, can only work with one of --small, --medium and --large')
+    hasErrors = true
+  } else if (sizes.length === 0) {
+    console.error('error: size type missing, expected either --small, --medium or --large')
+    hasErrors = true
+  }
+
+  if (output) {
+    output = fs.createWriteStream(output)
+  } else {
+    output = process.stdout
+  }
+
+  if (hasErrors) {
+    process.exit(1)
+  }
+
+  const compressionType = args.ascii ? COMPRESSION_ASCII : COMPRESSION_BINARY
+  const dictionarySize = args.small
+    ? DICTIONARY_SIZE_SMALL
+    : args.medium
+    ? DICTIONARY_SIZE_MEDIUM
+    : DICTIONARY_SIZE_LARGE
+
+  const offset = parseNumberString(args.offset, 0)
+
+  const keepHeader = !args['drop-before-offset']
+  const config = {
+    debug: args.debug,
+    inputBufferSize: parseNumberString(args['input-buffer-size'], 0x10000),
+    outputBufferSize: parseNumberString(args['output-buffer-size'], 0x12000)
+  }
+
+  decompress(input, output, offset, keepHeader, compressionType, dictionarySize, config)
+    .then(() => {
+      process.exit(0)
+    })
+    .catch(e => {
+      console.error(`error: ${e.message}`)
+      process.exit(1)
+    })
+})()

package/package.json

@@ -1,20 +1,21 @@
 {
   "name": "node-pkware",
-  "version": "0.6.0",
+  "version": "1.0.0--beta.8",
   "description": "The nodejs implementation of StormLib's pkware compressor/de-compressor",
-  "main": "src/index.mjs",
+  "main": "src/index.js",
+  "types": "types/index.d.ts",
   "bin": {
-    "implode": "bin/implode.mjs",
-    "explode": "bin/explode.mjs"
+    "explode": "bin/explode.js",
+    "implode": "bin/implode.js"
   },
   "scripts": {
-    "test": "node --experimental-modules test/index.mjs",
-    "test:watch": "nodemon --experimental-modules test/index.mjs --ext .mjs",
-    "lint": "eslint \"src/**/*.+(js|mjs)\"",
+    "lint": "eslint \"src/**/*.js\"",
     "lint:fix": "npm run lint -- --fix",
-    "unit": "mocha --experimental-modules test/**/*.spec.mjs --timeout 5000",
-    "unit:watch": "nodemon --exec \"npm run unit\" --watch test --watch src",
-    "lint:staged": "lint-staged"
+    "lint:staged": "lint-staged",
+    "test:unit": "set FORCE_COLOR=true && mocha test/**/*.spec.js --timeout 5000",
+    "test:unit:watch": "nodemon --exec \"npm run test:unit\" --watch test --watch src",
+    "test:files": "set FORCE_COLOR=true && mocha test/**/*.files.js --timeout 10000",
+    "test:files:watch": "nodemon --exec \"npm run test:files\" --watch test --watch src"
   },
   "repository": {
     "type": "git",
@@ -27,33 +28,36 @@
   "author": "Lajos Meszaros <m_lajos@hotmail.com>",
   "license": "GPL-3.0-or-later",
   "dependencies": {
-    "minimist": "^1.2.5",
-    "ramda": "^0.27.1"
+    "minimist-lite": "^2.0.0",
+    "ramda": "^0.27.1",
+    "ramda-adjunct": "^2.33.0"
   },
   "devDependencies": {
-    "assert": "^2.0.0",
-    "eslint": "^7.6.0",
-    "eslint-config-prettier": "^6.11.0",
-    "eslint-config-prettier-standard": "^3.0.1",
-    "eslint-config-standard": "^14.1.1",
-    "eslint-plugin-import": "^2.22.0",
+    "arx-header-size": "^0.4.4",
+    "binary-comparator": "^0.5.0",
+    "eslint": "^7.32.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-config-prettier-standard": "^4.0.1",
+    "eslint-config-standard": "^16.0.3",
+    "eslint-plugin-import": "^2.24.2",
     "eslint-plugin-node": "^11.1.0",
-    "eslint-plugin-prettier": "^3.1.4",
-    "eslint-plugin-promise": "^4.2.1",
+    "eslint-plugin-prettier": "^4.0.0",
+    "eslint-plugin-promise": "^5.1.0",
     "eslint-plugin-ramda": "^2.5.1",
-    "eslint-plugin-standard": "^4.0.1",
-    "lint-staged": "^10.2.11",
-    "mocha": "^8.1.1",
-    "nodemon": "^2.0.4",
+    "eslint-plugin-standard": "^4.1.0",
+    "lint-staged": "^11.1.2",
+    "mocha": "^9.1.1",
+    "nodemon": "^2.0.12",
     "pre-commit": "^1.2.2",
-    "prettier": "^2.0.5",
-    "prettier-config-standard": "^1.0.1"
+    "prettier": "^2.3.2",
+    "prettier-config-standard": "^4.0.0"
   },
   "pre-commit": [
     "lint:staged",
-    "unit"
+    "test:unit",
+    "test:files"
   ],
   "lint-staged": {
-    "*.{js,mjs}": "eslint --fix"
+    "*.js": "eslint --fix"
   }
 }