node-pkware 1.0.0-beta.4 → 1.0.0

package/README.md

@@ -64,7 +64,7 @@ Returns a function, that you can use as a [transform.\_transform](https://nodejs
 
 Takes an optional config object, which has the following properties:
 
-```
+```js
 {
   debug: boolean, // whether the code should display debug messages on the console or not (default = false)
   inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)
@@ -78,7 +78,7 @@ Takes an optional config object, which has the following properties:
 
 Takes an optional config object, which has the following properties:
 
-```
+```js
 {
   debug: boolean, // whether the code should display debug messages on the console or not (default = false)
   inputBufferSize: int, // the starting size of the input buffer, may expand later as needed. Not having to expand may have performance impact (default 0)

package/bin/explode.js

@@ -1,45 +1,24 @@
 #!/usr/bin/env node
 
 const fs = require('fs')
-const minimist = require('minimist')
+const minimist = require('minimist-lite')
 const { getPackageVersion, parseNumberString, fileExists } = require('../src/helpers/functions.js')
 const { transformEmpty, transformIdentity, transformSplitBy, splitAt, through } = require('../src/helpers/stream.js')
 const { explode } = require('../src/explode.js')
-// const {
-//   COMPRESSION_BINARY,
-//   COMPRESSION_ASCII,
-//   DICTIONARY_SIZE_SMALL,
-//   DICTIONARY_SIZE_MEDIUM,
-//   DICTIONARY_SIZE_LARGE
-// } = require('../src/constants.js')
 
 const args = minimist(process.argv.slice(2), {
   string: ['output', 'offset', 'input-buffer-size', 'output-buffer-size'],
-  boolean: ['version', 'drop-before-offset', 'debug' /*, 'auto-detect' */],
+  boolean: ['version', 'drop-before-offset', 'debug'],
   alias: {
     v: 'version'
   }
 })
 
-const decompress = (input, output, offset, /* autoDetect, */ keepHeader, config) => {
+const decompress = (input, output, offset, keepHeader, config) => {
   const leftHandler = keepHeader ? transformIdentity() : transformEmpty()
   const rightHandler = explode(config)
 
-  let handler = rightHandler
-
-  // if (autoDetect) {
-  //   const everyPkwareHeader = [
-  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_SMALL]),
-  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_MEDIUM]),
-  //     Buffer.from([COMPRESSION_BINARY, DICTIONARY_SIZE_LARGE]),
-  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_SMALL]),
-  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_MEDIUM]),
-  //     Buffer.from([COMPRESSION_ASCII, DICTIONARY_SIZE_LARGE])
-  //   ]
-  //   handler = transformSplitBy(splitAtMatch(everyPkwareHeader, offset, config.debug), leftHandler, rightHandler)
-  // } else if (offset > 0) {
-  handler = transformSplitBy(splitAt(offset), leftHandler, rightHandler)
-  // }
+  const handler = transformSplitBy(splitAt(offset), leftHandler, rightHandler)
 
   return new Promise((resolve, reject) => {
     input.pipe(through(handler).on('error', reject)).pipe(output).on('finish', resolve).on('error', reject)
@@ -79,7 +58,6 @@ const decompress = (input, output, offset, /* autoDetect, */ keepHeader, config)
   }
 
   const offset = parseNumberString(args.offset, 0)
-  // const autoDetect = args['auto-detect']
 
   const keepHeader = !args['drop-before-offset']
   const config = {
@@ -88,7 +66,7 @@ const decompress = (input, output, offset, /* autoDetect, */ keepHeader, config)
     outputBufferSize: parseNumberString(args['output-buffer-size'], 0x40000)
   }
 
-  decompress(input, output, offset, /* autoDetect, */ keepHeader, config)
+  decompress(input, output, offset, keepHeader, config)
     .then(() => {
       process.exit(0)
     })

package/bin/implode.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 const fs = require('fs')
-const minimist = require('minimist')
+const minimist = require('minimist-lite')
 const {
   COMPRESSION_BINARY,
   COMPRESSION_ASCII,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-pkware",
-  "version": "1.0.0-beta.4",
+  "version": "1.0.0",
   "description": "The nodejs implementation of StormLib's pkware compressor/de-compressor",
   "main": "src/index.js",
   "types": "types/index.d.ts",
@@ -28,28 +28,28 @@
   "author": "Lajos Meszaros <m_lajos@hotmail.com>",
   "license": "GPL-3.0-or-later",
   "dependencies": {
-    "minimist": "^1.2.5",
+    "minimist-lite": "^2.0.0",
     "ramda": "^0.27.1",
-    "ramda-adjunct": "^2.33.0"
+    "ramda-adjunct": "^2.35.0"
   },
   "devDependencies": {
-    "arx-header-size": "^0.4.4",
+    "arx-header-size": "^0.6.1",
     "binary-comparator": "^0.5.0",
-    "eslint": "^7.32.0",
+    "eslint": "^8.5.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-config-prettier-standard": "^4.0.1",
     "eslint-config-standard": "^16.0.3",
-    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-import": "^2.25.3",
     "eslint-plugin-node": "^11.1.0",
     "eslint-plugin-prettier": "^4.0.0",
-    "eslint-plugin-promise": "^5.1.0",
+    "eslint-plugin-promise": "^6.0.0",
     "eslint-plugin-ramda": "^2.5.1",
     "eslint-plugin-standard": "^4.1.0",
-    "lint-staged": "^11.1.2",
-    "mocha": "^9.1.1",
-    "nodemon": "^2.0.12",
+    "lint-staged": "^12.1.3",
+    "mocha": "^9.1.3",
+    "nodemon": "^2.0.15",
     "pre-commit": "^1.2.2",
-    "prettier": "^2.3.2",
+    "prettier": "^2.5.1",
     "prettier-config-standard": "^4.0.0"
   },
   "pre-commit": [

package/src/explode.js

@@ -128,9 +128,9 @@ const parseInitialData = (state, debug = false) => {
   }
 
   if (debug) {
-    console.log(`compression type: ${state.compressionType === COMPRESSION_BINARY ? 'binary' : 'ascii'}`)
+    console.log(`explode: compression type: ${state.compressionType === COMPRESSION_BINARY ? 'binary' : 'ascii'}`)
     console.log(
-      `compression level: ${
+      `explode: compression level: ${
         state.dictionarySizeBits === 4 ? 'small' : state.dictionarySizeBits === 5 ? 'medium' : 'large'
       }`
     )
@@ -331,7 +331,7 @@ const explode = (config = {}) => {
       }
 
       if (debug) {
-        console.log(`reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
+        console.log(`explode: reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
       }
 
       processChunkData(state, debug)
@@ -369,9 +369,9 @@ const explode = (config = {}) => {
 
       if (debug) {
         console.log('---------------')
-        console.log('total number of chunks read:', state.stats.chunkCounter)
-        console.log('inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
-        console.log('outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
+        console.log('explode: total number of chunks read:', state.stats.chunkCounter)
+        console.log('explode: inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
+        console.log('explode: outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
       }
 
       if (state.needMoreInput) {

package/src/helpers/ExpandingBuffer.js

@@ -44,6 +44,7 @@ class ExpandingBuffer {
     }
   }
 
+  // watch out! the buffer returned by Buffer.slice() will point to the same memory!
   read(offset, limit) {
     if (offset < 0 || limit < 1) {
       return Buffer.from([])

package/src/helpers/stream.js

@@ -3,6 +3,8 @@ const { promisify } = require('util')
 const { isFunction } = require('ramda-adjunct')
 const ExpandingBuffer = require('./ExpandingBuffer.js')
 
+const emptyBuffer = Buffer.from([])
+
 class QuasiTransform {
   constructor(handler) {
     this.handler = handler
@@ -33,12 +35,12 @@ const splitAt = index => {
 
     if (index <= cntr) {
       // index ..... cntr ..... chunk.length
-      left = Buffer.from([])
+      left = emptyBuffer
       right = chunk
     } else if (index >= cntr + chunk.length) {
       // cntr ..... chunk.length ..... index
       left = chunk
-      right = Buffer.from([])
+      right = emptyBuffer
       isLeftDone = index === cntr + chunk.length
     } else {
       // cntr ..... index ..... chunk.length
@@ -60,7 +62,7 @@ const transformIdentity = () => {
 
 const transformEmpty = () => {
   return function (chunk, encoding, callback) {
-    callback(null, Buffer.from([]))
+    callback(null, emptyBuffer)
   }
 }
 
@@ -73,6 +75,8 @@ const through = handler => {
 const transformSplitBy = (predicate, leftHandler, rightHandler) => {
   let isFirstChunk = true
   let wasLeftFlushCalled = false
+  const damChunkSize = 0x10000
+  const dam = new ExpandingBuffer()
 
   const leftTransform = new QuasiTransform(leftHandler)
   const rightTransform = new QuasiTransform(rightHandler)
@@ -86,8 +90,12 @@ const transformSplitBy = (predicate, leftHandler, rightHandler) => {
     if (isFirstChunk) {
       isFirstChunk = false
       this._flush = flushCallback => {
-        let leftFiller = Promise.resolve(Buffer.from([]))
-        let rightFiller = Promise.resolve(Buffer.from([]))
+        if (!dam.isEmpty()) {
+          this.push(dam.read())
+        }
+
+        let leftFiller = Promise.resolve(emptyBuffer)
+        let rightFiller = Promise.resolve(emptyBuffer)
 
         if (!wasLeftFlushCalled && isFunction(leftTransform._flush)) {
           leftFiller = new Promise((resolve, reject) => {
@@ -123,7 +131,7 @@ const transformSplitBy = (predicate, leftHandler, rightHandler) => {
       }
     }
 
-    let filler = Promise.resolve(Buffer.from([]))
+    let filler = Promise.resolve(emptyBuffer)
     if (isLeftDone && !wasLeftFlushCalled && isFunction(leftTransform._flush)) {
       wasLeftFlushCalled = true
       filler = new Promise((resolve, reject) => {
@@ -139,7 +147,18 @@ const transformSplitBy = (predicate, leftHandler, rightHandler) => {
 
     Promise.all([_left, filler, _right])
       .then(buffers => {
-        callback(null, Buffer.concat(buffers))
+        dam.append(Buffer.concat(buffers))
+        if (dam.size() > damChunkSize) {
+          const chunks = Math.floor(dam.size() / damChunkSize)
+          const data = Buffer.from(dam.read(0, chunks * damChunkSize))
+          dam.flushStart(chunks * damChunkSize)
+          for (let i = 0; i < chunks - 1; i++) {
+            this.push(data.slice(i * damChunkSize, i * damChunkSize + damChunkSize))
+          }
+          callback(null, data.slice((chunks - 1) * damChunkSize))
+        } else {
+          callback(null, emptyBuffer)
+        }
       })
       .catch(err => {
         callback(err)
@@ -161,56 +180,11 @@ const streamToBuffer = done => {
   })
 }
 
-/*
-export const splitAtMatch = (matches, skipBytes = 0, debug = false) => {
-  let alreadyMatched = false
-  const empty = Buffer.from([])
-
-  return (chunk, offset) => {
-    if (alreadyMatched) {
-      return {
-        left: empty,
-        right: chunk
-      }
-    }
-
-    const idxs = matches
-      .map(bytes => chunk.indexOf(bytes))
-      .filter(idx => idx > -1)
-      .sort(subtract)
-      .filter(idx => idx + offset >= skipBytes)
-
-    if (idxs.length === 0) {
-      return {
-        left: empty,
-        right: chunk
-      }
-    }
-
-    alreadyMatched = true
-    if (debug) {
-      console.log(`found pkware header ${dumpBytes(chunk.slice(idxs[0], idxs[0] + 2))} at ${toHex(idxs[0])}`)
-    }
-    return splitAtIndex(idxs[0])(chunk, offset)
-  }
-}
-*/
-
-const outputInChunks = (buffer, stream) => {
-  const chunks = Math.ceil(buffer.length / 1000)
-  for (let i = 0; i < chunks - 1; i++) {
-    stream.write(buffer.slice(i * 1000, (i + 1) * 1000))
-  }
-  stream.write(buffer.slice((chunks - 1) * 1000))
-  stream.end()
-}
-
 module.exports = {
   splitAt,
   transformIdentity,
   transformEmpty,
   through,
   transformSplitBy,
-  streamToBuffer,
-  outputInChunks
+  streamToBuffer
 }

package/src/helpers/testing.js

@@ -54,6 +54,11 @@ const buffersShouldEqual = (expected, result, offset = 0, displayAsHex = false)
   if (!Buffer.isBuffer(expected)) {
     throw new Error('expected is not a Buffer')
   }
+
+  if (!Buffer.isBuffer(result)) {
+    throw new Error('result is not a Buffer')
+  }
+
   const diff = report(expected, result, compare(expected, result, offset), displayAsHex)
   assert.ok(expected.equals(result), diff)
 }

package/src/implode.js

@@ -114,7 +114,9 @@ const getSizeOfMatching = (inputBytes, a, b) => {
 // us to store backward length in less amount of bits
 // currently the code goes from the furthest point
 const findRepetitions = (inputBytes, endOfLastMatch, cursor) => {
-  if (endOfLastMatch === cursor || cursor - endOfLastMatch < 2) {
+  const notEnoughBytes = inputBytes.length - cursor < 2
+  const tooClose = cursor === endOfLastMatch || cursor - endOfLastMatch < 2
+  if (notEnoughBytes || tooClose) {
     return { size: 0, distance: 0 }
   }
 
@@ -191,41 +193,37 @@ const processChunkData = (state, debug = false) => {
 
     /* eslint-disable prefer-const */
 
-    let endOfLastMatch = 0
+    let endOfLastMatch = 0 // used when searching for longer repetitions later
     while (state.startIndex < state.inputBuffer.size()) {
       let { size, distance } = findRepetitions(state.inputBuffer.read(endOfLastMatch), endOfLastMatch, state.startIndex)
 
-      const isFlushable = isRepetitionFlushable(size, distance, state.startIndex, state.inputBuffer.size())
+      let isFlushable = isRepetitionFlushable(size, distance, state.startIndex, state.inputBuffer.size())
 
       if (isFlushable === false) {
         const byte = state.inputBuffer.read(state.startIndex, 1)
         outputBits(state, state.nChBits[byte], state.nChCodes[byte])
         state.startIndex += 1
       } else {
-        /*
         if (isFlushable === null) {
+          /*
           // Try to find better repetition 1 byte later.
           // stormlib/implode.c L517
-          
-          // let cursor = state.startIndex
-          // let newSize = size
-          // let newDistance = distance
-          // let currentSize
-          // let currentDistance
-          // while (newSize <= currentSize && isRepetitionFlushable(newSize, newDistance, state.startIndex, state.inputBuffer.size())) {
-          //   currentSize = newSize
-          //   currentDistance = newDistance
-          //   const reps = findRepetitions(state.inputBuffer.read(endOfLastMatch), endOfLastMatch, ++cursor)
-          //   newSize = reps.size
-          //   newDistance = reps.distance
-          // }
-          // size = newSize
-          // distance = currentDistance
+          let cursor = state.startIndex
+          let newSize = size
+          let newDistance = distance
+          let currentSize
+          let currentDistance
+          while (newSize <= currentSize && isRepetitionFlushable(newSize, newDistance, state.startIndex, state.inputBuffer.size())) {
+            currentSize = newSize
+            currentDistance = newDistance
+            const reps = findRepetitions(state.inputBuffer.read(endOfLastMatch), endOfLastMatch, ++cursor)
+            newSize = reps.size
+            newDistance = reps.distance
+          }
+          size = newSize
+          distance = currentDistance
+          */
         }
-        */
-
-        /*
-        endOfLastMatch = state.startIndex + size
 
         const byte = size + 0xfe
         outputBits(state, state.nChBits[byte], state.nChCodes[byte])
@@ -240,18 +238,24 @@ const processChunkData = (state, debug = false) => {
         }
 
         state.startIndex += size
-        */
-
-        // TODO: temporarily write out data byte-by-byte here too, because above block with minimal repetition
-        // flushing breaks the compression self check tests
-        const byte = state.inputBuffer.read(state.startIndex, 1)
-        outputBits(state, state.nChBits[byte], state.nChCodes[byte])
-        state.startIndex += 1
       }
 
+      /*
       state.inputBuffer.dropStart(endOfLastMatch)
       state.startIndex -= endOfLastMatch
       endOfLastMatch = 0
+      */
+
+      if (state.dictionarySizeBits === DICTIONARY_SIZE_SMALL && state.startIndex >= 0x400) {
+        state.inputBuffer.dropStart(0x400)
+        state.startIndex -= 0x400
+      } else if (state.dictionarySizeBits === DICTIONARY_SIZE_MEDIUM && state.startIndex >= 0x800) {
+        state.inputBuffer.dropStart(0x800)
+        state.startIndex -= 0x800
+      } else if (state.dictionarySizeBits === DICTIONARY_SIZE_LARGE && state.startIndex >= 0x1000) {
+        state.inputBuffer.dropStart(0x1000)
+        state.startIndex -= 0x1000
+      }
     }
 
     /* eslint-enable prefer-const */
@@ -286,7 +290,7 @@ const implode = (compressionType, dictionarySizeBits, config = {}) => {
       }
 
       if (debug) {
-        console.log(`reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
+        console.log(`implode: reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
       }
 
       processChunkData(state, debug)
@@ -331,9 +335,9 @@ const implode = (compressionType, dictionarySizeBits, config = {}) => {
 
         if (debug) {
           console.log('---------------')
-          console.log('total number of chunks read:', state.stats.chunkCounter)
-          console.log('inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
-          console.log('outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
+          console.log('implode: total number of chunks read:', state.stats.chunkCounter)
+          console.log('implode: inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
+          console.log('implode: outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
         }
 
         callback(null, state.outputBuffer.read())

package/types/helpers/ExpandingBuffer.d.ts

@@ -13,7 +13,7 @@ declare class ExpandingBuffer {
   isEmpty(): boolean
   heapSize(): number
   append(buffer: Buffer): void
-  read(offset: number, limit: number): number | Buffer
+  read(offset?: number, limit?: number): number | Buffer
   flushStart(numberOfBytes: number): void
   flushEnd(numberOfBytes: number): void
   dropStart(numberOfBytes: number): void