node-pkware 0.7.0 → 1.0.0-alpha.1

package/src/{constants.mjs → constants.js}

@@ -1,28 +1,23 @@
-export const BINARY_COMPRESSION = 0
-export const ASCII_COMPRESSION = 1
+const COMPRESSION_BINARY = 0
+const COMPRESSION_ASCII = 1
 
-export const ERROR_INVALID_DICTIONARY_SIZE = 'Invalid dictionary size'
-export const ERROR_INVALID_COMPRESSION_TYPE = 'Invalid compression type'
-export const ERROR_INVALID_DATA = 'Invalid data'
-export const ERROR_ABORTED = 'Aborted'
+const DICTIONARY_SIZE_SMALL = 4
+const DICTIONARY_SIZE_MEDIUM = 5
+const DICTIONARY_SIZE_LARGE = 6
 
-export const DICTIONARY_SIZE1 = 0x0400
-export const DICTIONARY_SIZE2 = 0x0800
-export const DICTIONARY_SIZE3 = 0x1000
+const LONGEST_ALLOWED_REPETITION = 0x204
 
-export const LONGEST_ALLOWED_REPETITION = 0x204
+const PKDCL_OK = 'OK'
+const PKDCL_STREAM_END = 'All data from the input stream is read'
+const PKDCL_NEED_DICT = 'Need more data (dictionary)'
+const PKDCL_CONTINUE = 'Continue (internal flag)'
+const PKDCL_GET_INPUT = 'Get input (internal flag)'
 
-export const PKDCL_OK = 'OK'
-export const PKDCL_STREAM_END = 'All data from the input stream is read'
-export const PKDCL_NEED_DICT = 'Need more data (dictionary)'
-export const PKDCL_CONTINUE = 'Continue (internal flag)'
-export const PKDCL_GET_INPUT = 'Get input (internal flag)'
-
-export const LITERAL_END_STREAM = 0x305
-export const LITERAL_STREAM_ABORTED = 0x306
+const LITERAL_END_STREAM = 0x305
+const LITERAL_STREAM_ABORTED = 0x306
 
 // prettier-ignore
-export const DistCode = [
+const DistCode = [
   0x03, 0x0D, 0x05, 0x19, 0x09, 0x11, 0x01, 0x3E, 0x1E, 0x2E, 0x0E, 0x36, 0x16, 0x26, 0x06, 0x3A,
   0x1A, 0x2A, 0x0A, 0x32, 0x12, 0x22, 0x42, 0x02, 0x7C, 0x3C, 0x5C, 0x1C, 0x6C, 0x2C, 0x4C, 0x0C,
   0x74, 0x34, 0x54, 0x14, 0x64, 0x24, 0x44, 0x04, 0x78, 0x38, 0x58, 0x18, 0x68, 0x28, 0x48, 0x08,
@@ -30,7 +25,7 @@ export const DistCode = [
 ]
 
 // prettier-ignore
-export const DistBits = [
+const DistBits = [
   0x02, 0x04, 0x04, 0x05, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
   0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
   0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07,
@@ -38,28 +33,28 @@ export const DistBits = [
 ]
 
 // prettier-ignore
-export const LenBits = [
+const LenBits = [
   0x03, 0x02, 0x03, 0x03, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x05, 0x06, 0x06, 0x06, 0x07, 0x07
 ]
 
 // prettier-ignore
-export const LenCode = [
+const LenCode = [
   0x05, 0x03, 0x01, 0x06, 0x0a, 0x02, 0x0c, 0x14, 0x04, 0x18, 0x08, 0x30, 0x10, 0x20, 0x40, 0x00
 ]
 
 // prettier-ignore
-export const ExLenBits = [
+const ExLenBits = [
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08
 ]
 
 // prettier-ignore
-export const LenBase = [
+const LenBase = [
   0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007,
   0x0008, 0x000A, 0x000E, 0x0016, 0x0026, 0x0046, 0x0086, 0x0106
 ]
 
 // prettier-ignore
-export const ChBitsAsc = [
+const ChBitsAsc = [
   0x0B, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x08, 0x07, 0x0C, 0x0C, 0x07, 0x0C, 0x0C,
   0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C, 0x0D, 0x0C, 0x0C, 0x0C, 0x0C, 0x0C,
   0x04, 0x0A, 0x08, 0x0C, 0x0A, 0x0C, 0x0A, 0x08, 0x07, 0x07, 0x08, 0x09, 0x07, 0x06, 0x07, 0x08,
@@ -79,7 +74,7 @@ export const ChBitsAsc = [
 ]
 
 // prettier-ignore
-export const ChCodeAsc = [
+const ChCodeAsc = [
   0x0490, 0x0FE0, 0x07E0, 0x0BE0, 0x03E0, 0x0DE0, 0x05E0, 0x09E0,
   0x01E0, 0x00B8, 0x0062, 0x0EE0, 0x06E0, 0x0022, 0x0AE0, 0x02E0,
   0x0CE0, 0x04E0, 0x08E0, 0x00E0, 0x0F60, 0x0760, 0x0B60, 0x0360,
@@ -113,3 +108,32 @@ export const ChCodeAsc = [
   0x0600, 0x1A00, 0x0E40, 0x0640, 0x0A40, 0x0A00, 0x1200, 0x0200,
   0x1C00, 0x0C00, 0x1400, 0x0400, 0x1800, 0x0800, 0x1000, 0x0000
 ]
+
+module.exports = {
+  COMPRESSION_BINARY,
+  COMPRESSION_ASCII,
+
+  DICTIONARY_SIZE_SMALL,
+  DICTIONARY_SIZE_MEDIUM,
+  DICTIONARY_SIZE_LARGE,
+
+  LONGEST_ALLOWED_REPETITION,
+
+  PKDCL_OK,
+  PKDCL_STREAM_END,
+  PKDCL_NEED_DICT,
+  PKDCL_CONTINUE,
+  PKDCL_GET_INPUT,
+
+  LITERAL_END_STREAM,
+  LITERAL_STREAM_ABORTED,
+
+  DistCode,
+  DistBits,
+  LenBits,
+  LenCode,
+  ExLenBits,
+  LenBase,
+  ChBitsAsc,
+  ChCodeAsc
+}

package/src/errors.js

@@ -0,0 +1,50 @@
+class InvalidDictionarySizeError extends Error {
+  constructor() {
+    super('Invalid dictionary size')
+    this.name = 'InvalidDictionarySizeError'
+  }
+}
+
+class InvalidCompressionTypeError extends Error {
+  constructor() {
+    super('Invalid compression type')
+    this.name = 'InvalidCompressionTypeError'
+  }
+}
+
+class InvalidDataError extends Error {
+  constructor() {
+    super('Invalid data')
+    this.name = 'InvalidDataError'
+  }
+}
+
+class AbortedError extends Error {
+  constructor() {
+    super('Aborted')
+    this.name = 'AbortedError'
+  }
+}
+
+class ExpectedBufferError extends TypeError {
+  constructor() {
+    super('Expected variable to be of type Buffer')
+    this.name = 'ExpectedBufferError'
+  }
+}
+
+class ExpectedFunctionError extends TypeError {
+  constructor() {
+    super('Expected variable to be a Function')
+    this.name = 'ExpectedFunctionError'
+  }
+}
+
+module.exports = {
+  InvalidDictionarySizeError,
+  InvalidCompressionTypeError,
+  InvalidDataError,
+  AbortedError,
+  ExpectedBufferError,
+  ExpectedFunctionError
+}

package/src/explode.js

@@ -0,0 +1,414 @@
+const { repeat, unfold, reduce, has, includes } = require('ramda')
+const { isFunction } = require('ramda-adjunct')
+const {
+  InvalidDataError,
+  InvalidCompressionTypeError,
+  InvalidDictionarySizeError,
+  ExpectedBufferError,
+  ExpectedFunctionError,
+  AbortedError
+} = require('./errors.js')
+const { mergeSparseArrays, getLowestNBits, nBitsOfOnes, toHex } = require('./helpers/functions.js')
+const {
+  ChBitsAsc,
+  ChCodeAsc,
+  COMPRESSION_BINARY,
+  COMPRESSION_ASCII,
+  DICTIONARY_SIZE_SMALL,
+  DICTIONARY_SIZE_MEDIUM,
+  DICTIONARY_SIZE_LARGE,
+  PKDCL_OK,
+  PKDCL_STREAM_END,
+  LITERAL_STREAM_ABORTED,
+  LITERAL_END_STREAM,
+  LenBits,
+  LenBase,
+  ExLenBits,
+  DistBits,
+  LenCode,
+  DistCode
+} = require('./constants.js')
+const ExpandingBuffer = require('./helpers/ExpandingBuffer.js')
+
+const readHeader = buffer => {
+  if (!Buffer.isBuffer(buffer)) {
+    throw new ExpectedBufferError()
+  }
+  if (buffer.length < 4) {
+    throw new InvalidDataError()
+  }
+
+  const compressionType = buffer.readUInt8(0)
+  const dictionarySizeBits = buffer.readUInt8(1)
+  if (compressionType !== COMPRESSION_BINARY && compressionType !== COMPRESSION_ASCII) {
+    throw new InvalidCompressionTypeError()
+  }
+  if (!includes(dictionarySizeBits, [DICTIONARY_SIZE_SMALL, DICTIONARY_SIZE_MEDIUM, DICTIONARY_SIZE_LARGE])) {
+    throw new InvalidDictionarySizeError()
+  }
+
+  return {
+    compressionType,
+    dictionarySizeBits
+  }
+}
+
+// PAT = populate ascii table
+const createPATIterator = (limit, stepper) => n => {
+  return n >= limit ? false : [n, n + (1 << stepper)]
+}
+
+const populateAsciiTable = (value, index, bits, limit) => {
+  const iterator = createPATIterator(limit, value - bits)
+  const seed = ChCodeAsc[index] >> bits
+  const idxs = unfold(iterator, seed)
+
+  return reduce(
+    (acc, idx) => {
+      acc[idx] = index
+      return acc
+    },
+    [],
+    idxs
+  )
+}
+
+const generateAsciiTables = () => {
+  const tables = {
+    asciiTable2C34: repeat(0, 0x100),
+    asciiTable2D34: repeat(0, 0x100),
+    asciiTable2E34: repeat(0, 0x80),
+    asciiTable2EB4: repeat(0, 0x100)
+  }
+
+  tables.chBitsAsc = ChBitsAsc.map((value, index) => {
+    if (value <= 8) {
+      tables.asciiTable2C34 = mergeSparseArrays(populateAsciiTable(value, index, 0, 0x100), tables.asciiTable2C34)
+      return value - 0
+    }
+
+    const acc = getLowestNBits(8, ChCodeAsc[index])
+    if (acc === 0) {
+      tables.asciiTable2EB4 = mergeSparseArrays(populateAsciiTable(value, index, 8, 0x100), tables.asciiTable2EB4)
+      return value - 8
+    }
+
+    tables.asciiTable2C34[acc] = 0xff
+
+    if (getLowestNBits(6, ChCodeAsc[index]) === 0) {
+      tables.asciiTable2E34 = mergeSparseArrays(populateAsciiTable(value, index, 6, 0x80), tables.asciiTable2E34)
+      return value - 6
+    }
+
+    tables.asciiTable2D34 = mergeSparseArrays(populateAsciiTable(value, index, 4, 0x100), tables.asciiTable2D34)
+    return value - 4
+  })
+
+  return tables
+}
+
+const parseInitialData = (state, debug = false) => {
+  if (state.inputBuffer.size() < 4) {
+    return false
+  }
+
+  const { compressionType, dictionarySizeBits } = readHeader(state.inputBuffer.read())
+
+  state.compressionType = compressionType
+  state.dictionarySizeBits = dictionarySizeBits
+  state.bitBuffer = state.inputBuffer.read(2, 1)
+  state.inputBuffer.dropStart(3)
+  state.dictionarySizeMask = nBitsOfOnes(dictionarySizeBits)
+
+  if (compressionType === COMPRESSION_ASCII) {
+    const tables = generateAsciiTables()
+    Object.entries(tables).forEach(([key, value]) => {
+      state[key] = value
+    })
+  }
+
+  if (debug) {
+    console.log(`explode: compression type: ${state.compressionType === COMPRESSION_BINARY ? 'binary' : 'ascii'}`)
+    console.log(
+      `explode: compression level: ${
+        state.dictionarySizeBits === 4 ? 'small' : state.dictionarySizeBits === 5 ? 'medium' : 'large'
+      }`
+    )
+  }
+
+  return true
+}
+
+const wasteBits = (state, numberOfBits) => {
+  if (numberOfBits > state.extraBits && state.inputBuffer.isEmpty()) {
+    return PKDCL_STREAM_END
+  }
+
+  if (numberOfBits <= state.extraBits) {
+    state.bitBuffer = state.bitBuffer >> numberOfBits
+    state.extraBits = state.extraBits - numberOfBits
+  } else {
+    const nextByte = state.inputBuffer.read(0, 1)
+    state.inputBuffer.dropStart(1)
+
+    state.bitBuffer = ((state.bitBuffer >> state.extraBits) | (nextByte << 8)) >> (numberOfBits - state.extraBits)
+    state.extraBits = state.extraBits + 8 - numberOfBits
+  }
+
+  return PKDCL_OK
+}
+
+const decodeNextLiteral = state => {
+  const lastBit = state.bitBuffer & 1
+
+  if (wasteBits(state, 1) === PKDCL_STREAM_END) {
+    return LITERAL_STREAM_ABORTED
+  }
+
+  if (lastBit) {
+    let lengthCode = state.lengthCodes[getLowestNBits(8, state.bitBuffer)]
+
+    if (wasteBits(state, LenBits[lengthCode]) === PKDCL_STREAM_END) {
+      return LITERAL_STREAM_ABORTED
+    }
+
+    const extraLenghtBits = ExLenBits[lengthCode]
+    if (extraLenghtBits !== 0) {
+      const extraLength = getLowestNBits(extraLenghtBits, state.bitBuffer)
+
+      if (wasteBits(state, extraLenghtBits) === PKDCL_STREAM_END && lengthCode + extraLength !== 0x10e) {
+        return LITERAL_STREAM_ABORTED
+      }
+
+      lengthCode = LenBase[lengthCode] + extraLength
+    }
+
+    return lengthCode + 0x100
+  } else {
+    const lastByte = getLowestNBits(8, state.bitBuffer)
+
+    if (state.compressionType === COMPRESSION_BINARY) {
+      return wasteBits(state, 8) === PKDCL_STREAM_END ? LITERAL_STREAM_ABORTED : lastByte
+    } else {
+      let value
+      if (lastByte > 0) {
+        value = state.asciiTable2C34[lastByte]
+
+        if (value === 0xff) {
+          if (getLowestNBits(6, state.bitBuffer)) {
+            if (wasteBits(state, 4) === PKDCL_STREAM_END) {
+              return LITERAL_STREAM_ABORTED
+            }
+
+            value = state.asciiTable2D34[getLowestNBits(8, state.bitBuffer)]
+          } else {
+            if (wasteBits(state, 6) === PKDCL_STREAM_END) {
+              return LITERAL_STREAM_ABORTED
+            }
+
+            value = state.asciiTable2E34[getLowestNBits(7, state.bitBuffer)]
+          }
+        }
+      } else {
+        if (wasteBits(state, 8) === PKDCL_STREAM_END) {
+          return LITERAL_STREAM_ABORTED
+        }
+
+        value = state.asciiTable2EB4[getLowestNBits(8, state.bitBuffer)]
+      }
+
+      return wasteBits(state, state.chBitsAsc[value]) === PKDCL_STREAM_END ? LITERAL_STREAM_ABORTED : value
+    }
+  }
+}
+
+const decodeDistance = (state, repeatLength) => {
+  const distPosCode = state.distPosCodes[getLowestNBits(8, state.bitBuffer)]
+  const distPosBits = DistBits[distPosCode]
+  if (wasteBits(state, distPosBits) === PKDCL_STREAM_END) {
+    return 0
+  }
+
+  let distance
+  let bitsToWaste
+
+  if (repeatLength === 2) {
+    distance = (distPosCode << 2) | getLowestNBits(2, state.bitBuffer)
+    bitsToWaste = 2
+  } else {
+    distance = (distPosCode << state.dictionarySizeBits) | (state.bitBuffer & state.dictionarySizeMask)
+    bitsToWaste = state.dictionarySizeBits
+  }
+
+  if (wasteBits(state, bitsToWaste) === PKDCL_STREAM_END) {
+    return 0
+  }
+
+  return distance + 1
+}
+
+const processChunkData = (state, debug = false) => {
+  if (state.inputBuffer.isEmpty()) {
+    return
+  }
+
+  if (!has('compressionType', state)) {
+    const parsedHeader = parseInitialData(state, debug)
+    if (!parsedHeader || state.inputBuffer.isEmpty()) {
+      return
+    }
+  }
+
+  state.needMoreInput = false
+
+  state.backup()
+  let nextLiteral = decodeNextLiteral(state)
+
+  while (nextLiteral !== LITERAL_END_STREAM && nextLiteral !== LITERAL_STREAM_ABORTED) {
+    let addition
+    if (nextLiteral >= 0x100) {
+      const repeatLength = nextLiteral - 0xfe
+      const minusDistance = decodeDistance(state, repeatLength)
+      if (minusDistance === 0) {
+        state.needMoreInput = true
+        break
+      }
+
+      const availableData = state.outputBuffer.read(state.outputBuffer.size() - minusDistance, repeatLength)
+
+      if (repeatLength > minusDistance) {
+        const multipliedData = repeat(availableData, Math.ceil(repeatLength / availableData.length))
+        addition = Buffer.concat(multipliedData).slice(0, repeatLength)
+      } else {
+        addition = availableData
+      }
+    } else {
+      addition = Buffer.from([nextLiteral])
+    }
+
+    state.outputBuffer.append(addition)
+
+    state.backup()
+    nextLiteral = decodeNextLiteral(state)
+  }
+
+  if (nextLiteral === LITERAL_STREAM_ABORTED) {
+    state.needMoreInput = true
+  }
+
+  if (state.needMoreInput) {
+    state.restore()
+  }
+}
+
+const generateDecodeTables = (startIndexes, lengthBits) => {
+  return lengthBits.reduce((acc, lengthBit, i) => {
+    for (let index = startIndexes[i]; index < 0x100; index += 1 << lengthBit) {
+      acc[index] = i
+    }
+
+    return acc
+  }, repeat(0, 0x100))
+}
+
+const explode = (config = {}) => {
+  const { debug = false, inputBufferSize = 0x0, outputBufferSize = 0x0 } = config
+
+  const handler = function (chunk, encoding, callback) {
+    if (!isFunction(callback)) {
+      // can't call callback to pass in data or errors, so we throw up
+      throw new ExpectedFunctionError()
+    }
+
+    const state = handler._state
+    state.needMoreInput = true
+
+    try {
+      state.inputBuffer.append(chunk)
+      if (state.isFirstChunk) {
+        state.isFirstChunk = false
+        this._flush = state.onInputFinished
+      }
+
+      if (debug) {
+        console.log(`explode: reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
+      }
+
+      processChunkData(state, debug)
+
+      const blockSize = 0x1000
+      if (state.outputBuffer.size() > blockSize) {
+        const numberOfBytes = (Math.floor(state.outputBuffer.size() / blockSize) - 1) * blockSize
+        const output = Buffer.from(state.outputBuffer.read(0, numberOfBytes))
+        state.outputBuffer.flushStart(numberOfBytes)
+
+        callback(null, output)
+      } else {
+        callback(null, Buffer.from([]))
+      }
+    } catch (e) {
+      callback(e)
+    }
+  }
+
+  handler._state = {
+    _backup: {
+      extraBits: null,
+      bitBuffer: null
+    },
+    needMoreInput: true,
+    isFirstChunk: true,
+    extraBits: 0,
+    chBitsAsc: repeat(0, 0x100), // DecodeLit and GenAscTabs uses this
+    lengthCodes: generateDecodeTables(LenCode, LenBits),
+    distPosCodes: generateDecodeTables(DistCode, DistBits),
+    inputBuffer: new ExpandingBuffer(inputBufferSize),
+    outputBuffer: new ExpandingBuffer(outputBufferSize),
+    onInputFinished: callback => {
+      const state = handler._state
+
+      if (debug) {
+        console.log('---------------')
+        console.log('explode: total number of chunks read:', state.stats.chunkCounter)
+        console.log('explode: inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
+        console.log('explode: outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
+      }
+
+      if (state.needMoreInput) {
+        callback(new AbortedError())
+      } else {
+        callback(null, state.outputBuffer.read())
+      }
+    },
+    backup: () => {
+      const state = handler._state
+      state._backup.extraBits = state.extraBits
+      state._backup.bitBuffer = state.bitBuffer
+      state.inputBuffer._saveIndices()
+    },
+    restore: () => {
+      const state = handler._state
+      state.extraBits = state._backup.extraBits
+      state.bitBuffer = state._backup.bitBuffer
+      state.inputBuffer._restoreIndices()
+    },
+    stats: {
+      chunkCounter: 0
+    }
+  }
+
+  return handler
+}
+
+module.exports = {
+  readHeader,
+  explode,
+  createPATIterator,
+  populateAsciiTable,
+  generateAsciiTables,
+  processChunkData,
+  wasteBits,
+  decodeNextLiteral,
+  decodeDistance,
+  generateDecodeTables
+}

package/src/{QuasiImmutableBuffer.mjs → helpers/ExpandingBuffer.js}

@@ -1,6 +1,7 @@
-import { clamp } from '../node_modules/ramda/src/index.mjs'
+const { clamp } = require('ramda')
+const { ExpectedBufferError } = require('../errors')
 
-export default class QuasiImmutableBuffer {
+class ExpandingBuffer {
   constructor(numberOfBytes = 0) {
     this._heap = Buffer.allocUnsafe(numberOfBytes)
     this._startIndex = 0
@@ -29,6 +30,10 @@ export default class QuasiImmutableBuffer {
   }
 
   append(buffer) {
+    if (!Buffer.isBuffer(buffer)) {
+      throw new ExpectedBufferError()
+    }
+
     if (this._endIndex + buffer.length < this.heapSize()) {
       buffer.copy(this._heap, this._endIndex)
       this._endIndex += buffer.length
@@ -39,6 +44,7 @@ export default class QuasiImmutableBuffer {
     }
   }
 
+  // watch out! the buffer returned by Buffer.slice() will point to the same memory!
   read(offset, limit) {
     if (offset < 0 || limit < 1) {
       return Buffer.from([])
@@ -54,6 +60,8 @@ export default class QuasiImmutableBuffer {
     return this._getActualData(offset)
   }
 
+  // hard delete
+  // removes data from the buffer by copying bytes to lower indices
   flushStart(numberOfBytes) {
     numberOfBytes = clamp(0, this.heapSize(), numberOfBytes)
     if (numberOfBytes > 0) {
@@ -72,6 +80,8 @@ export default class QuasiImmutableBuffer {
     }
   }
 
+  // soft delete
+  // removes data from the buffer by moving the startIndex forward
   dropStart(numberOfBytes) {
     if (numberOfBytes > 0) {
       this._startIndex += numberOfBytes
@@ -109,3 +119,5 @@ export default class QuasiImmutableBuffer {
     this._endIndex = this._backup._endIndex
   }
 }
+
+module.exports = ExpandingBuffer