node-pkware 0.8.1 → 1.0.0--beta.8

package/src/implode.js

@@ -0,0 +1,361 @@
+const { has, repeat, clone, last, clamp } = require('ramda')
+const { isFunction } = require('ramda-adjunct')
+const ExpandingBuffer = require('./helpers/ExpandingBuffer.js')
+const { toHex, getLowestNBits, nBitsOfOnes } = require('./helpers/functions.js')
+const { ExpectedFunctionError, InvalidDictionarySizeError, InvalidCompressionTypeError } = require('./errors.js')
+const {
+  ChBitsAsc,
+  ChCodeAsc,
+  LONGEST_ALLOWED_REPETITION,
+  DICTIONARY_SIZE_LARGE,
+  DICTIONARY_SIZE_MEDIUM,
+  DICTIONARY_SIZE_SMALL,
+  COMPRESSION_BINARY,
+  COMPRESSION_ASCII,
+  ExLenBits,
+  LenBits,
+  LenCode,
+  DistCode,
+  DistBits
+} = require('./constants.js')
+
+const setup = state => {
+  state.nChBits = repeat(0, 0x306)
+  state.nChCodes = repeat(0, 0x306)
+
+  switch (state.dictionarySizeBits) {
+    case DICTIONARY_SIZE_LARGE:
+      state.dictionarySizeMask = nBitsOfOnes(6)
+      break
+    case DICTIONARY_SIZE_MEDIUM:
+      state.dictionarySizeMask = nBitsOfOnes(5)
+      break
+    case DICTIONARY_SIZE_SMALL:
+      state.dictionarySizeMask = nBitsOfOnes(4)
+      break
+    default:
+      throw new InvalidDictionarySizeError()
+  }
+
+  switch (state.compressionType) {
+    case COMPRESSION_BINARY:
+      for (let nChCode = 0, nCount = 0; nCount < 0x100; nCount++) {
+        state.nChBits[nCount] = 9
+        state.nChCodes[nCount] = nChCode
+        nChCode = getLowestNBits(16, nChCode) + 2
+      }
+      break
+    case COMPRESSION_ASCII:
+      for (let nCount = 0; nCount < 0x100; nCount++) {
+        state.nChBits[nCount] = ChBitsAsc[nCount] + 1
+        state.nChCodes[nCount] = ChCodeAsc[nCount] * 2
+      }
+      break
+    default:
+      throw new InvalidCompressionTypeError()
+  }
+
+  let nCount = 0x100
+  for (let i = 0; i < 0x10; i++) {
+    for (let nCount2 = 0; nCount2 < 1 << ExLenBits[i]; nCount2++) {
+      state.nChBits[nCount] = ExLenBits[i] + LenBits[i] + 1
+      state.nChCodes[nCount] = (nCount2 << (LenBits[i] + 1)) | (LenCode[i] * 2) | 1
+      nCount++
+    }
+  }
+
+  state.outputBuffer.append(Buffer.from([state.compressionType, state.dictionarySizeBits, 0]))
+  state.outBits = 0
+}
+
+const outputBits = (state, nBits, bitBuffer) => {
+  if (nBits > 8) {
+    outputBits(state, 8, bitBuffer)
+    bitBuffer = bitBuffer >> 8
+    nBits = nBits - 8
+  }
+
+  const outBits = state.outBits
+
+  // in the original code bitBuffer is long, but is cast to char
+  const lastBytes = state.outputBuffer.read(state.outputBuffer.size() - 1, 1)
+  state.outputBuffer.dropEnd(1)
+  state.outputBuffer.append(Buffer.from([lastBytes | getLowestNBits(8, bitBuffer << outBits)]))
+
+  state.outBits = state.outBits + nBits
+
+  if (state.outBits > 8) {
+    bitBuffer = bitBuffer >> (8 - outBits)
+    state.outputBuffer.append(Buffer.from([getLowestNBits(8, bitBuffer)]))
+    state.outBits = getLowestNBits(3, state.outBits)
+  } else {
+    state.outBits = getLowestNBits(3, state.outBits)
+    if (state.outBits === 0) {
+      state.outputBuffer.append(Buffer.from([0]))
+    }
+  }
+}
+
+// ---------------------------------
+
+const getSizeOfMatching = (inputBytes, a, b) => {
+  const limit = clamp(2, LONGEST_ALLOWED_REPETITION, b - a)
+
+  for (let i = 2; i <= limit; i++) {
+    if (inputBytes[a + i] !== inputBytes[b + i]) {
+      return i
+    }
+  }
+
+  return limit
+}
+
+// TODO: make sure that we find the most recent one, which in turn allows
+// us to store backward length in less amount of bits
+// currently the code goes from the furthest point
+const findRepetitions = (inputBytes, endOfLastMatch, cursor) => {
+  if (endOfLastMatch === cursor || cursor - endOfLastMatch < 2) {
+    return { size: 0, distance: 0 }
+  }
+
+  const haystack = inputBytes.slice(endOfLastMatch, cursor)
+  const needle = inputBytes.slice(cursor, cursor + 2)
+
+  const matchIndex = haystack.indexOf(needle)
+  if (matchIndex !== -1) {
+    const distance = cursor - endOfLastMatch - matchIndex
+    return {
+      distance: distance - 1,
+      size: distance > 2 ? getSizeOfMatching(inputBytes, endOfLastMatch + matchIndex, cursor) : 2
+    }
+  }
+
+  return { size: 0, distance: 0 }
+}
+
+// this function can return:
+//   false - not flushable
+//   true - flushable
+//   null - flushable, but there might be a better repetition
+const isRepetitionFlushable = (size, distance, startIndex, inputBufferSize) => {
+  if (size === 0) {
+    return false
+  }
+
+  // If we found repetition of 2 bytes, that is 0x100 or fuhrter back,
+  // don't bother. Storing the distance of 0x100 bytes would actually
+  // take more space than storing the 2 bytes as-is.
+  if (size === 2 && distance >= 0x100) {
+    return false
+  }
+
+  if (size >= 8 || startIndex + 1 >= inputBufferSize) {
+    return true
+  }
+
+  return null
+}
+
+// ---------------------------------
+
+// repetitions are at least 2 bytes long,
+// so the initial 2 bytes can be moved to the output as is
+const handleFirstTwoBytes = state => {
+  if (state.handledFirstTwoBytes) {
+    return
+  }
+
+  if (state.inputBuffer.size() < 3) {
+    return
+  }
+
+  const [byte1, byte2] = state.inputBuffer.read(0, 2)
+  outputBits(state, state.nChBits[byte1], state.nChCodes[byte1])
+  outputBits(state, state.nChBits[byte2], state.nChCodes[byte2])
+
+  state.handledFirstTwoBytes = true
+  state.startIndex += 2
+}
+
+const processChunkData = (state, debug = false) => {
+  if (!has('dictionarySizeMask', state)) {
+    setup(state)
+  }
+
+  if (!state.inputBuffer.isEmpty()) {
+    state.startIndex = 0
+
+    handleFirstTwoBytes(state)
+
+    // -------------------------------
+
+    /* eslint-disable prefer-const */
+
+    let endOfLastMatch = 0
+    while (state.startIndex < state.inputBuffer.size()) {
+      let { size, distance } = findRepetitions(state.inputBuffer.read(endOfLastMatch), endOfLastMatch, state.startIndex)
+
+      const isFlushable = isRepetitionFlushable(size, distance, state.startIndex, state.inputBuffer.size())
+
+      if (isFlushable === false) {
+        const byte = state.inputBuffer.read(state.startIndex, 1)
+        outputBits(state, state.nChBits[byte], state.nChCodes[byte])
+        state.startIndex += 1
+      } else {
+        /*
+        if (isFlushable === null) {
+          // Try to find better repetition 1 byte later.
+          // stormlib/implode.c L517
+          
+          // let cursor = state.startIndex
+          // let newSize = size
+          // let newDistance = distance
+          // let currentSize
+          // let currentDistance
+          // while (newSize <= currentSize && isRepetitionFlushable(newSize, newDistance, state.startIndex, state.inputBuffer.size())) {
+          //   currentSize = newSize
+          //   currentDistance = newDistance
+          //   const reps = findRepetitions(state.inputBuffer.read(endOfLastMatch), endOfLastMatch, ++cursor)
+          //   newSize = reps.size
+          //   newDistance = reps.distance
+          // }
+          // size = newSize
+          // distance = currentDistance
+        }
+        */
+
+        /*
+        endOfLastMatch = state.startIndex + size
+
+        const byte = size + 0xfe
+        outputBits(state, state.nChBits[byte], state.nChCodes[byte])
+        if (size === 2) {
+          const byte = distance >> 2
+          outputBits(state, state.distBits[byte], state.distCodes[byte])
+          outputBits(state, 2, distance & 3)
+        } else {
+          const byte = distance >> state.dictionarySizeBits
+          outputBits(state, state.distBits[byte], state.distCodes[byte])
+          outputBits(state, state.dictionarySizeBits, state.dictionarySizeMask & distance)
+        }
+
+        state.startIndex += size
+        */
+
+        // TODO: temporarily write out data byte-by-byte here too, because above block with minimal repetition
+        // flushing breaks the compression self check tests
+        const byte = state.inputBuffer.read(state.startIndex, 1)
+        outputBits(state, state.nChBits[byte], state.nChCodes[byte])
+        state.startIndex += 1
+      }
+
+      state.inputBuffer.dropStart(endOfLastMatch)
+      state.startIndex -= endOfLastMatch
+      endOfLastMatch = 0
+    }
+
+    /* eslint-enable prefer-const */
+
+    // -------------------------------
+
+    state.inputBuffer.dropStart(state.inputBuffer.size())
+  }
+
+  if (state.streamEnded) {
+    // Write the termination literal
+    outputBits(state, last(state.nChBits), last(state.nChCodes))
+  }
+}
+
+const implode = (compressionType, dictionarySizeBits, config = {}) => {
+  const { debug = false, inputBufferSize = 0x0, outputBufferSize = 0x0 } = config
+
+  const handler = function (chunk, encoding, callback) {
+    if (!isFunction(callback)) {
+      // can't call callback to pass in data or errors, so we throw up
+      throw new ExpectedFunctionError()
+    }
+
+    const state = handler._state
+
+    try {
+      state.inputBuffer.append(chunk)
+      if (state.isFirstChunk) {
+        state.isFirstChunk = false
+        this._flush = state.onInputFinished
+      }
+
+      if (debug) {
+        console.log(`reading ${toHex(chunk.length)} bytes from chunk #${state.stats.chunkCounter++}`)
+      }
+
+      processChunkData(state, debug)
+
+      const blockSize = 0x800
+      if (state.outputBuffer.size() > blockSize) {
+        const numberOfBytes = (Math.floor(state.outputBuffer.size() / blockSize) - 1) * blockSize
+        const output = Buffer.from(state.outputBuffer.read(0, numberOfBytes))
+        state.outputBuffer.flushStart(numberOfBytes)
+
+        if (state.outBits === 0) {
+          // set last byte to 0
+          state.outputBuffer.dropEnd(1)
+          state.outputBuffer.append(Buffer.from([0]))
+        }
+
+        callback(null, output)
+      } else {
+        callback(null, Buffer.from([]))
+      }
+    } catch (e) {
+      callback(e)
+    }
+  }
+
+  handler._state = {
+    isFirstChunk: true,
+    streamEnded: false,
+    compressionType: compressionType,
+    dictionarySizeBits: dictionarySizeBits,
+    distCodes: clone(DistCode),
+    distBits: clone(DistBits),
+    startIndex: 0,
+    inputBuffer: new ExpandingBuffer(inputBufferSize),
+    outputBuffer: new ExpandingBuffer(outputBufferSize),
+    handledFirstTwoBytes: false,
+    onInputFinished: callback => {
+      const state = handler._state
+      state.streamEnded = true
+      try {
+        processChunkData(state, debug)
+
+        if (debug) {
+          console.log('---------------')
+          console.log('total number of chunks read:', state.stats.chunkCounter)
+          console.log('inputBuffer heap size', toHex(state.inputBuffer.heapSize()))
+          console.log('outputBuffer heap size', toHex(state.outputBuffer.heapSize()))
+        }
+
+        callback(null, state.outputBuffer.read())
+      } catch (e) {
+        callback(e)
+      }
+    },
+    stats: {
+      chunkCounter: 0
+    }
+  }
+
+  return handler
+}
+
+module.exports = {
+  setup,
+  outputBits,
+  getSizeOfMatching,
+  findRepetitions,
+  isRepetitionFlushable,
+  handleFirstTwoBytes,
+  processChunkData,
+  implode
+}

package/src/index.js

@@ -0,0 +1,18 @@
+const { implode } = require('./implode.js')
+const { explode } = require('./explode.js')
+const constants = require('./constants.js')
+const errors = require('./errors.js')
+const stream = require('./helpers/stream.js')
+
+const compress = implode
+const decompress = explode
+
+module.exports = {
+  implode,
+  compress,
+  explode,
+  decompress,
+  constants,
+  errors,
+  stream
+}

package/types/constants.d.ts

@@ -0,0 +1,41 @@
+export const COMPRESSION_BINARY: 0
+export const COMPRESSION_ASCII: 1
+export const DICTIONARY_SIZE_SMALL: 4
+export const DICTIONARY_SIZE_MEDIUM: 5
+export const DICTIONARY_SIZE_LARGE: 6
+export const LONGEST_ALLOWED_REPETITION: 0x204
+
+export const PKDCL_OK: 'OK'
+export const PKDCL_STREAM_END: 'All data from the input stream is read'
+export const PKDCL_NEED_DICT: 'Need more data (dictionary)'
+export const PKDCL_CONTINUE: 'Continue (internal flag)'
+export const PKDCL_GET_INPUT: 'Get input (internal flag)'
+
+export const LITERAL_END_STREAM: 0x305
+export const LITERAL_STREAM_ABORTED: 0x306
+
+export const DistCode: number[]
+export const DistBits: number[]
+export const LenBits: number[]
+export const LenCode: number[]
+export const ExLenBits: number[]
+export const LenBase: number[]
+export const ChBitsAsc: number[]
+export const ChCodeAsc: number[]
+
+// Additional types
+
+/**
+ * Compression types for implode
+ */
+export type CompressionType = typeof COMPRESSION_BINARY | typeof COMPRESSION_ASCII
+/**
+ * Dictionary sizes for implode, determines how well the file get compressed.
+ *
+ * Small dictionary size means less memory to lookback in data for repetitions, meaning it will be less effective, the file stays larger, less compressed.
+ * On the other hand, large compression allows more lookback allowing more effective compression, thus generating smaller, more compressed files.
+ */
+export type DictionarySizeBits =
+  | typeof DICTIONARY_SIZE_SMALL
+  | typeof DICTIONARY_SIZE_MEDIUM
+  | typeof DICTIONARY_SIZE_LARGE

package/types/errors.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Thrown by
+ * - `implode` when invalid dictionary size was specified
+ * - `explode` when it encounters invalid data in the header section (the first 2 bytes of a compressed files)
+ */
+export class InvalidDictionarySizeError extends Error {}
+
+/**
+ * Thrown by
+ * - `implode` when invalid compression type was specified
+ * - `explode` when it encounters invalid data in the header section (the first 2 bytes of a compressed files)
+ */
+export class InvalidCompressionTypeError extends Error {}
+
+/**
+ * Thrown by
+ * - `explode`, when compressed data is less, than `5` bytes long
+ * 
+ * Pkware compressed files have 2 bytes header followed by at lest 2 bytes of data and an end literal.
+ */
+export class InvalidDataError extends Error {}
+
+/**
+ * Thrown by
+ * - `explode` when compressed data ends without reaching the end literal or in mid decompression
+ */
+export class AbortedError extends Error {}
+
+export class ExpectedBufferError extends TypeError {}
+export class ExpectedFunctionError extends TypeError {}

package/types/explode.d.ts

@@ -0,0 +1,56 @@
+import ExpandingBuffer from './helpers/ExpandingBuffer'
+import { Callback, Config, Handler, PrivateState } from './helpers/Shared'
+import { CompressionType, DictionarySizeBits, PKDCL_STREAM_END, PKDCL_OK, LITERAL_STREAM_ABORTED } from './constants'
+
+export function readHeader(buffer: Buffer): {
+  compressionType: CompressionType
+  dictionarySizeBits: DictionarySizeBits
+}
+
+type PrivateExplodeState = {
+  _backup: {
+    extraBits: number
+    bitBuffer: Buffer
+  }
+  needMoreInput: boolean
+  isFirstChunk: boolean
+  extraBits: number
+  bitBuffer: Buffer
+  chBitsAsc: number[] // DecodeLit and GenAscTabs uses this
+  lengthCodes: number[]
+  distPosCodes: number[]
+  inputBuffer: ExpandingBuffer
+  outputBuffer: ExpandingBuffer
+  onInputFinished(callback: Callback): void
+  backup(): void
+  restore(): void
+  stats: {
+    chunkCounter: number
+  }
+  compressionType: CompressionType
+  dictionarySizeBits: DictionarySizeBits
+  dictionarySizeMask: number
+  asciiTable2C34: number[]
+  asciiTable2D34: number[]
+  asciiTable2E34: number[]
+  asciiTable2EB4: number[]
+}
+
+/**
+ * Decompresses stream
+ * @returns a function, that you can use as a `transform._transform` method.
+ */
+export function explode(config?: Config): PrivateState<PrivateExplodeState> & Handler
+export function createPATIterator(limit: number, stepper: number): (n: number) => false | [number, number]
+export function populateAsciiTable(value: number, index: number, bits: number, limit: number): number[]
+export function generateAsciiTables(): {
+  asciiTable2C34: number[]
+  asciiTable2D34: number[]
+  asciiTable2E34: number[]
+  asciiTable2EB4: number[]
+}
+export function processChunkData(state: PrivateExplodeState, debug?: boolean): void
+export function wasteBits(state: PrivateExplodeState, numberOfBits: number): typeof PKDCL_STREAM_END | typeof PKDCL_OK
+export function decodeNextLiteral(state: PrivateExplodeState): typeof LITERAL_STREAM_ABORTED | number
+export function decodeDistance(state: PrivateExplodeState, repeatLength: number): number
+export function generateDecodeTables(startIndexes: number[], lengthBits: number[]): number[]

package/types/helpers/ExpandingBuffer.d.ts

@@ -0,0 +1,25 @@
+export = ExpandingBuffer
+declare class ExpandingBuffer {
+  constructor(numberOfBytes?: number)
+  _heap: Buffer
+  _startIndex: number
+  _endIndex: number
+  _backup: {
+    _startIndex: number
+    _endIndex: number
+  }
+  _getActualData(offset?: number): Buffer
+  size(): number
+  isEmpty(): boolean
+  heapSize(): number
+  append(buffer: Buffer): void
+  read(offset: number, limit: number): number | Buffer
+  flushStart(numberOfBytes: number): void
+  flushEnd(numberOfBytes: number): void
+  dropStart(numberOfBytes: number): void
+  dropEnd(numberOfBytes: number): void
+  getHeap(): Buffer
+  clear(): void
+  _saveIndices(): void
+  _restoreIndices(): void
+}

package/types/helpers/Shared.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Shared type signatures used throughout the library.
+ */
+
+/**
+ * A traditional "nodeback<Buffer>" callback type.
+ * @param error The error, if any.
+ * @param data The data, if any.
+ */
+export type Callback = (error: Error | null, chunk: Buffer) => void
+
+/**
+ * Handler for one chunk of bytes
+ * @param chunk The chunk of bytes
+ * @param encoding The encoding of the chunk (not used)
+ * @param callback The callback to call when done
+ */
+export type Handler = (chunk: Buffer, encoding: unknown, callback: Callback) => void
+
+/**
+ * How the implode & explode functions store their internal state.
+ */
+export type PrivateState<T> = { _state: T }
+
+/**
+ * Configuration options for the implode & explode functions.
+ */
+export type Config = {
+  /**
+   * Whether the code should display debug messages on the console or not
+   * @default false
+   */
+  debug?: boolean
+  /**
+   * The starting size of the input buffer, may expand later as needed.
+   * Not having to expand may have performance impact.
+   * @default 0
+   */
+  inputBufferSize?: number
+  /**
+   * The starting size of the output buffer, may expand later as needed.
+   * Not having to expand may have performance impact.
+   * @default 0
+   */
+  outputBufferSize?: number
+}

package/types/helpers/functions.d.ts

@@ -0,0 +1,10 @@
+export function isBetween(min: number, max: number, num: number): boolean
+export function nBitsOfOnes(numberOfBits: number): number
+export function maskBits(numberOfBits: number, number: number): number
+export function getLowestNBits(numberOfBits: number, number: number): number
+export function isFullHexString(str: string): boolean
+export function toHex(num: number, digits?: number, withoutPrefix?: boolean): string
+export function mergeSparseArrays<T>(a: T[], b: T[]): T[]
+export function parseNumberString(n: string, defaultValue?: number): number
+export function getPackageVersion(): Promise<string>
+export function fileExists(filename: string): Promise<boolean>

package/types/helpers/stream.d.ts

@@ -0,0 +1,61 @@
+import { Transform, Writable } from 'stream'
+import { Callback } from './Shared'
+
+type QuasiTransformConstructorParameter = (
+  obj: {
+    handler: QuasiTransformConstructorParameter
+    handle(chunk: Buffer, encoding: string): Promise<void>
+  },
+  chunk: Buffer,
+  encoding: string
+) => void
+
+/**
+ * Creates a "**predicate**" function, that awaits Buffers, keeps an internal counter of the bytes from them and splits the appropriate buffer at the given index.
+ * Splitting is done by returning an array with `[left: Buffer, right: Buffer, isLeftDone: bool]`.
+ * If you want to split data at the 100th byte and you keep feeding 60 byte long buffers to the function returned by `splitAt(100)`, then it will return arrays in the following manner:
+ * 1. `[inputBuffer, emptyBuffer, false]`
+ * 2. `[inputBuffer.slice(0, 40), inputBuffer.slice(40, 60), true]`
+ * 3. `[emptyBuffer, inputBuffer, true]`
+ * 4. `[emptyBuffer, inputBuffer, true]`
+ * 5. ... and so on
+ * @param index at which to split the buffer
+ */
+export function splitAt(index: number): (chunk: Buffer) => null | [Buffer, Buffer, boolean]
+
+/**
+ * A `transform._transform` type function, which lets the input chunks through without any change
+ */
+export function transformIdentity(): (chunk: Buffer, encoding: unknown, callback: Callback) => void
+
+/**
+ * A `transform._transform` type function, which for every input chunk will output an empty buffer
+ */
+export function transformEmpty(): (chunk: unknown, encoding: unknown, callback: Callback) => void
+
+/**
+ * Takes a `transform._transform` type function and turns it into a Transform stream instance
+ * @param handler a transform._transform type function
+ * @returns a Transform stream instance
+ */
+export function through(handler: Exclude<ConstructorParameters<typeof Transform>[0], undefined>['transform']): Transform
+
+/**
+ * Higher order function for introducing conditional logic to `transform._transform` functions.
+ * This is used internally to handle offsets for `explode()`.
+ * @param predicate
+ * @param leftHandler
+ * @param rightHandler
+ * @returns `transform._transform`
+ */
+export function transformSplitBy(
+  predicate: (chunk: Buffer) => [Buffer, Buffer, boolean],
+  leftHandler: QuasiTransformConstructorParameter,
+  rightHandler: QuasiTransformConstructorParameter
+): (chunk: Buffer, encoding: string, callback: Callback) => void
+
+/**
+ * Data can be piped to the returned function from a stream and it will concatenate all chunks into a single buffer.
+ * @param done a callback function, which will receive the concatenated buffer as a parameter
+ */
+export function streamToBuffer(done: (heap: Buffer) => void): Writable

package/types/helpers/testing.d.ts

@@ -0,0 +1,6 @@
+import { Handler } from './Shared'
+
+export function isClass(obj: any): obj is Object
+export function buffersShouldEqual(expected: Buffer, result: Buffer, offset?: number, displayAsHex?: boolean): void
+export function bufferToString(buffer: Buffer, limit?: number): string
+export function transformToABC(): Handler