@dragon708/docmind-markdown 1.2.2 → 1.2.3

package/node_modules/@tokenizer/token/README.md

@@ -0,0 +1,19 @@
+[![npm version](https://badge.fury.io/js/%40tokenizer%2Ftoken.svg)](https://www.npmjs.com/package/@tokenizer/token)
+[![npm downloads](http://img.shields.io/npm/dm/@tokenizer/token.svg)](https://npmcharts.com/compare/@tokenizer/token?interval=30)
+
+# @tokenizer/token
+
+TypeScript definition of an [strtok3](https://github.com/Borewit/strtok3) token.
+
+## Licence
+
+(The MIT License)
+
+Copyright (c) 2020 Borewit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

package/node_modules/@tokenizer/token/index.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Read-only token
+ * See https://github.com/Borewit/strtok3 for more information
+ */
+export interface IGetToken<Value, Array extends Uint8Array = Uint8Array> {
+
+  /**
+   * Length of encoded token in bytes
+   */
+  len: number;
+
+  /**
+   * Decode value from buffer at offset
+   * @param array - Uint8Array to read the decoded value from
+   * @param offset - Decode offset
+   * @return decoded value
+   */
+  get(array: Array, offset: number): Value;
+}
+
+export interface IToken<Value, Array extends Uint8Array = Uint8Array> extends IGetToken<Value, Array> {
+  /**
+   * Encode value to buffer
+   * @param array - Uint8Array to write the encoded value to
+   * @param offset - Buffer write offset
+   * @param value - Value to decode of type T
+   * @return offset plus number of bytes written
+   */
+  put(array: Array, offset: number, value: Value): number
+}

package/node_modules/@tokenizer/token/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@tokenizer/token",
+  "version": "0.3.0",
+  "description": "TypeScript definition for strtok3 token",
+  "main": "",
+  "types": "index.d.ts",
+  "files": [
+    "index.d.ts"
+  ],
+  "keywords": [
+    "token",
+    "interface",
+    "tokenizer",
+    "TypeScript"
+  ],
+  "author": {
+    "name": "Borewit",
+    "url": "https://github.com/Borewit"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/Borewit/tokenizer-token.git"
+  },
+  "bugs": {
+    "url": "https://github.com/Borewit/tokenizer-token/issues"
+  },
+  "typeScriptVersion": "3.0",
+  "dependencies": {},
+  "devDependencies": {
+    "@types/node": "^13.1.0"
+  }
+}

package/node_modules/ieee754/LICENSE

@@ -0,0 +1,11 @@
+Copyright 2008 Fair Oaks Labs, Inc.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/node_modules/ieee754/README.md

@@ -0,0 +1,51 @@
+# ieee754 [![travis][travis-image]][travis-url] [![npm][npm-image]][npm-url] [![downloads][downloads-image]][downloads-url] [![javascript style guide][standard-image]][standard-url]
+
+[travis-image]: https://img.shields.io/travis/feross/ieee754/master.svg
+[travis-url]: https://travis-ci.org/feross/ieee754
+[npm-image]: https://img.shields.io/npm/v/ieee754.svg
+[npm-url]: https://npmjs.org/package/ieee754
+[downloads-image]: https://img.shields.io/npm/dm/ieee754.svg
+[downloads-url]: https://npmjs.org/package/ieee754
+[standard-image]: https://img.shields.io/badge/code_style-standard-brightgreen.svg
+[standard-url]: https://standardjs.com
+
+[![saucelabs][saucelabs-image]][saucelabs-url]
+
+[saucelabs-image]: https://saucelabs.com/browser-matrix/ieee754.svg
+[saucelabs-url]: https://saucelabs.com/u/ieee754
+
+### Read/write IEEE754 floating point numbers from/to a Buffer or array-like object.
+
+## install
+
+```
+npm install ieee754
+```
+
+## methods
+
+`var ieee754 = require('ieee754')`
+
+The `ieee754` object has the following functions:
+
+```
+ieee754.read = function (buffer, offset, isLE, mLen, nBytes)
+ieee754.write = function (buffer, value, offset, isLE, mLen, nBytes)
+```
+
+The arguments mean the following:
+
+- buffer = the buffer
+- offset = offset into the buffer
+- value = value to set (only for `write`)
+- isLe = is little endian?
+- mLen = mantissa length
+- nBytes = number of bytes
+
+## what is ieee754?
+
+The IEEE Standard for Floating-Point Arithmetic (IEEE 754) is a technical standard for floating-point computation. [Read more](http://en.wikipedia.org/wiki/IEEE_floating_point).
+
+## license
+
+BSD 3 Clause. Copyright (c) 2008, Fair Oaks Labs, Inc.

package/node_modules/ieee754/index.d.ts

@@ -0,0 +1,10 @@
+declare namespace ieee754 {
+    export function read(
+        buffer: Uint8Array, offset: number, isLE: boolean, mLen: number,
+        nBytes: number): number;
+    export function write(
+        buffer: Uint8Array, value: number, offset: number, isLE: boolean,
+        mLen: number, nBytes: number): void;
+  }
+  
+  export = ieee754;

package/node_modules/ieee754/index.js

@@ -0,0 +1,85 @@
+/*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh <https://feross.org/opensource> */
+exports.read = function (buffer, offset, isLE, mLen, nBytes) {
+  var e, m
+  var eLen = (nBytes * 8) - mLen - 1
+  var eMax = (1 << eLen) - 1
+  var eBias = eMax >> 1
+  var nBits = -7
+  var i = isLE ? (nBytes - 1) : 0
+  var d = isLE ? -1 : 1
+  var s = buffer[offset + i]
+
+  i += d
+
+  e = s & ((1 << (-nBits)) - 1)
+  s >>= (-nBits)
+  nBits += eLen
+  for (; nBits > 0; e = (e * 256) + buffer[offset + i], i += d, nBits -= 8) {}
+
+  m = e & ((1 << (-nBits)) - 1)
+  e >>= (-nBits)
+  nBits += mLen
+  for (; nBits > 0; m = (m * 256) + buffer[offset + i], i += d, nBits -= 8) {}
+
+  if (e === 0) {
+    e = 1 - eBias
+  } else if (e === eMax) {
+    return m ? NaN : ((s ? -1 : 1) * Infinity)
+  } else {
+    m = m + Math.pow(2, mLen)
+    e = e - eBias
+  }
+  return (s ? -1 : 1) * m * Math.pow(2, e - mLen)
+}
+
+exports.write = function (buffer, value, offset, isLE, mLen, nBytes) {
+  var e, m, c
+  var eLen = (nBytes * 8) - mLen - 1
+  var eMax = (1 << eLen) - 1
+  var eBias = eMax >> 1
+  var rt = (mLen === 23 ? Math.pow(2, -24) - Math.pow(2, -77) : 0)
+  var i = isLE ? 0 : (nBytes - 1)
+  var d = isLE ? 1 : -1
+  var s = value < 0 || (value === 0 && 1 / value < 0) ? 1 : 0
+
+  value = Math.abs(value)
+
+  if (isNaN(value) || value === Infinity) {
+    m = isNaN(value) ? 1 : 0
+    e = eMax
+  } else {
+    e = Math.floor(Math.log(value) / Math.LN2)
+    if (value * (c = Math.pow(2, -e)) < 1) {
+      e--
+      c *= 2
+    }
+    if (e + eBias >= 1) {
+      value += rt / c
+    } else {
+      value += rt * Math.pow(2, 1 - eBias)
+    }
+    if (value * c >= 2) {
+      e++
+      c /= 2
+    }
+
+    if (e + eBias >= eMax) {
+      m = 0
+      e = eMax
+    } else if (e + eBias >= 1) {
+      m = ((value * c) - 1) * Math.pow(2, mLen)
+      e = e + eBias
+    } else {
+      m = value * Math.pow(2, eBias - 1) * Math.pow(2, mLen)
+      e = 0
+    }
+  }
+
+  for (; mLen >= 8; buffer[offset + i] = m & 0xff, i += d, m /= 256, mLen -= 8) {}
+
+  e = (e << mLen) | m
+  eLen += mLen
+  for (; eLen > 0; buffer[offset + i] = e & 0xff, i += d, e /= 256, eLen -= 8) {}
+
+  buffer[offset + i - d] |= s * 128
+}

package/node_modules/ieee754/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "ieee754",
+  "description": "Read/write IEEE754 floating point numbers from/to a Buffer or array-like object",
+  "version": "1.2.1",
+  "author": {
+    "name": "Feross Aboukhadijeh",
+    "email": "feross@feross.org",
+    "url": "https://feross.org"
+  },
+  "contributors": [
+    "Romain Beauxis <toots@rastageeks.org>"
+  ],
+  "devDependencies": {
+    "airtap": "^3.0.0",
+    "standard": "*",
+    "tape": "^5.0.1"
+  },
+  "keywords": [
+    "IEEE 754",
+    "buffer",
+    "convert",
+    "floating point",
+    "ieee754"
+  ],
+  "license": "BSD-3-Clause",
+  "main": "index.js",
+  "types": "index.d.ts",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/feross/ieee754.git"
+  },
+  "scripts": {
+    "test": "standard && npm run test-node && npm run test-browser",
+    "test-browser": "airtap -- test/*.js",
+    "test-browser-local": "airtap --local -- test/*.js",
+    "test-node": "tape test/*.js"
+  },
+  "funding": [
+    {
+      "type": "github",
+      "url": "https://github.com/sponsors/feross"
+    },
+    {
+      "type": "patreon",
+      "url": "https://www.patreon.com/feross"
+    },
+    {
+      "type": "consulting",
+      "url": "https://feross.org/support"
+    }
+  ]
+}

package/node_modules/peek-readable/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2010-2017 Borewit
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/node_modules/peek-readable/README.md

@@ -0,0 +1,93 @@
+![Node.js CI](https://github.com/Borewit/peek-readable/workflows/Node.js%20CI/badge.svg)
+[![NPM version](https://badge.fury.io/js/peek-readable.svg)](https://npmjs.org/package/peek-readable)
+[![npm downloads](http://img.shields.io/npm/dm/peek-readable.svg)](https://npmcharts.com/compare/peek-readable?start=600&interval=30)
+[![Coverage Status](https://coveralls.io/repos/github/Borewit/peek-readable/badge.svg?branch=master)](https://coveralls.io/github/Borewit/peek-readable?branch=master)
+[![Codacy Badge](https://app.codacy.com/project/badge/Grade/d4b511481b3a4634b6ca5c0724407eb9)](https://www.codacy.com/gh/Borewit/peek-readable/dashboard?utm_source=github.com&utm_medium=referral&utm_content=Borewit/peek-readable&utm_campaign=Badge_Grade)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/Borewit/peek-readable.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Borewit/peek-readable/alerts/)
+[![Language grade: JavaScript](https://img.shields.io/lgtm/grade/javascript/g/Borewit/peek-readable.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/Borewit/peek-readable/context:javascript)
+[![Known Vulnerabilities](https://snyk.io/test/github/Borewit/peek-readable/badge.svg?targetFile=package.json)](https://snyk.io/test/github/Borewit/peek-readable?targetFile=package.json)
+
+# peek-readable
+
+A promise based asynchronous stream reader, which makes reading from a stream easy.
+
+Allows to read and peek from a [Readable Stream](https://nodejs.org/api/stream.html#stream_readable_streams) 
+
+Note that [peek-readable](https://github.com/Borewit/peek-readable) was formally released as [then-read-stream](https://github.com/Borewit/peek-readable).
+
+## Usage
+
+### Installation
+
+```shell script
+npm install --save peek-readable
+```
+
+The `peek-readable` contains one class: `StreamReader`, which reads from a [stream.Readable](https://nodejs.org/api/stream.html#stream_class_stream_readable).
+
+### Compatibility
+
+NPM module is compliant with [ECMAScript 2018 (ES9)](https://en.wikipedia.org/wiki/ECMAScript#9th_Edition_%E2%80%93_ECMAScript_2018).
+
+## Examples
+
+In the following example we read the first 16 bytes from a stream and store them in our buffer.
+Source code of examples can be found [here](test/examples.ts).
+
+```js
+const fs = require('fs');
+const { StreamReader } = require('peek-readable');
+
+(async () => {
+  const readable = fs.createReadStream('JPEG_example_JPG_RIP_001.jpg');
+  const streamReader = new StreamReader(readable);
+  const uint8Array = new Uint8Array(16);
+  const bytesRead = await streamReader.read(uint8Array, 0, 16);;
+  // buffer contains 16 bytes, if the end-of-stream has not been reached
+})();
+```
+
+End-of-stream detection:
+```js
+(async () => {
+
+  const fileReadStream = fs.createReadStream('JPEG_example_JPG_RIP_001.jpg');
+  const streamReader = new StreamReader(fileReadStream);
+  const buffer = Buffer.alloc(16); // or use: new Uint8Array(16);
+
+  try {
+    await streamReader.read(buffer, 0, 16);
+    // buffer contains 16 bytes, if the end-of-stream has not been reached
+  } catch(error) {
+    if (error instanceof EndOfStreamError) {
+      console.log('End-of-stream reached');
+    }
+  }
+})();
+```
+
+With peek you can read ahead:
+```js
+const fs = require('fs');
+const { StreamReader } = require('peek-readable');
+
+const fileReadStream = fs.createReadStream('JPEG_example_JPG_RIP_001.jpg');
+const streamReader = new StreamReader(fileReadStream);
+const buffer = Buffer.alloc(20);
+
+(async () => {
+  let bytesRead = await streamReader.peek(buffer, 0, 3);
+  if (bytesRead === 3 && buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) {
+    console.log('This is a JPEG file');
+  } else {
+    throw Error('Expected a JPEG file');
+  }
+
+  bytesRead = await streamReader.read(buffer, 0, 20); // Read JPEG header
+  if (bytesRead === 20) {
+    console.log('Got the JPEG header');
+  } else {
+    throw Error('Failed to read JPEG header');
+  }
+})();
+```

package/node_modules/peek-readable/lib/Deferred.d.ts

@@ -0,0 +1,6 @@
+export declare class Deferred<T> {
+    promise: Promise<T>;
+    resolve: (value: T) => void;
+    reject: (reason: any) => void;
+    constructor();
+}

package/node_modules/peek-readable/lib/Deferred.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Deferred = void 0;
+class Deferred {
+    constructor() {
+        this.resolve = () => null;
+        this.reject = () => null;
+        this.promise = new Promise((resolve, reject) => {
+            this.reject = reject;
+            this.resolve = resolve;
+        });
+    }
+}
+exports.Deferred = Deferred;

package/node_modules/peek-readable/lib/EndOfFileStream.d.ts

@@ -0,0 +1,7 @@
+export declare const defaultMessages = "End-Of-Stream";
+/**
+ * Thrown on read operation of the end of file or stream has been reached
+ */
+export declare class EndOfStreamError extends Error {
+    constructor();
+}

package/node_modules/peek-readable/lib/EndOfFileStream.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EndOfStreamError = exports.defaultMessages = void 0;
+exports.defaultMessages = 'End-Of-Stream';
+/**
+ * Thrown on read operation of the end of file or stream has been reached
+ */
+class EndOfStreamError extends Error {
+    constructor() {
+        super(exports.defaultMessages);
+    }
+}
+exports.EndOfStreamError = EndOfStreamError;

package/node_modules/peek-readable/lib/StreamReader.d.ts

@@ -0,0 +1,47 @@
+/// <reference types="node" />
+import { Readable } from 'stream';
+export { EndOfStreamError } from './EndOfFileStream';
+export declare class StreamReader {
+    private s;
+    /**
+     * Deferred used for postponed read request (as not data is yet available to read)
+     */
+    private deferred;
+    private endOfStream;
+    /**
+     * Store peeked data
+     * @type {Array}
+     */
+    private peekQueue;
+    constructor(s: Readable);
+    /**
+     * Read ahead (peek) from stream. Subsequent read or peeks will return the same data
+     * @param uint8Array - Uint8Array (or Buffer) to store data read from stream in
+     * @param offset - Offset target
+     * @param length - Number of bytes to read
+     * @returns Number of bytes peeked
+     */
+    peek(uint8Array: Uint8Array, offset: number, length: number): Promise<number>;
+    /**
+     * Read chunk from stream
+     * @param buffer - Target Uint8Array (or Buffer) to store data read from stream in
+     * @param offset - Offset target
+     * @param length - Number of bytes to read
+     * @returns Number of bytes read
+     */
+    read(buffer: Uint8Array, offset: number, length: number): Promise<number>;
+    /**
+     * Read chunk from stream
+     * @param buffer Target Uint8Array (or Buffer) to store data read from stream in
+     * @param offset Offset target
+     * @param length Number of bytes to read
+     * @returns Number of bytes read
+     */
+    private readFromStream;
+    /**
+     * Process deferred read request
+     * @param request Deferred read request
+     */
+    private readDeferred;
+    private reject;
+}

package/node_modules/peek-readable/lib/StreamReader.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamReader = exports.EndOfStreamError = void 0;
+const EndOfFileStream_1 = require("./EndOfFileStream");
+const Deferred_1 = require("./Deferred");
+var EndOfFileStream_2 = require("./EndOfFileStream");
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return EndOfFileStream_2.EndOfStreamError; } });
+const maxStreamReadSize = 1 * 1024 * 1024; // Maximum request length on read-stream operation
+class StreamReader {
+    constructor(s) {
+        this.s = s;
+        /**
+         * Deferred used for postponed read request (as not data is yet available to read)
+         */
+        this.deferred = null;
+        this.endOfStream = false;
+        /**
+         * Store peeked data
+         * @type {Array}
+         */
+        this.peekQueue = [];
+        if (!s.read || !s.once) {
+            throw new Error('Expected an instance of stream.Readable');
+        }
+        this.s.once('end', () => this.reject(new EndOfFileStream_1.EndOfStreamError()));
+        this.s.once('error', err => this.reject(err));
+        this.s.once('close', () => this.reject(new Error('Stream closed')));
+    }
+    /**
+     * Read ahead (peek) from stream. Subsequent read or peeks will return the same data
+     * @param uint8Array - Uint8Array (or Buffer) to store data read from stream in
+     * @param offset - Offset target
+     * @param length - Number of bytes to read
+     * @returns Number of bytes peeked
+     */
+    async peek(uint8Array, offset, length) {
+        const bytesRead = await this.read(uint8Array, offset, length);
+        this.peekQueue.push(uint8Array.subarray(offset, offset + bytesRead)); // Put read data back to peek buffer
+        return bytesRead;
+    }
+    /**
+     * Read chunk from stream
+     * @param buffer - Target Uint8Array (or Buffer) to store data read from stream in
+     * @param offset - Offset target
+     * @param length - Number of bytes to read
+     * @returns Number of bytes read
+     */
+    async read(buffer, offset, length) {
+        if (length === 0) {
+            return 0;
+        }
+        if (this.peekQueue.length === 0 && this.endOfStream) {
+            throw new EndOfFileStream_1.EndOfStreamError();
+        }
+        let remaining = length;
+        let bytesRead = 0;
+        // consume peeked data first
+        while (this.peekQueue.length > 0 && remaining > 0) {
+            const peekData = this.peekQueue.pop(); // Front of queue
+            if (!peekData)
+                throw new Error('peekData should be defined');
+            const lenCopy = Math.min(peekData.length, remaining);
+            buffer.set(peekData.subarray(0, lenCopy), offset + bytesRead);
+            bytesRead += lenCopy;
+            remaining -= lenCopy;
+            if (lenCopy < peekData.length) {
+                // remainder back to queue
+                this.peekQueue.push(peekData.subarray(lenCopy));
+            }
+        }
+        // continue reading from stream if required
+        while (remaining > 0 && !this.endOfStream) {
+            const reqLen = Math.min(remaining, maxStreamReadSize);
+            const chunkLen = await this.readFromStream(buffer, offset + bytesRead, reqLen);
+            bytesRead += chunkLen;
+            if (chunkLen < reqLen)
+                break;
+            remaining -= chunkLen;
+        }
+        return bytesRead;
+    }
+    /**
+     * Read chunk from stream
+     * @param buffer Target Uint8Array (or Buffer) to store data read from stream in
+     * @param offset Offset target
+     * @param length Number of bytes to read
+     * @returns Number of bytes read
+     */
+    async readFromStream(buffer, offset, length) {
+        const readBuffer = this.s.read(length);
+        if (readBuffer) {
+            buffer.set(readBuffer, offset);
+            return readBuffer.length;
+        }
+        else {
+            const request = {
+                buffer,
+                offset,
+                length,
+                deferred: new Deferred_1.Deferred()
+            };
+            this.deferred = request.deferred;
+            this.s.once('readable', () => {
+                this.readDeferred(request);
+            });
+            return request.deferred.promise;
+        }
+    }
+    /**
+     * Process deferred read request
+     * @param request Deferred read request
+     */
+    readDeferred(request) {
+        const readBuffer = this.s.read(request.length);
+        if (readBuffer) {
+            request.buffer.set(readBuffer, request.offset);
+            request.deferred.resolve(readBuffer.length);
+            this.deferred = null;
+        }
+        else {
+            this.s.once('readable', () => {
+                this.readDeferred(request);
+            });
+        }
+    }
+    reject(err) {
+        this.endOfStream = true;
+        if (this.deferred) {
+            this.deferred.reject(err);
+            this.deferred = null;
+        }
+    }
+}
+exports.StreamReader = StreamReader;

package/node_modules/peek-readable/lib/index.d.ts

@@ -0,0 +1,2 @@
+export { EndOfStreamError } from './EndOfFileStream';
+export { StreamReader } from './StreamReader';

package/node_modules/peek-readable/lib/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamReader = exports.EndOfStreamError = void 0;
+var EndOfFileStream_1 = require("./EndOfFileStream");
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return EndOfFileStream_1.EndOfStreamError; } });
+var StreamReader_1 = require("./StreamReader");
+Object.defineProperty(exports, "StreamReader", { enumerable: true, get: function () { return StreamReader_1.StreamReader; } });

package/node_modules/peek-readable/package.json

@@ -0,0 +1,87 @@
+{
+  "name": "peek-readable",
+  "version": "4.1.0",
+  "description": "Read and peek from a readable stream",
+  "author": {
+    "name": "Borewit",
+    "url": "https://github.com/Borewit"
+  },
+  "funding": {
+    "type": "github",
+    "url": "https://github.com/sponsors/Borewit"
+  },
+  "scripts": {
+    "clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map coverage .nyc_output",
+    "build": "npm run clean && npm run compile",
+    "compile-src": "tsc -p lib",
+    "compile-test": "tsc -p test",
+    "compile": "npm run compile-src && yarn run compile-test",
+    "eslint": "eslint lib test --ext .ts --ignore-pattern *.d.ts",
+    "lint-md": "remark -u preset-lint-recommended .",
+    "lint": "npm run lint-md && npm run eslint",
+    "test": "mocha --require ts-node/register --require source-map-support/register --full-trace test/*.ts",
+    "test-coverage": "nyc npm run test",
+    "send-coveralls": "nyc report --reporter=text-lcov | coveralls",
+    "start": "npm run compile && npm run lint && npm run cover-test"
+  },
+  "engines": {
+    "node": ">=8"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Borewit/peek-readable"
+  },
+  "license": "MIT",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "bugs": {
+    "url": "https://github.com/Borewit/peek-readable/issues"
+  },
+  "files": [
+    "lib/**/*.js",
+    "lib/**/*.d.ts"
+  ],
+  "devDependencies": {
+    "@types/chai": "^4.3.0",
+    "@types/mocha": "^9.1.0",
+    "@types/node": "^17.0.0",
+    "@typescript-eslint/eslint-plugin": "^4.33.0",
+    "@typescript-eslint/parser": "^4.33.0",
+    "add": "^2.0.6",
+    "chai": "^4.3.6",
+    "coveralls": "^3.1.1",
+    "del-cli": "^4.0.1",
+    "eslint": "^7.32.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-import-resolver-typescript": "^2.5.0",
+    "eslint-plugin-import": "^2.25.4",
+    "eslint-plugin-jsdoc": "^37.7.1",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-unicorn": "^40.1.0",
+    "mocha": "^9.2.0",
+    "nyc": "^15.1.0",
+    "remark-cli": "^10.0.1",
+    "remark-preset-lint-recommended": "^6.1.2",
+    "ts-node": "^10.4.0",
+    "typescript": "^4.5.5"
+  },
+  "keywords": [
+    "readable",
+    "buffer",
+    "stream",
+    "read"
+  ],
+  "nyc": {
+    "check-coverage": true,
+    "extension": [
+      ".ts"
+    ],
+    "sourceMap": true,
+    "instrument": true,
+    "reporter": [
+      "lcov",
+      "text"
+    ],
+    "report-dir": "coverage"
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dragon708/docmind-markdown",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "StructuredDocumentResult → Markdown and LLM-oriented plain text for DocMind.",
   "type": "module",
   "sideEffects": false,
@@ -46,6 +46,7 @@
   "bundledDependencies": [
     "@cognipeer/to-markdown",
     "@opendocsg/pdf2md",
+    "@tokenizer/token",
     "adm-zip",
     "adler-32",
     "boolbase",
@@ -65,6 +66,7 @@
     "enumify",
     "file-type",
     "htmlparser2",
+    "ieee754",
     "iconv-lite",
     "image-size",
     "mime-db",
@@ -73,6 +75,7 @@
     "music-metadata",
     "nth-check",
     "papaparse",
+    "peek-readable",
     "parse5",
     "parse5-htmlparser2-tree-adapter",
     "parse5-parser-stream",