cspell-io 6.26.2 → 6.27.0

package/dist/common/BufferEncoding.d.ts

@@ -0,0 +1,4 @@
+import type { BufferEncoding } from '../models/BufferEncoding';
+export type { BufferEncoding } from '../models/BufferEncoding';
+export type BufferEncodingExt = BufferEncoding | 'utf16be';
+//# sourceMappingURL=BufferEncoding.d.ts.map

package/dist/common/BufferEncoding.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=BufferEncoding.js.map

package/dist/common/encode-decode.d.ts

@@ -0,0 +1,12 @@
+/// <reference types="node" />
+import type { BufferEncodingExt } from './BufferEncoding';
+export declare function decodeUtf16LE(buf: Buffer): string;
+export declare function decodeUtf16BE(buf: Buffer): string;
+export declare function decode(buf: Buffer, encoding?: BufferEncodingExt): string;
+export declare function swapBytesInPlace(buf: Buffer): Buffer;
+export declare function swapBytes(buf: Buffer): Buffer;
+export declare function encodeString(str: string, encoding?: BufferEncodingExt, bom?: boolean): Buffer;
+export declare function encodeUtf16LE(str: string, bom?: boolean): Buffer;
+export declare function encodeUtf16BE(str: string, bom?: boolean): Buffer;
+export declare function calcEncodingFromBom(buf: Buffer): 'utf16be' | 'utf16le' | undefined;
+//# sourceMappingURL=encode-decode.d.ts.map

package/dist/common/encode-decode.js

@@ -0,0 +1,84 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.calcEncodingFromBom = exports.encodeUtf16BE = exports.encodeUtf16LE = exports.encodeString = exports.swapBytes = exports.swapBytesInPlace = exports.decode = exports.decodeUtf16BE = exports.decodeUtf16LE = void 0;
+const BOM_BE = 0xfeff;
+const BOM_LE = 0xfffe;
+function decodeUtf16LE(buf) {
+    const bom = (buf[0] << 8) | buf[1];
+    buf = bom === BOM_LE ? buf.subarray(2) : buf;
+    return buf.toString('utf16le');
+}
+exports.decodeUtf16LE = decodeUtf16LE;
+function decodeUtf16BE(buf) {
+    return decodeUtf16LE(swapBytes(buf));
+}
+exports.decodeUtf16BE = decodeUtf16BE;
+function decode(buf, encoding) {
+    switch (encoding) {
+        case 'utf16be':
+            return decodeUtf16BE(buf);
+        case 'utf16le':
+            return decodeUtf16LE(buf);
+    }
+    if (buf.length < 2 || (encoding && !encoding.startsWith('utf')))
+        return buf.toString(encoding);
+    const bom = (buf[0] << 8) | buf[1];
+    if (bom === BOM_BE || (buf[0] === 0 && buf[1] !== 0))
+        return decodeUtf16BE(buf);
+    if (bom === BOM_LE || (buf[0] !== 0 && buf[1] === 0))
+        return decodeUtf16LE(buf);
+    return buf.toString(encoding);
+}
+exports.decode = decode;
+function swapBytesInPlace(buf) {
+    for (let i = 0; i < buf.length - 1; i += 2) {
+        const v = buf[i];
+        buf[i] = buf[i + 1];
+        buf[i + 1] = v;
+    }
+    return buf;
+}
+exports.swapBytesInPlace = swapBytesInPlace;
+function swapBytes(buf) {
+    const tBuf = Buffer.from(buf);
+    return swapBytesInPlace(tBuf);
+}
+exports.swapBytes = swapBytes;
+function encodeString(str, encoding, bom) {
+    switch (encoding) {
+        case 'utf16be':
+            return encodeUtf16BE(str, bom);
+        case 'utf16le':
+            return encodeUtf16LE(str, bom);
+    }
+    return Buffer.from(str, encoding);
+}
+exports.encodeString = encodeString;
+function encodeUtf16LE(str, bom = true) {
+    const buf = Buffer.from(str, 'utf16le');
+    if (bom) {
+        const target = Buffer.alloc(buf.length + 2);
+        target.writeUint16LE(BOM_BE);
+        buf.copy(target, 2);
+        return target;
+    }
+    return buf;
+}
+exports.encodeUtf16LE = encodeUtf16LE;
+function encodeUtf16BE(str, bom = true) {
+    return swapBytesInPlace(encodeUtf16LE(str, bom));
+}
+exports.encodeUtf16BE = encodeUtf16BE;
+function calcEncodingFromBom(buf) {
+    if (buf.length < 2)
+        return undefined;
+    switch ((buf[0] << 8) | buf[1]) {
+        case BOM_BE:
+            return 'utf16be';
+        case BOM_LE:
+            return 'utf16le';
+    }
+    return undefined;
+}
+exports.calcEncodingFromBom = calcEncodingFromBom;
+//# sourceMappingURL=encode-decode.js.map

package/dist/common/transformers.d.ts

@@ -0,0 +1,7 @@
+/// <reference types="node" />
+import type { BufferEncodingExt } from './BufferEncoding';
+export declare function createDecoderTransformer(encoding?: BufferEncodingExt): (iterable: AsyncIterable<string | Buffer> | Iterable<string | Buffer>) => AsyncIterable<string>;
+export declare function encoderTransformer(iterable: Iterable<string>, encoding?: BufferEncodingExt): Iterable<Buffer>;
+export declare function encoderTransformer(iterable: AsyncIterable<string>, encoding?: BufferEncodingExt): AsyncIterable<Buffer>;
+export declare function encoderTransformer(iterable: Iterable<string> | AsyncIterable<string>, encoding?: BufferEncodingExt): Iterable<Buffer> | AsyncIterable<Buffer>;
+//# sourceMappingURL=transformers.d.ts.map

package/dist/common/transformers.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encoderTransformer = exports.createDecoderTransformer = void 0;
+const encode_decode_1 = require("./encode-decode");
+function createDecoderTransformer(encoding) {
+    function standardDecoder(buf) {
+        return (0, encode_decode_1.decode)(buf, encoding);
+    }
+    let decoder = undefined;
+    if (encoding && !encoding.startsWith('utf'))
+        return decoderNonUtf;
+    return decoderUtf;
+    async function* decoderNonUtf(iterable) {
+        for await (const buf of iterable) {
+            yield typeof buf === 'string' ? buf : (0, encode_decode_1.decode)(buf, encoding);
+        }
+    }
+    async function* decoderUtf(iterable) {
+        for await (const sb of iterable) {
+            if (typeof sb === 'string') {
+                yield sb;
+                continue;
+            }
+            if (sb.length < 2) {
+                yield standardDecoder(sb);
+                continue;
+            }
+            if (decoder) {
+                yield decoder(sb);
+                continue;
+            }
+            decoder = standardDecoder;
+            const _encoding = (0, encode_decode_1.calcEncodingFromBom)(sb);
+            if (_encoding === 'utf16le') {
+                decoder = encode_decode_1.decodeUtf16LE;
+                yield decoder(sb.subarray(2));
+                continue;
+            }
+            if (_encoding === 'utf16be') {
+                decoder = encode_decode_1.decodeUtf16BE;
+                yield decoder(sb.subarray(2));
+                continue;
+            }
+            yield decoder(sb);
+        }
+    }
+}
+exports.createDecoderTransformer = createDecoderTransformer;
+function encoderTransformer(iterable, encoding) {
+    return isAsyncIterable(iterable) ? encoderAsyncIterable(iterable, encoding) : encoderIterable(iterable, encoding);
+}
+exports.encoderTransformer = encoderTransformer;
+function* encoderIterable(iterable, encoding) {
+    let useBom = true;
+    for (const chunk of iterable) {
+        yield (0, encode_decode_1.encodeString)(chunk, encoding, useBom);
+        useBom = false;
+    }
+}
+async function* encoderAsyncIterable(iterable, encoding) {
+    let useBom = true;
+    for await (const chunk of iterable) {
+        yield (0, encode_decode_1.encodeString)(chunk, encoding, useBom);
+        useBom = false;
+    }
+}
+function isAsyncIterable(v) {
+    return v && typeof v === 'object' && !!v[Symbol.asyncIterator];
+}
+//# sourceMappingURL=transformers.js.map

package/dist/file/index.d.ts

@@ -1,3 +1,3 @@
-export { writeToFile, writeToFileIterable, writeToFileIterableP } from './../node/file';
+export { writeToFile, writeToFileIterable, writeToFileIterable as writeToFileIterableP } from './../node/file';
 export { getStat, getStatSync, readFile, readFileSync } from './file';
 //# sourceMappingURL=index.d.ts.map

package/dist/file/index.js

@@ -4,7 +4,7 @@ exports.readFileSync = exports.readFile = exports.getStatSync = exports.getStat
 var file_1 = require("./../node/file");
 Object.defineProperty(exports, "writeToFile", { enumerable: true, get: function () { return file_1.writeToFile; } });
 Object.defineProperty(exports, "writeToFileIterable", { enumerable: true, get: function () { return file_1.writeToFileIterable; } });
-Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return file_1.writeToFileIterableP; } });
+Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return file_1.writeToFileIterable; } });
 var file_2 = require("./file");
 Object.defineProperty(exports, "getStat", { enumerable: true, get: function () { return file_2.getStat; } });
 Object.defineProperty(exports, "getStatSync", { enumerable: true, get: function () { return file_2.getStatSync; } });

package/dist/handlers/node/file.js

@@ -9,6 +9,7 @@ const assert_1 = __importDefault(require("assert"));
 const fs_1 = require("fs");
 const url_1 = require("url");
 const zlib_1 = require("zlib");
+const encode_decode_1 = require("../../common/encode-decode");
 const errors_1 = require("../../errors");
 const dataUrl_1 = require("../../node/dataUrl");
 const fetch_1 = require("../../node/file/fetch");
@@ -97,7 +98,7 @@ const handleRequestFsReadBinaryFileData = requests_1.RequestFsReadBinaryFile.cre
     return (0, cspell_service_bus_1.createResponse)(Promise.resolve(res.value));
 }, undefined, 'Node: Read data: urls.');
 function bufferToText(buf, encoding) {
-    return buf[0] === 0x1f && buf[1] === 0x8b ? bufferToText((0, zlib_1.gunzipSync)(buf), encoding) : buf.toString(encoding);
+    return buf[0] === 0x1f && buf[1] === 0x8b ? (0, encode_decode_1.decode)((0, zlib_1.gunzipSync)(buf), encoding) : (0, encode_decode_1.decode)(buf, encoding);
 }
 /**
  * Handle fs:stat

package/dist/node/file/fileReader.js

@@ -27,62 +27,57 @@ exports.readFileSync = exports.readFile = void 0;
 // cSpell:ignore curr
 // cSpell:words zlib iconv
 const fs = __importStar(require("fs"));
+const Stream = __importStar(require("stream"));
 const url_1 = require("url");
+const util_1 = require("util");
 const zlib = __importStar(require("zlib"));
+const encode_decode_1 = require("../../common/encode-decode");
+const transformers_1 = require("../../common/transformers");
 const fetch_1 = require("./fetch");
 const FetchError_1 = require("./FetchError");
-const util_1 = require("./util");
+const util_2 = require("./util");
 const defaultEncoding = 'utf8';
+const pipeline = (0, util_1.promisify)(Stream.pipeline);
 async function readFile(filename, encoding) {
-    const url = (0, util_1.toURL)(filename);
-    if (!(0, util_1.isSupportedURL)(url)) {
+    const url = (0, util_2.toURL)(filename);
+    if (!(0, util_2.isSupportedURL)(url)) {
         throw new Error('Unsupported network protocol');
     }
-    return (0, util_1.isFileURL)(url) ? _readFile(url, encoding) : _fetchURL(url, encoding);
+    return (0, util_2.isFileURL)(url) ? _readFile(url, encoding) : _fetchURL(url, encoding);
 }
 exports.readFile = readFile;
 function _readFile(url, encoding) {
     // Convert it to a string because Yarn2 cannot handle URLs.
     const filename = (0, url_1.fileURLToPath)(url);
-    return _read(() => fs.createReadStream(filename), (0, util_1.isZipped)(filename), encoding);
+    return _read(() => fs.createReadStream(filename), (0, util_2.isZipped)(filename), encoding);
 }
 async function _fetchURL(url, encoding) {
     const response = await (0, fetch_1.fetch)(url);
     if (!response.ok) {
         throw FetchError_1.FetchUrlError.create(url, response.status);
     }
-    return _read(() => response.body, (0, util_1.isZipped)(url), encoding);
+    return _read(() => response.body, (0, util_2.isZipped)(url), encoding);
 }
-function _read(getStream, isZipped, encoding = defaultEncoding) {
-    return new Promise((resolve, reject) => {
-        const data = [];
-        const stream = prepareFileStream(getStream, isZipped, encoding, reject);
-        let resolved = false;
-        function complete() {
-            resolve(data.join(''));
-            resolved = resolved || (resolve(data.join('')), true);
-        }
-        stream.on('error', reject);
-        stream.on('data', (d) => data.push(d));
-        stream.on('close', complete);
-        stream.on('end', complete);
-    });
-}
-function prepareFileStream(getStream, isZipped, encoding, fnError) {
-    const pipes = [];
-    if (isZipped) {
-        pipes.push(zlib.createGunzip());
-    }
-    const fileStream = getStream();
-    fileStream.on('error', fnError);
-    const stream = pipes.reduce((s, p) => s.pipe(p).on('error', fnError), fileStream);
-    stream.setEncoding(encoding);
-    return stream;
+async function _read(getStream, isZipped, encoding) {
+    const stream = getStream();
+    const decoder = (0, transformers_1.createDecoderTransformer)(encoding);
+    const collector = createCollector(encoding || defaultEncoding);
+    return isZipped ? pipeline(stream, zlib.createGunzip(), decoder, collector) : pipeline(stream, decoder, collector);
 }
-function readFileSync(filename, encoding = defaultEncoding) {
+function readFileSync(filename, encoding) {
     const rawData = fs.readFileSync(filename);
-    const data = (0, util_1.isZipped)(filename) ? zlib.gunzipSync(rawData) : rawData;
-    return data.toString(encoding);
+    const data = (0, util_2.isZipped)(filename) ? zlib.gunzipSync(rawData) : rawData;
+    return !encoding || encoding.startsWith('utf') ? (0, encode_decode_1.decode)(data) : data.toString(encoding);
 }
 exports.readFileSync = readFileSync;
+function createCollector(encoding) {
+    async function collect(iterable) {
+        const buf = [];
+        for await (const sb of iterable) {
+            buf.push(typeof sb === 'string' ? sb : sb.toString(encoding));
+        }
+        return buf.join('');
+    }
+    return collect;
+}
 //# sourceMappingURL=fileReader.js.map

package/dist/node/file/fileWriter.d.ts

@@ -1,5 +1,4 @@
-/// <reference types="node" />
-export declare function writeToFile(filename: string, data: string): NodeJS.WritableStream;
-export declare function writeToFileIterable(filename: string, data: Iterable<string>): NodeJS.WritableStream;
-export declare function writeToFileIterableP(filename: string, data: Iterable<string>): Promise<void>;
+import type { BufferEncoding, BufferEncodingExt } from '../../common/BufferEncoding';
+export declare function writeToFile(filename: string, data: string | Iterable<string> | AsyncIterable<string>, encoding?: BufferEncoding): Promise<void>;
+export declare function writeToFileIterable(filename: string, data: Iterable<string> | AsyncIterable<string>, encoding?: BufferEncodingExt): Promise<void>;
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/node/file/fileWriter.js

@@ -23,27 +23,21 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeToFileIterableP = exports.writeToFileIterable = exports.writeToFile = void 0;
+exports.writeToFileIterable = exports.writeToFile = void 0;
 const fs = __importStar(require("fs"));
-const stream = __importStar(require("stream"));
+const Stream = __importStar(require("stream"));
+const util_1 = require("util");
 const zlib = __importStar(require("zlib"));
-function writeToFile(filename, data) {
-    return writeToFileIterable(filename, [data]);
+const transformers_1 = require("../../common/transformers");
+const pipeline = (0, util_1.promisify)(Stream.pipeline);
+function writeToFile(filename, data, encoding) {
+    return writeToFileIterable(filename, typeof data === 'string' ? [data] : data, encoding);
 }
 exports.writeToFile = writeToFile;
-function writeToFileIterable(filename, data) {
-    const sourceStream = stream.Readable.from(data);
-    const writeStream = fs.createWriteStream(filename);
-    const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new stream.PassThrough();
-    return sourceStream.pipe(zip).pipe(writeStream);
+function writeToFileIterable(filename, data, encoding) {
+    const stream = Stream.Readable.from((0, transformers_1.encoderTransformer)(data, encoding));
+    const zip = filename.match(/\.gz$/) ? zlib.createGzip() : new Stream.PassThrough();
+    return pipeline(stream, zip, fs.createWriteStream(filename));
 }
 exports.writeToFileIterable = writeToFileIterable;
-function writeToFileIterableP(filename, data) {
-    const stream = writeToFileIterable(filename, data);
-    return new Promise((resolve, reject) => {
-        stream.on('finish', () => resolve());
-        stream.on('error', (e) => reject(e));
-    });
-}
-exports.writeToFileIterableP = writeToFileIterableP;
 //# sourceMappingURL=fileWriter.js.map

package/dist/node/file/index.d.ts

@@ -1,4 +1,4 @@
 export { readFile, readFileSync } from './fileReader';
-export { writeToFile, writeToFileIterable, writeToFileIterableP } from './fileWriter';
+export { writeToFile, writeToFileIterable } from './fileWriter';
 export { getStat, getStatSync } from './stat';
 //# sourceMappingURL=index.d.ts.map

package/dist/node/file/index.js

@@ -1,13 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getStatSync = exports.getStat = exports.writeToFileIterableP = exports.writeToFileIterable = exports.writeToFile = exports.readFileSync = exports.readFile = void 0;
+exports.getStatSync = exports.getStat = exports.writeToFileIterable = exports.writeToFile = exports.readFileSync = exports.readFile = void 0;
 var fileReader_1 = require("./fileReader");
 Object.defineProperty(exports, "readFile", { enumerable: true, get: function () { return fileReader_1.readFile; } });
 Object.defineProperty(exports, "readFileSync", { enumerable: true, get: function () { return fileReader_1.readFileSync; } });
 var fileWriter_1 = require("./fileWriter");
 Object.defineProperty(exports, "writeToFile", { enumerable: true, get: function () { return fileWriter_1.writeToFile; } });
 Object.defineProperty(exports, "writeToFileIterable", { enumerable: true, get: function () { return fileWriter_1.writeToFileIterable; } });
-Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return fileWriter_1.writeToFileIterableP; } });
 var stat_1 = require("./stat");
 Object.defineProperty(exports, "getStat", { enumerable: true, get: function () { return stat_1.getStat; } });
 Object.defineProperty(exports, "getStatSync", { enumerable: true, get: function () { return stat_1.getStatSync; } });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cspell-io",
-  "version": "6.26.2",
+  "version": "6.27.0",
   "description": "A library of useful I/O functions used across various cspell tools.",
   "main": "dist/index.js",
   "typings": "dist/index.d.ts",
@@ -38,14 +38,14 @@
     "node": ">=14"
   },
   "devDependencies": {
-    "@types/node": "^18.13.0",
+    "@types/node": "^18.14.1",
     "@types/node-fetch": "^2.6.2",
     "jest": "^29.4.3",
     "lorem-ipsum": "^2.0.8"
   },
   "dependencies": {
-    "@cspell/cspell-service-bus": "6.26.2",
+    "@cspell/cspell-service-bus": "6.27.0",
     "node-fetch": "^2.6.9"
   },
-  "gitHead": "9c983b5f96c3d7b089e49178ab0b437b5eada256"
+  "gitHead": "b0e31c7ba91ba467d5fd9c66f238bb5d899f4833"
 }