@cspell/cspell-tools 6.11.1 → 6.13.0

package/dist/compiler/Reader.js

@@ -23,16 +23,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.__testing__ = exports.readHunspellFiles = exports.createArrayReader = exports.createReader = void 0;
-const gensequence_1 = require("gensequence");
-const HR = __importStar(require("hunspell-reader"));
-const fs = __importStar(require("fs-extra"));
+exports.readHunspellFiles = exports.createReader = void 0;
+const sync_1 = require("@cspell/cspell-pipe/sync");
 const cspell_trie_lib_1 = require("cspell-trie-lib");
-const zlib = __importStar(require("zlib"));
+const HR = __importStar(require("hunspell-reader"));
+const readTextFile_1 = require("./readTextFile");
+const wordListParser_1 = require("./wordListParser");
 const regHunspellFile = /\.(dic|aff)$/i;
 // cspell:word dedupe
 const DEDUPE_SIZE = 1000;
-const regExMatchComments = /\s*(#|\/\/).*/;
 // Readers first match wins
 const readers = [
     { test: /\.trie\b/, method: trieFileReader },
@@ -49,115 +48,66 @@ function findMatchingReader(filename, options) {
 async function createReader(filename, options) {
     const baseReader = await findMatchingReader(filename, options);
     return Object.assign(baseReader, {
-        [Symbol.iterator]: options.useAnnotation ? baseReader.annotatedWords : baseReader.rawWords,
+        [Symbol.iterator]: () => baseReader.words[Symbol.iterator](),
     });
 }
 exports.createReader = createReader;
-function createArrayReader(lines) {
-    const rawWords = () => (0, gensequence_1.genSequence)(lines);
-    const annotatedWords = () => (0, gensequence_1.genSequence)(lines).pipe(_mapText, dedupeAndSort);
-    return {
-        size: lines.length,
-        annotatedWords,
-        rawWords,
-    };
-}
-exports.createArrayReader = createArrayReader;
 async function readHunspellFiles(filename, options) {
     const dicFile = filename.replace(regHunspellFile, '.dic');
     const affFile = filename.replace(regHunspellFile, '.aff');
     const reader = await HR.IterableHunspellReader.createFromFiles(affFile, dicFile);
     reader.maxDepth = options.maxDepth !== undefined ? options.maxDepth : reader.maxDepth;
-    const normalizeAndDedupe = gensequence_1.operators.pipe(_stripCaseAndAccents, dedupeAndSort);
-    const rawWords = () => reader.seqWords();
+    const words = (0, sync_1.pipe)(reader.seqAffWords(), _mapAffWords, dedupeAndSort);
     return {
         size: reader.dic.length,
-        annotatedWords() {
-            return reader.seqAffWords().pipe(_mapAffWords, normalizeAndDedupe);
-        },
-        rawWords,
+        words,
     };
 }
 exports.readHunspellFiles = readHunspellFiles;
 async function trieFileReader(filename) {
-    const trieRoot = (0, cspell_trie_lib_1.importTrie)(await readTextFile(filename));
+    const trieRoot = (0, cspell_trie_lib_1.importTrie)(await (0, readTextFile_1.readTextFileLines)(filename));
     const trie = new cspell_trie_lib_1.Trie(trieRoot);
-    const rawWords = () => trie.words();
+    const words = trie.words();
     return {
         get size() {
             return trie.size();
         },
-        annotatedWords: rawWords,
-        rawWords,
+        words,
     };
 }
-function readTextFile(filename) {
-    const lines = fs
-        .readFile(filename)
-        .then((buffer) => (/\.gz$/.test(filename) ? zlib.gunzipSync(buffer) : buffer))
-        .then((buffer) => buffer.toString('utf8'))
-        .then((content) => content.split(/\r?\n/g));
-    return lines;
-}
-async function textFileReader(filename, _options) {
-    const lines = await readTextFile(filename);
-    return createArrayReader(lines);
-}
-const _mapText = gensequence_1.operators.pipe(_comments, _compoundBegin, _compoundEnd, _stripCaseAndAccents);
-function* _comments(lines) {
-    for (const line of lines) {
-        const w = line.replace(regExMatchComments, '').trim();
-        if (w)
-            yield w;
-    }
-}
-function* _compoundEnd(lines) {
-    for (const line of lines) {
-        if (line[0] !== cspell_trie_lib_1.OPTIONAL_COMPOUND_FIX) {
-            yield line;
-            continue;
-        }
-        const w = line.slice(1);
-        yield w;
-        yield cspell_trie_lib_1.COMPOUND_FIX + w;
-    }
-}
-function* _compoundBegin(lines) {
-    for (const line of lines) {
-        if (line[line.length - 1] !== cspell_trie_lib_1.OPTIONAL_COMPOUND_FIX) {
-            yield line;
-            continue;
-        }
-        const w = line.slice(0, -1);
-        yield w;
-        yield w + cspell_trie_lib_1.COMPOUND_FIX;
-    }
-}
-function* _stripCaseAndAccents(words) {
-    for (const word of words) {
-        // Words are normalized to the compact format: e + ` => è
-        yield word.normalize();
-        // covert to lower case and strip accents.
-        const n = word.toLowerCase().normalize('NFD').replace(/\p{M}/gu, '');
-        // All words are added for case-insensitive searches.
-        // It is a space / speed trade-off. In this case, speed is more important.
-        yield cspell_trie_lib_1.CASE_INSENSITIVE_PREFIX + n;
-    }
+async function textFileReader(filename, options) {
+    const content = await (0, readTextFile_1.readTextFile)(filename);
+    const words = [...(0, wordListParser_1.parseFileLines)(content, { legacy: options.legacy, split: options.splitWords })];
+    return {
+        size: words.length,
+        words,
+    };
 }
+// function* _stripCaseAndAccents(words: Iterable<AnnotatedWord>): Iterable<AnnotatedWord> {
+//     for (const word of words) {
+//         // Words are normalized to the compact format: e + ` => è
+//         yield word.normalize();
+//         // covert to lower case and strip accents.
+//         const n = word.toLowerCase().normalize('NFD').replace(/\p{M}/gu, '');
+//         // All words are added for case-insensitive searches.
+//         // It is a space / speed trade-off. In this case, speed is more important.
+//         yield CASE_INSENSITIVE_PREFIX + n;
+//     }
+// }
 function* dedupeAndSort(words) {
-    const buffer = [];
-    function sortDedupeClear() {
-        const s = new Set(buffer.sort());
-        buffer.length = 0;
-        return s;
+    const buffer = new Set();
+    function flush() {
+        const result = [...buffer].sort();
+        buffer.clear();
+        return result;
     }
     for (const word of words) {
-        buffer.push(word);
-        if (buffer.length >= DEDUPE_SIZE) {
-            yield* sortDedupeClear();
+        buffer.add(word);
+        if (buffer.size >= DEDUPE_SIZE) {
+            yield* flush();
         }
     }
-    yield* sortDedupeClear();
+    yield* flush();
 }
 function* _mapAffWords(affWords) {
     const hasSpecial = /[~+!]/;
@@ -183,7 +133,4 @@ function* _mapAffWords(affWords) {
         }
     }
 }
-exports.__testing__ = {
-    _stripCaseAndAccents,
-};
 //# sourceMappingURL=Reader.js.map

package/dist/compiler/compile.d.ts

@@ -0,0 +1,11 @@
+import { CompileRequest, CompileSourceOptions, Target } from '../config';
+interface CompileOptions {
+    /**
+     * Optional filter function to filter targets.
+     */
+    filter?: (target: Target) => boolean;
+}
+export declare function compile(request: CompileRequest, options?: CompileOptions): Promise<void>;
+export declare function compileTarget(target: Target, options: CompileSourceOptions, rootDir: string): Promise<void>;
+export {};
+//# sourceMappingURL=compile.d.ts.map

package/dist/compiler/compile.js

@@ -0,0 +1,157 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.compileTarget = exports.compile = void 0;
+const cspell_pipe_1 = require("@cspell/cspell-pipe");
+const operators_1 = require("@cspell/cspell-pipe/operators");
+const sync_1 = require("@cspell/cspell-pipe/sync");
+const path = __importStar(require("path"));
+const config_1 = require("../config");
+const iterateWordsFromFile_1 = require("./iterateWordsFromFile");
+const logWithTimestamp_1 = require("./logWithTimestamp");
+const readTextFile_1 = require("./readTextFile");
+const wordListCompiler_1 = require("./wordListCompiler");
+const wordListParser_1 = require("./wordListParser");
+async function compile(request, options) {
+    const { targets } = request;
+    // console.log('Request: %o', request);
+    const rootDir = path.resolve(request.rootDir || '.');
+    const targetOptions = {
+        sort: request.sort,
+        generateNonStrict: request.generateNonStrict,
+    };
+    for (const target of targets) {
+        const keep = options?.filter?.(target) ?? true;
+        if (!keep)
+            continue;
+        const adjustedTarget = { ...targetOptions, ...target };
+        await compileTarget(adjustedTarget, request, rootDir);
+    }
+    (0, logWithTimestamp_1.logWithTimestamp)(`Complete.`);
+}
+exports.compile = compile;
+async function compileTarget(target, options, rootDir) {
+    (0, logWithTimestamp_1.logWithTimestamp)(`Start compile: ${target.name}`);
+    // console.log('Target: %o', target);
+    const { format, sources, trieBase, sort = true, generateNonStrict = false } = target;
+    const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? process.cwd());
+    const generateNonStrictTrie = target.generateNonStrict ?? true;
+    const name = normalizeTargetName(target.name);
+    const useTrie = format.startsWith('trie');
+    const filename = resolveTarget(name, targetDirectory, useTrie, target.compress ?? false);
+    const filesToProcessAsync = (0, cspell_pipe_1.pipeAsync)(readSourceList(sources, rootDir), (0, operators_1.opMapAsync)((src) => readFileSource(src, options)), (0, operators_1.opAwaitAsync)());
+    const filesToProcess = await (0, cspell_pipe_1.toArray)(filesToProcessAsync);
+    const normalizer = (0, wordListParser_1.normalizeTargetWords)({ sort: useTrie || sort, generateNonStrict });
+    const action = useTrie
+        ? async (words, dst) => {
+            return (0, wordListCompiler_1.compileTrie)((0, sync_1.pipe)(words, normalizer), dst, {
+                base: trieBase,
+                sort: false,
+                trie3: format === 'trie3',
+                trie4: format === 'trie4',
+                generateNonStrict: generateNonStrictTrie,
+            });
+        }
+        : async (words, dst) => {
+            return (0, wordListCompiler_1.compileWordList)((0, sync_1.pipe)(words, normalizer), dst, { sort, generateNonStrict });
+        };
+    await processFiles(action, filesToProcess, filename);
+    (0, logWithTimestamp_1.logWithTimestamp)(`Done compile: ${target.name}`);
+}
+exports.compileTarget = compileTarget;
+function rel(filePath) {
+    return path.relative(process.cwd(), filePath);
+}
+async function processFiles(action, filesToProcess, mergeTarget) {
+    const toProcess = filesToProcess;
+    const dst = mergeTarget;
+    const words = (0, sync_1.pipe)(toProcess, (0, sync_1.opMap)((ftp) => {
+        const { src } = ftp;
+        (0, logWithTimestamp_1.logWithTimestamp)('Process "%s" to "%s"', rel(src), rel(dst));
+        return ftp;
+    }), (0, sync_1.opConcatMap)(function* (ftp) {
+        yield* ftp.words;
+        (0, logWithTimestamp_1.logWithTimestamp)('Done processing %s', rel(ftp.src));
+    })
+    // opMap((a) => (console.warn(a), a))
+    );
+    await action(words, dst);
+    (0, logWithTimestamp_1.logWithTimestamp)('Done "%s"', rel(dst));
+}
+function resolveTarget(name, directory, useTrie, useGzCompress) {
+    const ext = ((useTrie && '.trie') || '.txt') + ((useGzCompress && '.gz') || '');
+    const filename = name + ext;
+    return path.resolve(directory, filename);
+}
+function readSourceList(sources, rootDir) {
+    async function* mapSrc() {
+        for (const src of sources) {
+            if ((0, config_1.isFilePath)(src)) {
+                yield { filename: path.resolve(rootDir, src) };
+                continue;
+            }
+            if ((0, config_1.isFileSource)(src)) {
+                yield { ...src, filename: path.resolve(rootDir, src.filename) };
+                continue;
+            }
+            if ((0, config_1.isFileListSource)(src)) {
+                const { listFile, ...rest } = src;
+                const absListFile = path.resolve(rootDir, listFile);
+                const listFileDir = path.dirname(absListFile);
+                const files = await readFileList(absListFile);
+                for (const filename of files) {
+                    yield { ...rest, filename: path.resolve(listFileDir, filename) };
+                }
+            }
+        }
+    }
+    return mapSrc();
+}
+async function readFileList(fileList) {
+    const content = await (0, readTextFile_1.readTextFile)(fileList);
+    return content
+        .split('\n')
+        .map((a) => a.trim())
+        .filter((a) => !!a);
+}
+async function readFileSource(fileSource, sourceOptions) {
+    const { filename, keepRawCase = sourceOptions.keepRawCase || false, split = sourceOptions.split || false, maxDepth, } = fileSource;
+    const legacy = split === 'legacy';
+    const splitWords = legacy ? false : split;
+    // console.warn('fileSource: %o,\n targetOptions %o, \n opt: %o', fileSource, targetOptions, opt);
+    const readerOptions = { maxDepth, legacy, splitWords, keepCase: keepRawCase };
+    (0, logWithTimestamp_1.logWithTimestamp)(`Reading ${path.basename(filename)}`);
+    const stream = await (0, iterateWordsFromFile_1.streamWordsFromFile)(filename, readerOptions);
+    (0, logWithTimestamp_1.logWithTimestamp)(`Done reading ${path.basename(filename)}`);
+    const f = {
+        src: filename,
+        words: stream,
+    };
+    return f;
+}
+function normalizeTargetName(name) {
+    return name.replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '').replace(/[^\p{L}\p{M}.\w\\/-]/gu, '_');
+}
+//# sourceMappingURL=compile.js.map

package/dist/compiler/createCompileRequest.d.ts

@@ -0,0 +1,4 @@
+import { CompileCommonAppOptions } from '../AppOptions';
+import { CompileRequest } from '../config';
+export declare function createCompileRequest(sources: string[], options: CompileCommonAppOptions): CompileRequest;
+//# sourceMappingURL=createCompileRequest.d.ts.map

package/dist/compiler/createCompileRequest.js

@@ -0,0 +1,86 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createCompileRequest = void 0;
+const path = __importStar(require("path"));
+function createCompileRequest(sources, options) {
+    const { max_depth, maxDepth, experimental = [], split, keepRawCase, useLegacySplitter } = options;
+    const targets = calcTargets(sources, options);
+    const generateNonStrict = experimental.includes('compound') || undefined;
+    const req = {
+        targets,
+        generateNonStrict,
+        maxDepth: parseNumber(maxDepth) ?? parseNumber(max_depth),
+        split: useLegacySplitter ? 'legacy' : split,
+        /**
+         * Do not generate lower case / accent free versions of words.
+         * @default false
+         */
+        keepRawCase,
+    };
+    return req;
+}
+exports.createCompileRequest = createCompileRequest;
+function calcTargets(sources, options) {
+    const { merge, output = '.' } = options;
+    const format = calcFormat(options);
+    if (merge) {
+        const target = {
+            name: merge,
+            targetDirectory: output,
+            compress: options.compress,
+            format,
+            sources,
+            sort: options.sort,
+            trieBase: parseNumber(options.trieBase),
+        };
+        return [target];
+    }
+    const targets = sources.map((source) => {
+        const name = toTargetName(path.basename(source));
+        const target = {
+            name,
+            targetDirectory: output,
+            compress: options.compress,
+            format,
+            sources: [source],
+            sort: options.sort,
+            trieBase: parseNumber(options.trieBase),
+        };
+        return target;
+    });
+    return targets;
+}
+function calcFormat(options) {
+    return (options.trie4 && 'trie4') || (options.trie3 && 'trie3') || (options.trie && 'trie') || 'plaintext';
+}
+function toTargetName(sourceFile) {
+    return path.basename(sourceFile).replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '');
+}
+function parseNumber(s) {
+    const n = parseInt(s ?? '');
+    return isNaN(n) ? undefined : n;
+}
+//# sourceMappingURL=createCompileRequest.js.map

package/dist/compiler/fileWriter.d.ts

@@ -1,4 +1,3 @@
-import { Sequence } from 'gensequence';
-export { writeToFile, writeToFileIterableP, writeToFileIterable } from 'cspell-io';
-export declare function writeSeqToFile(seq: Sequence<string>, outFile: string): Promise<void>;
+export { writeTextLinesToFile as writeToFileIterableP } from './writeTextToFile';
+export declare function writeSeqToFile(seq: Iterable<string>, outFile: string): Promise<void>;
 //# sourceMappingURL=fileWriter.d.ts.map

package/dist/compiler/fileWriter.js

@@ -1,13 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeSeqToFile = exports.writeToFileIterable = exports.writeToFileIterableP = exports.writeToFile = void 0;
-const cspell_io_1 = require("cspell-io");
-var cspell_io_2 = require("cspell-io");
-Object.defineProperty(exports, "writeToFile", { enumerable: true, get: function () { return cspell_io_2.writeToFile; } });
-Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return cspell_io_2.writeToFileIterableP; } });
-Object.defineProperty(exports, "writeToFileIterable", { enumerable: true, get: function () { return cspell_io_2.writeToFileIterable; } });
+exports.writeSeqToFile = exports.writeToFileIterableP = void 0;
+const writeTextToFile_1 = require("./writeTextToFile");
+var writeTextToFile_2 = require("./writeTextToFile");
+Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return writeTextToFile_2.writeTextLinesToFile; } });
 function writeSeqToFile(seq, outFile) {
-    return (0, cspell_io_1.writeToFileIterableP)(outFile, seq);
+    return (0, writeTextToFile_1.writeTextLinesToFile)(outFile, seq);
 }
 exports.writeSeqToFile = writeSeqToFile;
 //# sourceMappingURL=fileWriter.js.map

package/dist/compiler/globP.d.ts

@@ -0,0 +1,2 @@
+export declare function globP(pattern: string): Promise<string[]>;
+//# sourceMappingURL=globP.d.ts.map

package/dist/compiler/globP.js

@@ -0,0 +1,16 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.globP = void 0;
+const glob_1 = __importDefault(require("glob"));
+const util_1 = require("util");
+const pGlob = (0, util_1.promisify)(glob_1.default);
+function globP(pattern) {
+    // Convert windows separators.
+    pattern = pattern.replace(/\\/g, '/');
+    return pGlob(pattern);
+}
+exports.globP = globP;
+//# sourceMappingURL=globP.js.map

package/dist/compiler/index.d.ts

@@ -1,2 +1,5 @@
-export * from './wordListCompiler';
+export { compileWordList, compileTrie } from './wordListCompiler';
+export { type Logger, setLogger } from './logger';
+export { compile, compileTarget } from './compile';
+export type { CompileRequest, CompileTargetOptions, RunConfig } from '../config';
 //# sourceMappingURL=index.d.ts.map

package/dist/compiler/index.js

@@ -1,18 +1,12 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
 Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./wordListCompiler"), exports);
+exports.compileTarget = exports.compile = exports.setLogger = exports.compileTrie = exports.compileWordList = void 0;
+var wordListCompiler_1 = require("./wordListCompiler");
+Object.defineProperty(exports, "compileWordList", { enumerable: true, get: function () { return wordListCompiler_1.compileWordList; } });
+Object.defineProperty(exports, "compileTrie", { enumerable: true, get: function () { return wordListCompiler_1.compileTrie; } });
+var logger_1 = require("./logger");
+Object.defineProperty(exports, "setLogger", { enumerable: true, get: function () { return logger_1.setLogger; } });
+var compile_1 = require("./compile");
+Object.defineProperty(exports, "compile", { enumerable: true, get: function () { return compile_1.compile; } });
+Object.defineProperty(exports, "compileTarget", { enumerable: true, get: function () { return compile_1.compileTarget; } });
 //# sourceMappingURL=index.js.map

package/dist/compiler/iterateWordsFromFile.d.ts

@@ -1,4 +1,3 @@
-import { Sequence } from 'gensequence';
 import { ReaderOptions } from './Reader';
-export declare function streamWordsFromFile(filename: string, options: ReaderOptions): Promise<Sequence<string>>;
+export declare function streamWordsFromFile(filename: string, options: ReaderOptions): Promise<Iterable<string>>;
 //# sourceMappingURL=iterateWordsFromFile.d.ts.map

package/dist/compiler/iterateWordsFromFile.js

@@ -4,7 +4,7 @@ exports.streamWordsFromFile = void 0;
 const Reader_1 = require("./Reader");
 async function streamWordsFromFile(filename, options) {
     const reader = await (0, Reader_1.createReader)(filename, options);
-    return reader[Symbol.iterator]();
+    return reader;
 }
 exports.streamWordsFromFile = streamWordsFromFile;
 //# sourceMappingURL=iterateWordsFromFile.js.map

package/dist/compiler/legacyLineToWords.d.ts

@@ -0,0 +1,3 @@
+export declare function legacyLineToWords(line: string, keepCase: boolean): Iterable<string>;
+export declare function legacyLinesToWords(lines: Iterable<string>, keepCase?: boolean): Iterable<string>;
+//# sourceMappingURL=legacyLineToWords.d.ts.map

package/dist/compiler/legacyLineToWords.js

@@ -0,0 +1,54 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.legacyLinesToWords = exports.legacyLineToWords = void 0;
+const sync_1 = require("@cspell/cspell-pipe/sync");
+const Text = __importStar(require("./text"));
+const regNonWord = /[^\p{L}\p{M}' ]+/giu;
+const regExpSpaceOrDash = /[- ]+/g;
+const regExpRepeatChars = /(.)\1{4,}/i;
+function legacyLineToWords(line, keepCase) {
+    // Remove punctuation and non-letters.
+    const filteredLine = line.replace(regNonWord, '|');
+    const wordGroups = filteredLine.split('|');
+    const words = (0, sync_1.pipe)(wordGroups, (0, sync_1.opConcatMap)((a) => [a, ...a.split(regExpSpaceOrDash)]), (0, sync_1.opConcatMap)((a) => splitCamelCase(a)), (0, sync_1.opMap)((a) => a.trim()), (0, sync_1.opFilter)((a) => !!a), (0, sync_1.opFilter)((s) => !regExpRepeatChars.test(s)), (0, sync_1.opMap)((a) => (keepCase ? a : a.toLowerCase())));
+    return words;
+}
+exports.legacyLineToWords = legacyLineToWords;
+function* legacyLinesToWords(lines, keepCase = false) {
+    for (const line of lines) {
+        yield* legacyLineToWords(line, keepCase);
+    }
+}
+exports.legacyLinesToWords = legacyLinesToWords;
+function splitCamelCase(word) {
+    const splitWords = Text.splitCamelCaseWord(word);
+    // We only want to preserve this: "New York" and not "Namespace DNSLookup"
+    if (splitWords.length > 1 && regExpSpaceOrDash.test(word)) {
+        return (0, sync_1.pipe)(splitWords, (0, sync_1.opConcatMap)((w) => w.split(regExpSpaceOrDash)));
+    }
+    return splitWords;
+}
+//# sourceMappingURL=legacyLineToWords.js.map

package/dist/compiler/logWithTimestamp.d.ts

@@ -0,0 +1,3 @@
+import { Logger } from '.';
+export declare const logWithTimestamp: Logger;
+//# sourceMappingURL=logWithTimestamp.d.ts.map

package/dist/compiler/logWithTimestamp.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.logWithTimestamp = void 0;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const logWithTimestamp = (message, ...optionalParams) => {
+    console.log(`${new Date().toISOString()} ${message}`, ...optionalParams);
+};
+exports.logWithTimestamp = logWithTimestamp;
+//# sourceMappingURL=logWithTimestamp.js.map

package/dist/compiler/logger.d.ts

@@ -0,0 +1,4 @@
+export declare type Logger = (message?: any, ...optionalParams: any[]) => void;
+export declare function setLogger(logger?: Logger): void;
+export declare function getLogger(): Logger;
+//# sourceMappingURL=logger.d.ts.map

package/dist/compiler/logger.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getLogger = exports.setLogger = void 0;
+const defaultLogger = console.log;
+let log = defaultLogger;
+function setLogger(logger) {
+    log = logger ?? defaultLogger;
+}
+exports.setLogger = setLogger;
+function getLogger() {
+    return log;
+}
+exports.getLogger = getLogger;
+//# sourceMappingURL=logger.js.map

package/dist/compiler/readTextFile.d.ts

@@ -0,0 +1,3 @@
+export declare function readTextFile(filename: string): Promise<string>;
+export declare function readTextFileLines(filename: string): Promise<string[]>;
+//# sourceMappingURL=readTextFile.d.ts.map