cspell-tools 5.1.0 → 8.17.5

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2017 Jason Dent
+Copyright (c) 2019 cspell
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -1,9 +1,11 @@
-# cSpell-Tools
+# CSpell-Tools
 
-Tools used to assist cSpell Development.
+Tools used to assist CSpell Development.
 
 The Primary use of this tool is to build dictionaries used by cspell. This tool will convert a word list or a hunspell file into a file usable by cspell.
 
+This package is an alias for `@cspell/cspell-tools`.
+
 ## Install
 
 ```sh
@@ -35,3 +37,23 @@ This will read and expand the `english.dic` file based upon the rules in `englis
 For large files, this process can take a long time and us a lot of memory.
 
 The tool `cspell-trie` can be used to read the contents of a `.trie` or `.trie.gz` file.
+
+## CSpell for Enterprise
+
+<!--- @@inject: ../../static/tidelift.md --->
+
+Available as part of the Tidelift Subscription.
+
+The maintainers of cspell and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source packages you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact packages you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-cspell?utm_source=npm-cspell&utm_medium=referral&utm_campaign=enterprise&utm_term=repo)
+
+<!--- @@inject-end: ../../static/tidelift.md --->
+
+<!--- @@inject: ../../static/footer.md --->
+
+<br/>
+
+---
+
+<p align="center">Brought to you by<a href="https://streetsidesoftware.com" title="Street Side Software"><img width="16" alt="Street Side Software Logo" src="https://i.imgur.com/CyduuVY.png" /> Street Side Software</a></p>
+
+<!--- @@inject-end: ../../static/footer.md --->

package/bin.mjs

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+
+import '@cspell/cspell-tools/bin';

package/package.json

@@ -1,24 +1,24 @@
 {
   "name": "cspell-tools",
-  "version": "5.1.0",
-  "description": "Tools to assist with the development of cSpell",
-  "typings": "dist/index.d.ts",
+  "version": "8.17.5",
+  "description": "Tools to assist with the development of cSpell. Alias of @cspell/cspell-tools",
+  "publishConfig": {
+    "access": "public",
+    "provenance": true
+  },
+  "type": "module",
+  "sideEffects": false,
   "bin": {
-    "cspell-tools": "bin.js"
+    "cspell-tools": "bin.mjs"
   },
   "scripts": {
-    "build": "tsc -p .",
-    "watch": "tsc -p . -w",
-    "clean-build": "npm run clean && npm run build",
-    "clean": "rimraf dist temp coverage",
-    "coverage": "jest --coverage",
-    "test-watch": "jest --watch",
-    "test": "jest",
-    "update-snapshot": "jest --updateSnapshot"
+    "test": "pnpm test-shasum",
+    "test-shasum": "node ./bin.mjs shasum package.json"
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/streetsidesoftware/cspell.git"
+    "url": "https://github.com/streetsidesoftware/cspell.git",
+    "directory": "packages/cspell-tools-alias"
   },
   "keywords": [
     "cSpell",
@@ -27,42 +27,25 @@
     "Compiler"
   ],
   "files": [
-    "bin.js",
-    "dist/**",
-    "!**/*.map",
-    "!**/*.test.*"
+    "bin.mjs",
+    "!**/*.tsbuildInfo",
+    "!**/__mocks__",
+    "!**/*.test.*",
+    "!**/*.spec.*",
+    "!**/*.map"
   ],
   "author": "Jason Dent",
   "license": "MIT",
   "bugs": {
     "url": "https://github.com/streetsidesoftware/cspell/labels/cspell-tools"
   },
-  "homepage": "https://github.com/streetsidesoftware/cspell#readme",
+  "homepage": "https://github.com/streetsidesoftware/cspell/tree/main/packages/cspell-tools-alias#readme",
   "dependencies": {
-    "commander": "^6.2.1",
-    "cspell-io": "^5.1.0",
-    "cspell-trie-lib": "^5.1.0",
-    "cspell-util-bundle": "^5.1.0",
-    "fs-extra": "^9.0.1",
-    "gensequence": "^3.1.1",
-    "glob": "^7.1.6",
-    "hunspell-reader": "^3.2.0"
+    "@cspell/cspell-tools": "8.17.5"
   },
   "engines": {
-    "node": ">=12.0.0"
-  },
-  "devDependencies": {
-    "@types/fs-extra": "^9.0.6",
-    "@types/glob": "^7.1.3",
-    "@types/jest": "^26.0.19",
-    "@types/node": "^14.14.16",
-    "@types/rimraf": "^3.0.0",
-    "@types/shelljs": "^0.8.8",
-    "jest": "^26.6.3",
-    "lorem-ipsum": "^2.0.3",
-    "rimraf": "^3.0.2",
-    "shelljs": "^0.8.4"
+    "node": ">=18"
   },
-  "main": "bin.js",
-  "gitHead": "4160b964818f29a97a29b36e583a6f4d7c8de300"
+  "module": "bin.mjs",
+  "gitHead": "2d80ee2aa347d07c52aa9c85aa248538834d7f3c"
 }

package/CHANGELOG.md

@@ -1,118 +0,0 @@
-# Change Log
-
-All notable changes to this project will be documented in this file.
-See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
-
-# [5.1.0](https://github.com/streetsidesoftware/cspell/compare/v5.0.8...v5.1.0) (2020-12-27)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.8](https://github.com/streetsidesoftware/cspell/compare/v5.0.7...v5.0.8) (2020-12-17)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.5](https://github.com/streetsidesoftware/cspell/compare/v5.0.4...v5.0.5) (2020-12-15)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.4](https://github.com/streetsidesoftware/cspell/compare/v5.0.3...v5.0.4) (2020-12-15)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.3](https://github.com/streetsidesoftware/cspell/compare/v5.0.2...v5.0.3) (2020-12-04)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.2](https://github.com/streetsidesoftware/cspell/compare/v5.0.2-alpha.1...v5.0.2) (2020-11-26)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.1](https://github.com/streetsidesoftware/cspell/compare/v5.0.1-alpha.15...v5.0.1) (2020-11-20)
-
-**Note:** Version bump only for package cspell-tools
-
-
-
-
-
-## [5.0.1-alpha.15](https://github.com/streetsidesoftware/cspell/compare/v5.0.1-alpha.14...v5.0.1-alpha.15) (2020-11-18)
-
-
-### Bug Fixes
-
-* force new version ([3ab08ab](https://github.com/streetsidesoftware/cspell/commit/3ab08ab5ae1939d934b2f0fb23d33defc60c1a7f))
-
-
-
-
-
-## 5.0.1-alpha.14 (2020-11-17)
-
-**Note:** Version bump only for package cspell-tools
-
-## [5.0.1-alpha.0](https://github.com/streetsidesoftware/cspell/compare/cspell-tools@4.1.12...cspell-tools@5.0.1-alpha.0) (2020-02-20)
-
-**Note:** Version bump only for package cspell-tools
-
-# Release Notes
-
-## 2.0
-
-- Update to rxjs 6
-
-## 1.4.1
-
-- Update packages
-
-## 1.4.0
-
-- Update packages
-- Remove dependency on fs-promise
-
-## 1.3.2
-
-- Fix issue with writing files when the destination path does not exist.
-
-## 1.3.1
-
-- Add command line tool for building trie files.
-
-## 1.2.1
-
-- Hyphenated words are now treated as two words.
-
-## 1.1.0
-
-- use cspell-lib for file i/o.
-
-## 1.0.10
-
-- allow globs on the command line so `cspell-tools compile *.txt` works on windows.
-
-## 1.0.7
-
-- update packages

package/bin.js

@@ -1,13 +0,0 @@
-#!/usr/bin/env node
-
-'use strict';
-
-const app = require('./dist/app');
-const program = require('commander');
-
-app.run(program, process.argv).catch((e) => {
-    if (!(e instanceof program.CommanderError)) {
-        console.log(e);
-    }
-    process.exitCode = 1;
-});

package/dist/app.d.ts

@@ -1,2 +0,0 @@
-import * as program from 'commander';
-export declare function run(program: program.Command, argv: string[]): Promise<void>;

package/dist/app.js

@@ -1,174 +0,0 @@
-"use strict";
-// For large dictionaries, it is necessary to increase the memory limit.
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.run = void 0;
-const compiler_1 = require("./compiler");
-const compiler = __importStar(require("./compiler"));
-const path = __importStar(require("path"));
-const glob_1 = __importDefault(require("glob"));
-const gensequence_1 = require("gensequence");
-const iterateWordsFromFile_1 = require("./compiler/iterateWordsFromFile");
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const npmPackage = require(path.join(__dirname, '..', 'package.json'));
-function globP(pattern) {
-    return new Promise((resolve, reject) => {
-        glob_1.default(pattern, (err, result) => {
-            err ? reject(err) : resolve(result);
-        });
-    });
-}
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const log = (message, ...optionalParams) => {
-    console.log(`${new Date().toISOString()} ${message}`, ...optionalParams);
-};
-compiler.setLogger(log);
-function collect(value, previous) {
-    return previous.concat([value]);
-}
-function run(program, argv) {
-    program.exitOverride();
-    return new Promise((resolve, reject) => {
-        program.version(npmPackage.version);
-        program
-            .command('compile <src...>')
-            .description('compile words lists into simple dictionary files.')
-            .option('-o, --output <path>', 'Specify the output directory, otherwise files are written back to the same location.')
-            .option('-n, --no-compress', 'By default the files are Gzipped, this will turn that off.')
-            .option('-m, --max_depth <limit>', 'Maximum depth to apply suffix rules.')
-            .option('-M, --merge <target>', 'Merge all files into a single target file (extensions are applied)')
-            .option('-s, --no-split', 'Treat each line as a dictionary entry, do not split')
-            .option('-x, --experimental <flag>', 'Experimental flags, used for testing new concepts. Flags: compound', collect, [])
-            .option('--no-sort', 'Do not sort the result')
-            .action((src, options) => {
-            const experimental = new Set(options.experimental);
-            const skipNormalization = experimental.has('compound');
-            const result = processAction(src, '.txt', options, async (src, dst) => {
-                return compiler_1.compileWordList(src, dst, {
-                    splitWords: options.split,
-                    sort: options.sort,
-                    skipNormalization,
-                }).then(() => src);
-            });
-            resolve(result);
-        });
-        program
-            .command('compile-trie <src...>')
-            .description('Compile words lists or Hunspell dictionary into trie files used by cspell.')
-            .option('-o, --output <path>', 'Specify the output directory, otherwise files are written back to the same location.')
-            .option('-m, --max_depth <limit>', 'Maximum depth to apply suffix rules.')
-            .option('-M, --merge <target>', 'Merge all files into a single target file (extensions are applied)')
-            .option('-n, --no-compress', 'By default the files are Gzipped, this will turn that off.')
-            .option('-x, --experimental <flag>', 'Experimental flags, used for testing new concepts. Flags: compound', collect, [])
-            .option('--trie3', '[Beta] Use file format trie3')
-            .action((src, options) => {
-            const experimental = new Set(options.experimental);
-            const skipNormalization = experimental.has('compound');
-            const compileOptions = { ...options, skipNormalization };
-            const result = processAction(src, '.trie', options, async (words, dst) => {
-                return compiler_1.compileTrie(words, dst, compileOptions);
-            });
-            resolve(result);
-        });
-        try {
-            program.parse(argv);
-            if (!argv.slice(2).length) {
-                program.help();
-            }
-        }
-        catch (e) {
-            reject(e);
-        }
-        resolve();
-    });
-}
-exports.run = run;
-function parseNumber(s) {
-    const n = parseInt(s !== null && s !== void 0 ? s : '');
-    return isNaN(n) ? undefined : n;
-}
-async function processAction(src, fileExt, options, action) {
-    console.log('Compile:\n output: %s\n compress: %s\n files:\n  %s \n\n', options.output || 'default', options.compress ? 'true' : 'false', src.join('\n  '));
-    const ext = fileExt + (options.compress ? '.gz' : '');
-    const maxDepth = parseNumber(options.max_depth);
-    const experimental = new Set(options.experimental);
-    const useAnnotation = experimental.has('compound');
-    const readerOptions = { maxDepth, useAnnotation };
-    const globResults = await Promise.all(src.map((s) => globP(s)));
-    const filesToProcess = gensequence_1.genSequence(globResults)
-        .concatMap((files) => files)
-        .map(async (filename) => {
-        log(`Reading ${path.basename(filename)}`);
-        const words = await iterateWordsFromFile_1.streamWordsFromFile(filename, readerOptions);
-        log(`Done reading ${path.basename(filename)}`);
-        const f = {
-            src: filename,
-            words,
-        };
-        return f;
-    });
-    const r = options.merge
-        ? processFiles(action, filesToProcess, toMergeTargetFile(options.merge, options.output, ext))
-        : processFilesIndividually(action, filesToProcess, (s) => toTargetFile(s, options.output, ext));
-    await r;
-    log(`Complete.`);
-}
-function toFilename(name, ext) {
-    return path.basename(name).replace(/((\.txt|\.dic|\.aff|\.trie)(\.gz)?)?$/, '') + ext;
-}
-function toTargetFile(filename, destination, ext) {
-    const outFileName = toFilename(filename, ext);
-    const dir = destination !== null && destination !== void 0 ? destination : path.dirname(filename);
-    return path.join(dir, outFileName);
-}
-function toMergeTargetFile(filename, destination, ext) {
-    const outFileName = path.join(path.dirname(filename), toFilename(filename, ext));
-    return path.resolve(destination !== null && destination !== void 0 ? destination : '.', outFileName);
-}
-async function processFilesIndividually(action, filesToProcess, srcToTarget) {
-    const toProcess = filesToProcess.map(async (pFtp) => {
-        const { src, words } = await pFtp;
-        const dst = srcToTarget(src);
-        log('Process "%s" to "%s"', src, dst);
-        await action(words, dst);
-        log('Done "%s" to "%s"', src, dst);
-    });
-    for (const p of toProcess) {
-        await p;
-    }
-}
-async function processFiles(action, filesToProcess, mergeTarget) {
-    const toProcess = await Promise.all([...filesToProcess]);
-    const dst = mergeTarget;
-    const words = gensequence_1.genSequence(toProcess)
-        .map((ftp) => {
-        const { src } = ftp;
-        log('Process "%s" to "%s"', src, dst);
-        return ftp;
-    })
-        .concatMap((ftp) => ftp.words);
-    await action(words, dst);
-    log('Done "%s"', dst);
-}
-//# sourceMappingURL=app.js.map

package/dist/compiler/Reader.d.ts

@@ -1,18 +0,0 @@
-import { Sequence } from 'gensequence';
-export interface ReaderOptions {
-    useAnnotation?: boolean;
-    maxDepth?: number;
-}
-export declare type AnnotatedWord = string;
-interface BaseReader {
-    size: number;
-    annotatedWords: () => Sequence<AnnotatedWord>;
-    rawWords: () => Sequence<string>;
-}
-export interface Reader extends BaseReader {
-    [Symbol.iterator]: () => Sequence<string>;
-}
-export declare function createReader(filename: string, options: ReaderOptions): Promise<Reader>;
-export declare function createArrayReader(lines: string[]): BaseReader;
-export declare function readHunspellFiles(filename: string, options: ReaderOptions): Promise<BaseReader>;
-export {};

package/dist/compiler/Reader.js

@@ -1,181 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readHunspellFiles = exports.createArrayReader = exports.createReader = void 0;
-const gensequence_1 = require("gensequence");
-const HR = __importStar(require("hunspell-reader"));
-const fs = __importStar(require("fs-extra"));
-const cspell_trie_lib_1 = require("cspell-trie-lib");
-const zlib = __importStar(require("zlib"));
-const regHunspellFile = /\.(dic|aff)$/i;
-// cspell:word dedupe
-const DEDUPE_SIZE = 1000;
-const regExMatchComments = /\s*(#|\/\/).*/;
-// Readers first match wins
-const readers = [
-    { test: /\.trie\b/, method: trieFileReader },
-    { test: regHunspellFile, method: readHunspellFiles },
-];
-function findMatchingReader(filename, options) {
-    for (const reader of readers) {
-        if (reader.test.test(filename)) {
-            return reader.method(filename, options);
-        }
-    }
-    return textFileReader(filename, options);
-}
-async function createReader(filename, options) {
-    const baseReader = await findMatchingReader(filename, options);
-    return Object.assign(baseReader, {
-        [Symbol.iterator]: options.useAnnotation ? baseReader.annotatedWords : baseReader.rawWords,
-    });
-}
-exports.createReader = createReader;
-function createArrayReader(lines) {
-    const rawWords = () => gensequence_1.genSequence(lines);
-    const annotatedWords = () => gensequence_1.genSequence(lines).pipe(_mapText, dedupeAndSort);
-    return {
-        size: lines.length,
-        annotatedWords,
-        rawWords,
-    };
-}
-exports.createArrayReader = createArrayReader;
-async function readHunspellFiles(filename, options) {
-    const dicFile = filename.replace(regHunspellFile, '.dic');
-    const affFile = filename.replace(regHunspellFile, '.aff');
-    const reader = await HR.IterableHunspellReader.createFromFiles(affFile, dicFile);
-    reader.maxDepth = options.maxDepth !== undefined ? options.maxDepth : reader.maxDepth;
-    const normalizeAndDedupe = gensequence_1.operators.pipe(_stripCaseAndAccents, dedupeAndSort);
-    const rawWords = () => reader.seqWords();
-    return {
-        size: reader.dic.length,
-        annotatedWords() {
-            return reader.seqAffWords().pipe(_mapAffWords).pipe(normalizeAndDedupe);
-        },
-        rawWords,
-    };
-}
-exports.readHunspellFiles = readHunspellFiles;
-async function trieFileReader(filename) {
-    const trieRoot = cspell_trie_lib_1.importTrie(await readTextFile(filename));
-    const trie = new cspell_trie_lib_1.Trie(trieRoot);
-    const rawWords = () => trie.words();
-    return {
-        get size() {
-            return trie.size();
-        },
-        annotatedWords: rawWords,
-        rawWords,
-    };
-}
-function readTextFile(filename) {
-    const lines = fs
-        .readFile(filename)
-        .then((buffer) => (/\.gz$/.test(filename) ? zlib.gunzipSync(buffer) : buffer))
-        .then((buffer) => buffer.toString('utf8'))
-        .then((content) => content.split(/\r?\n/g));
-    return lines;
-}
-async function textFileReader(filename, _options) {
-    const lines = await readTextFile(filename);
-    return createArrayReader(lines);
-}
-const _mapText = gensequence_1.operators.pipe(_comments, _compoundBegin, _compoundEnd, _stripCaseAndAccents);
-function* _comments(lines) {
-    for (const line of lines) {
-        const w = line.replace(regExMatchComments, '').trim();
-        if (w)
-            yield w;
-    }
-}
-function* _compoundEnd(lines) {
-    for (const line of lines) {
-        if (line[0] !== cspell_trie_lib_1.OPTIONAL_COMPOUND) {
-            yield line;
-            continue;
-        }
-        const w = line.slice(1);
-        yield w;
-        yield cspell_trie_lib_1.COMPOUND + w;
-    }
-}
-function* _compoundBegin(lines) {
-    for (const line of lines) {
-        if (line[line.length - 1] !== cspell_trie_lib_1.OPTIONAL_COMPOUND) {
-            yield line;
-            continue;
-        }
-        const w = line.slice(0, -1);
-        yield w;
-        yield w + cspell_trie_lib_1.COMPOUND;
-    }
-}
-function* _stripCaseAndAccents(words) {
-    for (const word of words) {
-        // Words are normalized to the compact format: e + ` => è
-        yield word.normalize();
-        // covert to lower case and strip accents.
-        const n = word
-            .toLowerCase()
-            .normalize('NFD')
-            .replace(/[\u0300-\u036f]/g, '');
-        // All words are added for case-insensitive searches.
-        // It is a space / speed trade-off. In this case, speed is more important.
-        yield cspell_trie_lib_1.NORMALIZED + n;
-    }
-}
-function* dedupeAndSort(words) {
-    const buffer = [];
-    function sortDedupeClear() {
-        const s = new Set(buffer.sort());
-        buffer.length = 0;
-        return s;
-    }
-    for (const word of words) {
-        buffer.push(word);
-        if (buffer.length >= DEDUPE_SIZE) {
-            yield* sortDedupeClear();
-        }
-    }
-    yield* sortDedupeClear();
-}
-function* _mapAffWords(affWords) {
-    for (const affWord of affWords) {
-        const { word, flags } = affWord;
-        const compound = flags.isCompoundForbidden ? '' : cspell_trie_lib_1.COMPOUND;
-        const forbid = flags.isForbiddenWord ? cspell_trie_lib_1.FORBID : '';
-        if (!forbid) {
-            if (flags.canBeCompoundBegin)
-                yield word + compound;
-            if (flags.canBeCompoundEnd)
-                yield compound + word;
-            if (flags.canBeCompoundMiddle)
-                yield compound + word + compound;
-            if (!flags.isOnlyAllowedInCompound)
-                yield word;
-        }
-        else {
-            yield forbid + word;
-        }
-    }
-}
-//# sourceMappingURL=Reader.js.map

package/dist/compiler/fileWriter.d.ts

@@ -1,3 +0,0 @@
-import { Sequence } from 'gensequence';
-export { writeToFile, writeToFileIterableP, writeToFileIterable } from 'cspell-io';
-export declare function writeSeqToFile(seq: Sequence<string>, outFile: string): Promise<void>;

package/dist/compiler/fileWriter.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.writeSeqToFile = exports.writeToFileIterable = exports.writeToFileIterableP = exports.writeToFile = void 0;
-const cspell_io_1 = require("cspell-io");
-var cspell_io_2 = require("cspell-io");
-Object.defineProperty(exports, "writeToFile", { enumerable: true, get: function () { return cspell_io_2.writeToFile; } });
-Object.defineProperty(exports, "writeToFileIterableP", { enumerable: true, get: function () { return cspell_io_2.writeToFileIterableP; } });
-Object.defineProperty(exports, "writeToFileIterable", { enumerable: true, get: function () { return cspell_io_2.writeToFileIterable; } });
-function writeSeqToFile(seq, outFile) {
-    return cspell_io_1.writeToFileIterableP(outFile, seq);
-}
-exports.writeSeqToFile = writeSeqToFile;
-//# sourceMappingURL=fileWriter.js.map

package/dist/compiler/index.d.ts

@@ -1 +0,0 @@
-export * from './wordListCompiler';

package/dist/compiler/index.js

@@ -1,14 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-__exportStar(require("./wordListCompiler"), exports);
-//# sourceMappingURL=index.js.map

package/dist/compiler/iterateWordsFromFile.d.ts

@@ -1,3 +0,0 @@
-import { Sequence } from 'gensequence';
-import { ReaderOptions } from './Reader';
-export declare function streamWordsFromFile(filename: string, options: ReaderOptions): Promise<Sequence<string>>;

package/dist/compiler/iterateWordsFromFile.js

@@ -1,10 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.streamWordsFromFile = void 0;
-const Reader_1 = require("./Reader");
-async function streamWordsFromFile(filename, options) {
-    const reader = await Reader_1.createReader(filename, options);
-    return reader[Symbol.iterator]();
-}
-exports.streamWordsFromFile = streamWordsFromFile;
-//# sourceMappingURL=iterateWordsFromFile.js.map

package/dist/compiler/text.d.ts

@@ -1,4 +0,0 @@
-/**
- * Split camelCase words into an array of strings.
- */
-export declare function splitCamelCaseWord(word: string): string[];

package/dist/compiler/text.js

@@ -1,20 +0,0 @@
-"use strict";
-// cSpell:ignore ings ning gimuy
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.splitCamelCaseWord = void 0;
-const cspell_util_bundle_1 = require("cspell-util-bundle");
-const regExUpperSOrIng = cspell_util_bundle_1.xregexp("(\\p{Lu}+'?(?:s|ing|ies|es|ings|ed|ning))(?!\\p{Ll})", 'g');
-const regExSplitWords = cspell_util_bundle_1.xregexp('(\\p{Ll})(\\p{Lu})', 'g');
-const regExSplitWords2 = cspell_util_bundle_1.xregexp('(\\p{Lu})(\\p{Lu}\\p{Ll})', 'g');
-/**
- * Split camelCase words into an array of strings.
- */
-function splitCamelCaseWord(word) {
-    const wPrime = word.replace(regExUpperSOrIng, (s) => s[0] + s.substr(1).toLowerCase());
-    const separator = '_<^*_*^>_';
-    const pass1 = cspell_util_bundle_1.xregexp.replace(wPrime, regExSplitWords, '$1' + separator + '$2');
-    const pass2 = cspell_util_bundle_1.xregexp.replace(pass1, regExSplitWords2, '$1' + separator + '$2');
-    return cspell_util_bundle_1.xregexp.split(pass2, separator);
-}
-exports.splitCamelCaseWord = splitCamelCaseWord;
-//# sourceMappingURL=text.js.map

package/dist/compiler/wordListCompiler.d.ts

@@ -1,23 +0,0 @@
-import { Sequence } from 'gensequence';
-import * as Trie from 'cspell-trie-lib';
-export declare type Logger = (message?: any, ...optionalParams: any[]) => void;
-export declare function setLogger(logger?: Logger): void;
-declare type Normalizer = (lines: Sequence<string>) => Sequence<string>;
-export declare function normalizeWords(lines: Sequence<string>): Sequence<string>;
-export declare function lineToWords(line: string): Sequence<string>;
-export interface CompileOptions {
-    skipNormalization?: boolean;
-}
-export interface CompileWordListOptions extends CompileOptions {
-    splitWords: boolean;
-    sort: boolean;
-}
-export declare function compileWordList(words: Sequence<string>, destFilename: string, options: CompileWordListOptions): Promise<void>;
-export declare function normalizeWordsToTrie(words: Sequence<string>, normalizer?: Normalizer): Trie.TrieRoot;
-export interface CompileTrieOptions extends CompileOptions {
-    base?: number;
-    trie3?: boolean;
-}
-export declare const consolidate: typeof Trie.consolidate;
-export declare function compileTrie(words: Sequence<string>, destFilename: string, options: CompileTrieOptions): Promise<void>;
-export {};

package/dist/compiler/wordListCompiler.js

@@ -1,119 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.compileTrie = exports.consolidate = exports.normalizeWordsToTrie = exports.compileWordList = exports.lineToWords = exports.normalizeWords = exports.setLogger = void 0;
-const cspell_util_bundle_1 = require("cspell-util-bundle");
-const gensequence_1 = require("gensequence");
-const Text = __importStar(require("./text"));
-const path = __importStar(require("path"));
-const fs_extra_1 = require("fs-extra");
-const Trie = __importStar(require("cspell-trie-lib"));
-const fileWriter_1 = require("./fileWriter");
-const util_1 = require("hunspell-reader/dist/util");
-const regNonWordOrSpace = cspell_util_bundle_1.xregexp("[^\\p{L}' ]+", 'gi');
-const regExpSpaceOrDash = /(?:\s+)|(?:-+)/g;
-const regExpRepeatChars = /(.)\1{3,}/i;
-let log = defaultLogger;
-function setLogger(logger) {
-    log = logger !== null && logger !== void 0 ? logger : defaultLogger;
-}
-exports.setLogger = setLogger;
-function defaultLogger(message, ...optionalParams) {
-    console.log(message, ...optionalParams);
-}
-function normalizeWords(lines) {
-    return lines.concatMap((line) => lineToWords(line));
-}
-exports.normalizeWords = normalizeWords;
-function lineToWords(line) {
-    // Remove punctuation and non-letters.
-    const filteredLine = line.replace(regNonWordOrSpace, '|');
-    const wordGroups = filteredLine.split('|');
-    const words = gensequence_1.genSequence(wordGroups)
-        .concatMap((a) => [a, ...a.split(regExpSpaceOrDash)])
-        .concatMap((a) => splitCamelCase(a))
-        .map((a) => a.trim())
-        .filter((a) => !!a)
-        .filter((s) => !regExpRepeatChars.test(s))
-        .map((a) => a.toLowerCase());
-    return words;
-}
-exports.lineToWords = lineToWords;
-function splitCamelCase(word) {
-    const splitWords = Text.splitCamelCaseWord(word);
-    // We only want to preserve this: "New York" and not "Namespace DNSLookup"
-    if (splitWords.length > 1 && regExpSpaceOrDash.test(word)) {
-        return gensequence_1.genSequence(splitWords).concatMap((w) => w.split(regExpSpaceOrDash));
-    }
-    return splitWords;
-}
-async function compileWordList(words, destFilename, options) {
-    const destDir = path.dirname(destFilename);
-    const pDir = fs_extra_1.mkdirp(destDir);
-    const compile = options.skipNormalization
-        ? (a) => a
-        : options.splitWords
-            ? compileWordListWithSplitSeq
-            : compileSimpleWordListSeq;
-    const seq = compile(words)
-        .filter((a) => !!a)
-        .filter(util_1.uniqueFilter(10000));
-    const finalSeq = options.sort ? gensequence_1.genSequence(sort(seq)) : seq;
-    await pDir;
-    return fileWriter_1.writeSeqToFile(finalSeq.map((a) => a + '\n'), destFilename);
-}
-exports.compileWordList = compileWordList;
-function sort(words) {
-    return [...words].sort();
-}
-function compileWordListWithSplitSeq(words) {
-    return words.concatMap((line) => lineToWords(line).toArray());
-}
-function compileSimpleWordListSeq(words) {
-    return words.map((a) => a.toLowerCase());
-}
-function normalizeWordsToTrie(words, normalizer = normalizeWords) {
-    return Trie.buildTrie(normalizer(words)).root;
-}
-exports.normalizeWordsToTrie = normalizeWordsToTrie;
-exports.consolidate = Trie.consolidate;
-async function compileTrie(words, destFilename, options) {
-    var _a;
-    log('Reading Words into Trie');
-    const base = (_a = options.base) !== null && _a !== void 0 ? _a : 32;
-    const version = options.trie3 ? 3 : 1;
-    const destDir = path.dirname(destFilename);
-    const pDir = fs_extra_1.mkdirp(destDir);
-    const normalizer = options.skipNormalization ? (a) => a : normalizeWords;
-    const root = normalizeWordsToTrie(words, normalizer);
-    log('Reduce duplicate word endings');
-    const trie = exports.consolidate(root);
-    log(`Writing to file ${path.basename(destFilename)}`);
-    await pDir;
-    await fileWriter_1.writeSeqToFile(Trie.serializeTrie(trie, {
-        base,
-        comment: 'Built by cspell-tools.',
-        version,
-    }), destFilename);
-    log(`Done writing to file ${path.basename(destFilename)}`);
-}
-exports.compileTrie = compileTrie;
-//# sourceMappingURL=wordListCompiler.js.map