@mark1russell7/ste-lint 0.0.0-stage → 0.1.2

package/src/files.test.ts

@@ -0,0 +1,107 @@
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, relative, sep } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { findFiles, globBase, globToRegExp, isGlob } from "./files.js";
+
+describe("globToRegExp", () => {
+    const matches = (glob : string, path : string) : boolean => globToRegExp(glob).test(path);
+
+    it("matches *, ** and ?", () => {
+        expect(matches("*.md", "README.md")).toBe(true);
+        expect(matches("*.md", "docs/a.md")).toBe(false);
+        expect(matches("docs/**/*.md", "docs/a.md")).toBe(true);
+        expect(matches("docs/**/*.md", "docs/x/y/a.md")).toBe(true);
+        expect(matches("**/node_modules/**", "node_modules/a/b.js")).toBe(true);
+        expect(matches("**/node_modules/**", "packages/x/node_modules/a.js")).toBe(true);
+        expect(matches("**/dist/**", "packages/x/dist/")).toBe(true);
+        expect(matches("a?.ts", "ab.ts")).toBe(true);
+        expect(matches("a?.ts", "a/.ts")).toBe(false);
+        expect(matches("src/**", "src/a/b.ts")).toBe(true);
+        expect(matches("a**b", "axxb")).toBe(true);
+    });
+
+    it("matches alternatives and character sets", () => {
+        expect(matches("packages/*/src/**/*.{ts,tsx}", "packages/lag/src/a/b.tsx")).toBe(true);
+        expect(matches("packages/*/src/**/*.{ts,tsx}", "packages/lag/src/a.js")).toBe(false);
+        expect(matches("*.{md,{mdx,markdown}}", "a.markdown")).toBe(true);
+        expect(matches("file[0-9].md", "file3.md")).toBe(true);
+        expect(matches("file[!0-9].md", "file3.md")).toBe(false);
+        expect(matches("a{b", "a{b")).toBe(true);
+        expect(matches("a[b", "a[b")).toBe(true);
+    });
+
+    it("escapes regular expression characters and accepts ./ and backslashes", () => {
+        expect(matches("a+b.(c)", "a+b.(c)")).toBe(true);
+        expect(matches("./docs\\*.md", "docs/a.md")).toBe(true);
+    });
+
+    it("finds the static base directory of a pattern", () => {
+        expect(globBase("packages/*/src/**/*.ts")).toBe("packages");
+        expect(globBase("docs/**/*.md")).toBe("docs");
+        expect(globBase("*.md")).toBe("");
+        expect(isGlob("README.md")).toBe(false);
+        expect(isGlob("*.md")).toBe(true);
+    });
+});
+
+describe("findFiles", () => {
+    let root : string;
+
+    beforeEach(() => {
+        root = mkdtempSync(join(tmpdir(), "ste-files-"));
+        const files = [
+            "README.md",
+            "docs/guide.md",
+            "docs/deep/page.mdx",
+            "packages/a/src/index.ts",
+            "packages/a/src/index.test.ts",
+            "packages/a/dist/index.js",
+            "packages/a/node_modules/x/index.ts",
+            "packages/b/src/view.tsx",
+        ];
+        for (const file of files) {
+            mkdirSync(join(root, file, ".."), { recursive : true });
+            writeFileSync(join(root, file), "x");
+        }
+    });
+
+    afterEach(() => {
+        rmSync(root, { recursive : true, force : true });
+    });
+
+    const find = (include : string[], ignore : string[] = ["**/*.test.ts", "**/dist/**"]) : string[] =>
+        findFiles({ root, include, ignore }).map((file) => relative(root, file).split(sep).join("/"));
+
+    it("finds the files that match, without ignored files, node_modules or duplicates, in order", () => {
+        expect(find(["README.md", "docs/**/*.{md,mdx}", "packages/*/src/**/*.{ts,tsx}", "docs/guide.md"])).toEqual([
+            "README.md",
+            "docs/deep/page.mdx",
+            "docs/guide.md",
+            "packages/a/src/index.ts",
+            "packages/b/src/view.tsx",
+        ]);
+    });
+
+    it("lints a named file even when an ignore pattern matches it, and skips a missing file or directory", () => {
+        expect(find(["packages/a/src/index.test.ts", "missing.md", "docs", "nothing/**/*.md"])).toEqual(["packages/a/src/index.test.ts"]);
+    });
+
+    it("accepts absolute glob patterns, also for the ignore patterns", () => {
+        const absolute = root.split(sep).join("/");
+        expect(find([`${absolute}/docs/**/*.{md,mdx}`])).toEqual(["docs/deep/page.mdx", "docs/guide.md"]);
+        expect(find([`${absolute}/packages/*/src/**/*.ts`], [`${absolute}/packages/**/*.test.ts`])).toEqual(["packages/a/src/index.ts"]);
+        // A pattern outside the root
+        const docs = join(root, "docs");
+        expect(findFiles({ root : docs, include : [`${absolute}/packages/b/**/*.tsx`], ignore : [] }).map((file) => relative(root, file).split(sep).join("/")))
+            .toEqual(["packages/b/src/view.tsx"]);
+    });
+
+    it("applies the ignore patterns relative to ignoreRoot", () => {
+        const packages = join(root, "packages");
+        const found = findFiles({ root : packages, include : ["**/*.ts"], ignore : ["packages/a/**"], ignoreRoot : root });
+        expect(found.map((file) => relative(root, file).split(sep).join("/"))).toEqual([]);
+        const all = findFiles({ root : packages, include : ["**/*.{ts,tsx}"], ignore : ["**/node_modules/**", "packages/a/**"], ignoreRoot : root });
+        expect(all.map((file) => relative(root, file).split(sep).join("/"))).toEqual(["packages/b/src/view.tsx"]);
+    });
+});

package/src/files.ts

@@ -0,0 +1,191 @@
+/**
+ * File discovery: glob patterns to files.
+ *
+ * The glob syntax:
+ *
+ * - `*` matches characters in one path segment
+ * - `**` matches zero or more segments
+ * - `?` matches one character
+ * - `{a,b}` matches one of the alternatives
+ * - `[abc]` or `[!abc]` matches one character of a set.
+ *
+ * Patterns use `/` on all platforms. An absolute pattern is also correct.
+ */
+
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+
+const GLOB_CHARS = /[*?[\]{}]/;
+
+/** True when `pattern` has glob characters. */
+export function isGlob(pattern : string) : boolean {
+    return GLOB_CHARS.test(pattern);
+}
+
+function escape(char : string) : string {
+    return /[.+^$()|\\[\]{}]/.test(char) ? `\\${char}` : char;
+}
+
+/** The index of the `}` that closes the `{` at `open`, or -1. */
+function closingBrace(glob : string, open : number) : number {
+    let depth = 0;
+    for (let i = open; i < glob.length; i++) {
+        if (glob[i] === "{") depth++;
+        if (glob[i] === "}" && --depth === 0) return i;
+    }
+    return -1;
+}
+
+/** The alternatives of a `{...}` group. A comma in a nested group does not divide alternatives. */
+function splitAlternatives(body : string) : string[] {
+    const parts : string[] = [];
+    let depth = 0;
+    let start = 0;
+    for (let i = 0; i < body.length; i++) {
+        if (body[i] === "{") depth++;
+        if (body[i] === "}") depth--;
+        if (body[i] === "," && depth === 0) {
+            parts.push(body.slice(start, i));
+            start = i + 1;
+        }
+    }
+    parts.push(body.slice(start));
+    return parts;
+}
+
+function globSource(glob : string) : string {
+    let source = "";
+    let i = 0;
+    while (i < glob.length) {
+        const char = glob[i]!;
+        if (char === "*") {
+            if (glob[i + 1] === "*") {
+                const atStart = i === 0 || glob[i - 1] === "/";
+                const atEnd = i + 2 === glob.length || glob[i + 2] === "/";
+                if (atStart && atEnd) {
+                    if (glob[i + 2] === "/") {
+                        source += "(?:.*/)?";
+                        i += 3;
+                    } else {
+                        source += ".*";
+                        i += 2;
+                    }
+                    continue;
+                }
+            }
+            source += "[^/]*";
+            i++;
+            while (glob[i] === "*") i++;
+            continue;
+        }
+        if (char === "?") {
+            source += "[^/]";
+            i++;
+            continue;
+        }
+        if (char === "[") {
+            const close = glob.indexOf("]", i + 1);
+            if (close !== -1) {
+                let set = glob.slice(i + 1, close);
+                if (set.startsWith("!")) set = "^" + set.slice(1);
+                source += `[${set.replace(/\\/g, "\\\\")}]`;
+                i = close + 1;
+                continue;
+            }
+        }
+        if (char === "{") {
+            const close = closingBrace(glob, i);
+            if (close !== -1) {
+                source += `(?:${splitAlternatives(glob.slice(i + 1, close)).map(globSource).join("|")})`;
+                i = close + 1;
+                continue;
+            }
+        }
+        source += escape(char);
+        i++;
+    }
+    return source;
+}
+
+/** This function converts a glob pattern to a regular expression that matches a whole relative path. */
+export function globToRegExp(glob : string) : RegExp {
+    const normalized = glob.replace(/\\/g, "/").replace(/^\.\//, "");
+    return new RegExp(`^${globSource(normalized)}$`);
+}
+
+/** The part of a pattern before the first segment with glob characters. The walk starts in that directory. */
+export function globBase(glob : string) : string {
+    const segments = glob.replace(/\\/g, "/").split("/");
+    const base : string[] = [];
+    for (const segment of segments.slice(0, -1)) {
+        if (isGlob(segment)) break;
+        base.push(segment);
+    }
+    return base.join("/");
+}
+
+function toPosix(path : string) : string {
+    return path.split(sep).join("/");
+}
+
+/** The pattern with `/`, relative to `root`. An absolute pattern becomes relative to `root`, so that it can match a relative path. */
+function relativePattern(root : string, pattern : string) : string {
+    const normalized = pattern.replace(/\\/g, "/").replace(/^\.\//, "");
+    return isAbsolute(normalized) ? toPosix(relative(root, normalized)) : normalized;
+}
+
+/** Directories that the walk does not enter. */
+const SKIPPED_DIRECTORIES = new Set(["node_modules", ".git"]);
+
+export type FindOptions = {
+    /** The directory that the include patterns are relative to. */
+    readonly root : string;
+    readonly include : readonly string[];
+    readonly ignore : readonly string[];
+    /** The directory that the ignore patterns are relative to. The default is `root`. */
+    readonly ignoreRoot? : string;
+};
+
+/**
+ * This function finds the files that match the include patterns and no
+ * ignore pattern. A pattern without glob characters is a file name, and
+ * the ignore patterns do not apply to it. The result has absolute paths,
+ * in sorted order, without duplicates.
+ */
+export function findFiles(options : FindOptions) : string[] {
+    const root = resolve(options.root);
+    const ignoreRoot = resolve(options.ignoreRoot ?? root);
+    const ignore = options.ignore.map((pattern) => globToRegExp(relativePattern(ignoreRoot, pattern)));
+    const isIgnored = (path : string, suffix = "") : boolean => {
+        const relativePath = toPosix(relative(ignoreRoot, path)) + suffix;
+        return ignore.some((pattern) => pattern.test(relativePath));
+    };
+    const found = new Set<string>();
+
+    for (const pattern of options.include) {
+        const normalized = relativePattern(root, pattern);
+        if (!isGlob(normalized)) {
+            const path = resolve(root, normalized);
+            if (existsSync(path) && statSync(path).isFile()) found.add(path);
+            continue;
+        }
+        const matcher = globToRegExp(normalized);
+        const base = resolve(root, globBase(normalized));
+        if (!existsSync(base)) continue;
+        const walk = (directory : string) : void => {
+            for (const entry of readdirSync(directory, { withFileTypes : true })) {
+                const path = join(directory, entry.name);
+                const relativePath = toPosix(relative(root, path));
+                if (entry.isDirectory()) {
+                    // A pattern such as "**/dist/**" matches the files in a directory, so test the directory with a "/" after it.
+                    if (SKIPPED_DIRECTORIES.has(entry.name) || isIgnored(path) || isIgnored(path, "/")) continue;
+                    walk(path);
+                } else if (entry.isFile() && matcher.test(relativePath) && !isIgnored(path)) {
+                    found.add(path);
+                }
+            }
+        };
+        walk(base);
+    }
+    return [...found].sort();
+}

package/src/format.ts

@@ -0,0 +1,78 @@
+import { RULE_IDS, type Finding, type RuleId } from "./types.js";
+
+export type RuleCount = {
+    readonly errors : number;
+    readonly warnings : number;
+};
+
+export type Summary = {
+    /** The number of files that the linter examined. */
+    readonly files : number;
+    readonly filesWithFindings : number;
+    readonly errors : number;
+    readonly warnings : number;
+    /** The counts of each rule that has findings, in the order of the rule list. */
+    readonly byRule : Readonly<Partial<Record<RuleId, RuleCount>>>;
+};
+
+/** This function counts the findings, by severity and by rule. */
+export function summarize(findings : readonly Finding[], files : number) : Summary {
+    const byRule : Partial<Record<RuleId, { errors : number; warnings : number }>> = {};
+    for (const id of RULE_IDS) {
+        const own = findings.filter((finding) => finding.ruleId === id);
+        if (own.length === 0) continue;
+        byRule[id] = {
+            errors : own.filter((finding) => finding.severity === "error").length,
+            warnings : own.filter((finding) => finding.severity === "warning").length,
+        };
+    }
+    return {
+        files,
+        filesWithFindings : new Set(findings.map((finding) => finding.file)).size,
+        errors : findings.filter((finding) => finding.severity === "error").length,
+        warnings : findings.filter((finding) => finding.severity === "warning").length,
+        byRule,
+    };
+}
+
+const plural = (count : number, word : string) : string => `${count} ${word}${count === 1 ? "" : "s"}`;
+
+/** The findings as text: one group for each file, then a summary and the counts by rule. */
+export function formatText(findings : readonly Finding[], summary : Summary) : string {
+    const lines : string[] = [];
+    const byFile = new Map<string, Finding[]>();
+    for (const finding of findings) {
+        const list = byFile.get(finding.file) ?? [];
+        list.push(finding);
+        byFile.set(finding.file, list);
+    }
+    for (const [file, list] of byFile) {
+        lines.push(file);
+        const positions = list.map((finding) => `${finding.line}:${finding.column}`);
+        const width = Math.max(...positions.map((position) => position.length));
+        list.forEach((finding, index) => {
+            lines.push(`  ${positions[index]!.padEnd(width)}  ${finding.severity.padEnd(7)}  ${finding.message}  ${finding.ruleId}`);
+        });
+        lines.push("");
+    }
+    const total = summary.errors + summary.warnings;
+    if (total === 0) {
+        lines.push(`No problems found in ${plural(summary.files, "file")}.`);
+        return lines.join("\n") + "\n";
+    }
+    lines.push(`${plural(total, "problem")} (${plural(summary.errors, "error")}, ${plural(summary.warnings, "warning")}) in ${plural(summary.filesWithFindings, "file")} of ${summary.files}.`);
+    lines.push("");
+    lines.push("Findings by rule:");
+    const entries = Object.entries(summary.byRule) as Array<[RuleId, RuleCount]>;
+    const width = Math.max(...entries.map(([id]) => id.length));
+    for (const [id, count] of entries) {
+        const parts = [count.errors > 0 ? plural(count.errors, "error") : null, count.warnings > 0 ? plural(count.warnings, "warning") : null].filter((part) => part !== null);
+        lines.push(`  ${id.padEnd(width)}  ${String(count.errors + count.warnings).padStart(5)}  (${parts.join(", ")})`);
+    }
+    return lines.join("\n") + "\n";
+}
+
+/** The findings and the summary as JSON. */
+export function formatJson(findings : readonly Finding[], summary : Summary) : string {
+    return JSON.stringify({ tool : "ste-lint", summary, findings }, null, 2) + "\n";
+}

package/src/index.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import * as api from "./index.js";
+
+describe("public API", () => {
+    it("exports the linter, the configuration, the dictionary, the CLI and the rules", () => {
+        for (const name of [
+            "Linter", "lintText", "fileKind", "loadConfig", "resolveConfig", "validateConfig", "findConfigFile", "mergeWordList",
+            "ConfigError", "Dictionary", "DictionaryError", "loadDictionary", "parseDictionary", "findFiles", "globToRegExp",
+            "formatJson", "formatText", "summarize", "buildLexicon", "getRule", "RULES", "parseArgs", "runCli", "isRuleId",
+        ]) {
+            expect(api).toHaveProperty(name);
+        }
+        expect(api.CONFIG_FILE_NAME).toBe("ste.config.json");
+        expect(api.DICTIONARY_ENV).toBe("STE_DICTIONARY");
+        expect(api.DICTIONARY_FORMAT).toBe("ste-lint-dictionary");
+        expect(api.RULE_IDS).toHaveLength(14);
+        expect(api.DEFAULT_LIMITS.instructionWords).toBe(20);
+        expect(api.DEFAULT_INCLUDE).toContain("README.md");
+        expect(api.DEFAULT_IGNORE).toContain("**/node_modules/**");
+        expect(api.DEFAULT_WORD_LIST.length).toBeGreaterThan(40);
+        expect(api.USAGE).toContain("ste-lint");
+    });
+
+    it("lints text with the defaults", () => {
+        const config = api.resolveConfig({}, { root : "/" });
+        expect(api.lintText("You should stop.", "a.md", config).map((finding) => finding.ruleId)).toEqual(["modal-verb"]);
+    });
+});

package/src/index.ts

@@ -0,0 +1,38 @@
+/**
+ * `@mark1russell7/ste-lint`: a linter for prose in Markdown, MDX and TypeScript doc
+ * comments. It is an automated approximation of the writing rules of
+ * ASD-STE100 Simplified Technical English. It does not certify that a text
+ * obeys the standard.
+ */
+
+export {
+    baseConfig,
+    CONFIG_FILE_NAME,
+    ConfigError,
+    DEFAULT_IGNORE,
+    DEFAULT_INCLUDE,
+    DEFAULT_LIMITS,
+    DICTIONARY_ENV,
+    findConfigFile,
+    loadConfig,
+    mergeWordList,
+    resolveConfig,
+    validateConfig,
+    type BaseConfigOptions,
+    type Limits,
+    type LimitsConfig,
+    type ResolvedConfig,
+    type SteConfigFile,
+    type WordListEntryConfig,
+} from "./config.js";
+export { Dictionary, DICTIONARY_FORMAT, DictionaryError, loadDictionary, parseDictionary, type DictionaryEntry, type DictionaryLookup } from "./dictionary.js";
+export { findFiles, globToRegExp } from "./files.js";
+export { formatJson, formatText, summarize, type RuleCount, type Summary } from "./format.js";
+export { buildLexicon, type Lexicon } from "./lexicon.js";
+export { ProseLinter, type CodeExtractor, type ProseLinterOptions } from "./engine.js";
+export { fileKind, Linter, lintText, type FileKind } from "./lint.js";
+export { extractCode } from "./source/code.js";
+export { getRule, RULES, type Rule } from "./rules/index.js";
+export { parseArgs, runCli, USAGE, type CliIo, type CliOptions } from "./run-cli.js";
+export { isRuleId, RULE_IDS, type Finding, type RuleId, type RuleSetting, type Severity } from "./types.js";
+export { DEFAULT_WORD_LIST, type WordListEntry } from "./word-list.js";

package/src/lexicon.ts

@@ -0,0 +1,92 @@
+/**
+ * The word sets that the rules use: the project glossary from the
+ * configuration, the built-in word lists, and the local dictionary when
+ * there is one.
+ */
+
+import type { ResolvedConfig } from "./config.js";
+import type { Dictionary } from "./dictionary.js";
+import { hasLemmaIn } from "./text/grammar.js";
+import { PhraseMatcher, splitPhrase } from "./text/phrases.js";
+import { COMMON_NOUNS, COMMON_VERBS } from "./text/words.js";
+
+export type Lexicon = {
+    /** Single-word technical nouns, in lower case. */
+    readonly technicalNouns : ReadonlySet<string>;
+    /** Single-word technical verbs, in lower case. */
+    readonly technicalVerbs : ReadonlySet<string>;
+    /** Single-word proper nouns, in lower case. */
+    readonly properNouns : ReadonlySet<string>;
+    readonly allowedWords : ReadonlySet<string>;
+    /** All glossary terms, single words and phrases, for the unknown-word rule. */
+    readonly glossaryPhrases : PhraseMatcher;
+    /** Multi-word technical nouns. The noun-cluster rule counts each one as one noun. */
+    readonly technicalNounPhrases : PhraseMatcher;
+    /** Multi-word proper nouns. Each one counts as one word (STE Rule 8.6). */
+    readonly properNounPhrases : PhraseMatcher;
+    /** Nouns for the noun-cluster rule. */
+    readonly nouns : ReadonlySet<string>;
+    /** Base forms of verbs for the imperative and missing-subject heuristics. */
+    readonly verbs : ReadonlySet<string>;
+    readonly dictionary : Dictionary | null;
+};
+
+function words(terms : readonly string[]) : { single : Set<string>; phrases : string[] } {
+    const single = new Set<string>();
+    const phrases : string[] = [];
+    for (const term of terms) {
+        const parts = splitPhrase(term);
+        if (parts.length === 1) single.add(parts[0]!);
+        else if (parts.length > 1) phrases.push(parts.join(" "));
+    }
+    return { single, phrases };
+}
+
+/** This function makes the lexicon for a configuration and an optional dictionary. */
+export function buildLexicon(config : ResolvedConfig, dictionary : Dictionary | null = null) : Lexicon {
+    const nouns = words(config.technicalNouns);
+    const verbs = words(config.technicalVerbs);
+    const proper = words(config.properNouns);
+    const allowed = words(config.allowedWords);
+
+    const glossaryPhrases = new PhraseMatcher();
+    const technicalNounPhrases = new PhraseMatcher();
+    // A name counts as one word only with its own capital letters: "Event Timing", but not "event timing".
+    const properNounPhrases = new PhraseMatcher({ caseSensitive : true });
+    for (const phrase of nouns.phrases) {
+        glossaryPhrases.add(phrase, true);
+        technicalNounPhrases.add(phrase, true);
+    }
+    for (const phrase of verbs.phrases) glossaryPhrases.add(phrase, true);
+    for (const phrase of proper.phrases) glossaryPhrases.add(phrase, true);
+    for (const name of config.properNouns) {
+        if (name.trim().split(/\s+/).length > 1) properNounPhrases.add(name.trim(), true);
+    }
+
+    const verbSet = new Set<string>([...COMMON_VERBS, ...verbs.single, ...(dictionary?.verbs ?? [])]);
+    // A word that is often a verb counts as a noun only when the glossary or the dictionary says that it is a noun.
+    const nounSet = new Set<string>();
+    for (const noun of COMMON_NOUNS) if (!verbSet.has(noun)) nounSet.add(noun);
+    for (const noun of [...nouns.single, ...proper.single, ...(dictionary?.nouns ?? [])]) nounSet.add(noun);
+
+    return {
+        technicalNouns : nouns.single,
+        technicalVerbs : verbs.single,
+        properNouns : proper.single,
+        allowedWords : allowed.single,
+        glossaryPhrases,
+        technicalNounPhrases,
+        properNounPhrases,
+        nouns : nounSet,
+        verbs : verbSet,
+        dictionary,
+    };
+}
+
+/** True when the glossary permits the word: a technical noun or verb in any form, a proper noun, or a word of `allowedWords`. */
+export function isGlossaryWord(word : string, lexicon : Lexicon) : boolean {
+    return lexicon.properNouns.has(word)
+        || lexicon.allowedWords.has(word)
+        || hasLemmaIn(word, lexicon.technicalNouns)
+        || hasLemmaIn(word, lexicon.technicalVerbs);
+}