nollm 0.3.0 → 0.5.0

package/README.md

@@ -52,14 +52,16 @@ through a row of `..`.
 Ignore patterns from the config describe the project, so they apply under the
 current directory and leave paths outside it alone.
 
-| Option            | Effect                                                             |
-| ----------------- | ------------------------------------------------------------------ |
-| `--jobs <n>`      | Number of worker threads. Defaults to the CPU count.               |
-| `--config <path>` | Config file to use.                                                |
-| `--diff <ref>`    | Check only the lines this branch adds or changes since `<ref>`.    |
-| `--no-git`        | Do not ask git for the file list. Read `.gitignore` files instead. |
-| `--quiet`         | Print only the summary.                                            |
-| `--list-rules`    | Print every rule and exit.                                         |
+| Option                    | Effect                                                             |
+| ------------------------- | ------------------------------------------------------------------ |
+| `--jobs <n>`              | Number of worker threads. Defaults to the CPU count.               |
+| `--config <path>`         | Config file to use.                                                |
+| `--diff <ref>`            | Check only the lines this branch adds or changes since `<ref>`.    |
+| `--no-git`                | Do not ask git for the file list. Read `.gitignore` files instead. |
+| `--stdin`                 | Read the text from stdin. The path `-` does the same.              |
+| `--stdin-filename <name>` | Check stdin as if it were this file. Defaults to `stdin.md`.       |
+| `--quiet`                 | Print only the summary.                                            |
+| `--list-rules`            | Print every rule and exit.                                         |
 
 The exit code is 1 when there are findings, and 2 on a usage error.
 
@@ -80,6 +82,24 @@ src/index.js
 4 problems in 2 files (5 files checked, 0.07s)
 ```
 
+## Checking text from stdin
+
+Pass `-` to check text that is not in a file, such as a draft or a commit message:
+
+```
+pbpaste | nollm -
+git log -1 --format=%B | nollm -
+```
+
+Stdin is checked as markdown. To check it as another type, name a file.
+The name picks the language and labels the report. No file is read:
+
+```
+git show main:src/index.js | nollm --stdin-filename src/index.js
+```
+
+The config applies as usual. Stdin cannot be combined with paths or `--diff`.
+
 ## Checking only a pull request
 
 A large codebase written before you added `nollm` has findings everywhere.
@@ -136,7 +156,7 @@ Files of other types, binary files, lockfiles, minified files, and files over 2
 | `chat-opener`             | Lines that start with "Great question", "Certainly", "Let me", and more                                            |
 | `chat-closer`             | "Hope this helps", "Let me know if", "Feel free to", and more                                                      |
 | `ai-disclosure`           | "As an AI", "my training data", and more                                                                           |
-| `contrast-cliche`         | "not just X, but Y" and "it's not X, it's Y"                                                                       |
+| `contrast-cliche`         | "not just X, but Y", "it's not X, it's Y", and "X, not Y: the rest"                                                |
 | `rhetorical-question`     | "Why? Because" and "The result?"                                                                                   |
 | `emoji-list`              | List items that start with an emoji                                                                                |
 | `no-short-term-relevance` | Comments that stop making sense once the change lands: "no longer", "no behavior change", "for now", "Previously," |
@@ -144,6 +164,7 @@ Files of other types, binary files, lockfiles, minified files, and files over 2
 | `quoted-error`            | Comments that quote an error message: `"Cannot read properties of..."`                                             |
 | `dramatic-verb`           | blows up, dies with, falls over, chokes on, and friends                                                            |
 | `parenthetical-aside`     | Asides like `(and their compiled handles)`                                                                         |
+| `mid-phrase-break`        | A line that stops mid phrase, on a word such as the, of, or that                                                   |
 | `long-sentence`           | A sentence over 30 words                                                                                           |
 | `wall-of-text`            | A paragraph over 120 words or 7 sentences                                                                          |
 | `uniform-paragraphs`      | Three or more paragraphs in a row of about the same length                                                         |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nollm",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "lint against LLMisms in your codebase",
   "keywords": [
     "comments",

package/src/cli.js

@@ -1,18 +1,29 @@
 import { createRequire } from "node:module";
+import { resolve } from "node:path";
+import { text as readAll } from "node:stream/consumers";
 import { parseArgs, styleText } from "node:util";
+import { check } from "./check.js";
+import { findConfig, loadConfig } from "./config.js";
+import { classify } from "./languages.js";
 import { lint } from "./lint.js";
 import { rules } from "./rules.js";
 
+const STDIN_FILENAME = "stdin.md";
+
 const HELP = `Usage: nollm [options] [paths...]
 
 Checks files for LLMisms and prints each finding as soon as it is found.
 Paths may be relative or absolute. Files that git ignores are skipped.
+The path - reads the text from stdin instead.
 
 Options:
   --jobs, -j <n>     Number of worker threads (default: cpu count)
   --config <path>    Config file (default: nollm.config.js in the current directory)
   --diff <ref>       Check only the lines this branch adds or changes since <ref>
   --no-git           Do not ask git for the file list. Read .gitignore files instead
+  --stdin            Read the text from stdin. Same as the path -
+  --stdin-filename <name>
+                     Check stdin as if it were this file (default: ${STDIN_FILENAME})
   --quiet, -q        Print only the summary
   --list-rules       Print every rule and exit
   --version, -v      Print the version and exit
@@ -21,13 +32,20 @@ Options:
 Examples:
   nollm docs/guide.md            a path relative to the current directory
   nollm /srv/site/docs/guide.md  an absolute path
+  pbpaste | nollm -              text from the clipboard, as markdown
+  git show HEAD:a.py | nollm --stdin-filename a.py
 
 Exit code 1 when there are findings. Exit code 2 on a usage error.
 `;
 
 export async function main(
   argv,
-  { stdout = process.stdout, stderr = process.stderr, cwd = process.cwd() } = {},
+  {
+    stdin = process.stdin,
+    stdout = process.stdout,
+    stderr = process.stderr,
+    cwd = process.cwd(),
+  } = {},
 ) {
   let parsed;
   try {
@@ -40,6 +58,8 @@ export async function main(
         config: { type: "string" },
         diff: { type: "string" },
         git: { type: "boolean", default: true },
+        stdin: { type: "boolean", default: false },
+        "stdin-filename": { type: "string" },
         quiet: { type: "boolean", short: "q", default: false },
         "list-rules": { type: "boolean", default: false },
         version: { type: "boolean", short: "v", default: false },
@@ -83,21 +103,44 @@ export async function main(
 
   const paint = (style, text) => styleText(style, text, { stream: stdout });
   const started = performance.now();
+  const fromStdin =
+    values.stdin || values["stdin-filename"] !== undefined || positionals.includes("-");
+
+  if (fromStdin) {
+    if (positionals.some((path) => path !== "-")) {
+      stderr.write("Paths cannot be combined with stdin. Check one or the other\n");
+      return 2;
+    }
+    if (values.diff !== undefined) {
+      stderr.write("--diff cannot be combined with stdin, since stdin has no git history\n");
+      return 2;
+    }
+  }
+
+  const onResult = (result) => {
+    if (values.quiet || result.findings.length === 0) return;
+    stdout.write(formatFile(result.file, result.findings, paint));
+  };
 
   let summary;
   try {
-    summary = await lint({
-      roots: positionals.length > 0 ? positionals : ["."],
-      cwd,
-      configPath: values.config,
-      git: values.git,
-      diff: values.diff,
-      jobs,
-      onResult(result) {
-        if (values.quiet || result.findings.length === 0) return;
-        stdout.write(formatFile(result.file, result.findings, paint));
-      },
-    });
+    summary = fromStdin
+      ? await lintStdin({
+          stdin,
+          file: values["stdin-filename"] ?? STDIN_FILENAME,
+          cwd,
+          configPath: values.config,
+          onResult,
+        })
+      : await lint({
+          roots: positionals.length > 0 ? positionals : ["."],
+          cwd,
+          configPath: values.config,
+          git: values.git,
+          diff: values.diff,
+          jobs,
+          onResult,
+        });
   } catch (error) {
     stderr.write(`${error.message}\n`);
     return 2;
@@ -112,6 +155,30 @@ export async function main(
   return summary.findings > 0 ? 1 : 0;
 }
 
+/**
+ * Checks the text on stdin as if it were the named file.
+ *
+ * The name picks the language and labels the report. No file is read.
+ * Returns the same summary as lint, for one file.
+ */
+async function lintStdin({ stdin, file, cwd, configPath, onResult }) {
+  if (!classify(file)) {
+    throw new Error(`nollm does not know how to check "${file}". Pass --stdin-filename a.md`);
+  }
+
+  const resolvedConfig = configPath ? resolve(cwd, configPath) : await findConfig(cwd);
+  const config = await loadConfig(resolvedConfig);
+  const findings = check(file, await readAll(stdin), config.rules);
+
+  onResult({ file, findings, skipped: null });
+  return {
+    files: 1,
+    checked: 1,
+    findings: findings.length,
+    filesWithFindings: findings.length > 0 ? 1 : 0,
+  };
+}
+
 /**
  * One block per file:
  *

package/src/diff.js

@@ -22,14 +22,14 @@ export const ALL_LINES = true;
 /**
  * The lines a branch adds or changes, per file.
  *
- * Returns a Map from path to a Set of line numbers in the new file, or to
- * ALL_LINES when the whole file is new.
+ * Returns a Map from path to a Set of line numbers in the new file,
+ * or to ALL_LINES when the whole file is new.
  * Paths are relative to cwd, so they match what collectFiles returns.
  *
  * The comparison starts at the merge base of base and the working tree, so
  * commits that land on base after the branch point stay out of the result.
- * Uncommitted edits and files git does not track yet count as part of the
- * branch, so the same call works before a push.
+ * Uncommitted edits and files git does not track yet count as part of the branch,
+ * so the same call works before a push.
  *
  * Files with no added or changed lines are left out. So are deleted files.
  */
@@ -124,9 +124,10 @@ async function real(path) {
 /**
  * Says what is wrong with a base ref before git says it less clearly.
  *
- * A checkout that fetched one branch, or fetched to a shallow depth, is the
- * usual reason a ref is missing. CI does both by default, so the ref a pull
- * request is against is often the one that is not there.
+ * A checkout that fetched one branch, or fetched to a shallow depth,
+ * is the usual reason a ref is missing.
+ * CI does both by default,
+ * so the ref a branch is compared against is often the one that is missing.
  */
 async function checkBase(dir) {
   if ((await tryGit(["rev-parse", "--is-inside-work-tree"], dir)) === null) {
@@ -166,9 +167,10 @@ async function noMergeBase(base, dir) {
 /**
  * How to fetch a missing ref. "origin/develop" needs "origin develop".
  *
- * A branch name may hold a slash of its own, so the first part counts as a
- * remote only when the repository lists it as one. A repository with no
- * remotes gives nothing to check against, so the usual reading wins.
+ * A branch name may hold a slash of its own,
+ * so the first part counts as a remote only when the repository lists it as one.
+ * A repository with no remotes gives nothing to check against,
+ * so the usual reading wins.
  */
 async function fetchArgs(base, cwd) {
   const cut = base.indexOf("/");
@@ -220,8 +222,9 @@ async function untracked(cwd) {
 /**
  * Reads a unified diff produced with --unified=0 and --no-prefix.
  *
- * Every file starts with a "diff --git" line, so that line marks where the
- * next "+++" is a header and not a line of added content that starts with "++".
+ * Every file starts with a "diff --git" line,
+ * so that line marks where the next "+++" is a header,
+ * and not a line of added content that starts with "++".
  */
 function parse(patch) {
   const changed = new Map();

package/src/files.js

@@ -30,12 +30,13 @@ const EVERYTHING = ["**/*", "**/.*", "**/.*/**"];
  * git only knows about its own work tree, so roots outside cwd are always
  * walked.
  *
- * The ignore option takes patterns in .gitignore syntax. They describe the
- * project, so they apply under cwd and leave paths outside cwd alone.
+ * The ignore option takes patterns in .gitignore syntax.
+ * They describe the project, so they apply under cwd
+ * and leave paths outside cwd alone.
  *
- * Roots and cwd are compared after their symlinks are followed. On macOS a
- * temp directory is reached through /var and lives in /private/var, and
- * without this a path under cwd would look like a path outside it.
+ * Roots and cwd are compared after their symlinks are followed.
+ * On macOS a temp directory is reached through /var and lives in /private/var,
+ * so without this a path under cwd would look like a path outside it.
  */
 export async function collectFiles(
   roots,
@@ -113,11 +114,12 @@ async function fromWalk(roots, base) {
  * so an ignored tree is never opened.
  *
  * A .gitignore applies to the directory that holds it and to everything below.
- * So each entry is matched against the chain of files from the root down to
- * its own directory. Chains are built once per directory and kept.
+ * So each entry is matched against the chain of files
+ * from the root down to its own directory.
+ * Chains are built once per directory and kept.
  *
- * glob asks this question synchronously, so the reads are synchronous. It is
- * one small file per directory, which is what the walk read before.
+ * glob asks this question synchronously, so the reads are synchronous.
+ * It is one small file per directory, which is what the walk read before.
  */
 function skips(root) {
   const chains = new Map();

package/src/rules.js

@@ -1,4 +1,10 @@
-import { longSentences, uniformParagraphs, uniformSentences, wallOfText } from "./shape.js";
+import {
+  longSentences,
+  midPhraseBreaks,
+  uniformParagraphs,
+  uniformSentences,
+  wallOfText,
+} from "./shape.js";
 
 /**
  * A rule is a regular expression plus a message,
@@ -28,6 +34,23 @@ function escape(text) {
   return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
 
+/**
+ * Three shapes of one move: naming the thing it is not.
+ *
+ *   not just X, but Y
+ *   it is not X, it is Y
+ *   X, not Y: the rest
+ *
+ * The third wants a label colon, so "a, not https://b" stays out. It also
+ * stays on one line, so a wrapped block comment cannot join two sentences
+ * into a match.
+ */
+const CONTRAST_CLICHES = [
+  String.raw`\bnot (?:just|only|merely|simply) [^.\n]{1,60}?,? but(?: also)?\b`,
+  String.raw`\bit'?s not (?:just |about )?[^.\n]{1,40}?[,;] it'?s\b`,
+  String.raw`[^\s,:][^,:\n]{0,29},[ \t]+not[ \t]+[^,:\n]{1,30}:(?=\s|$)`,
+];
+
 const BANNED_WORDS = [
   "genuinely",
   "fails loudly",
@@ -508,9 +531,8 @@ export const rules = [
   },
   {
     id: "contrast-cliche",
-    message: "Contrast cliche (not just X, but Y)",
-    pattern:
-      /\b(?:not (?:just|only|merely|simply) [^.\n]{1,60}?,? but(?: also)?\b|it'?s not (?:just |about )?[^.\n]{1,40}?[,;] it'?s\b)/gi,
+    message: "Contrast cliche. State the thing directly",
+    pattern: new RegExp(CONTRAST_CLICHES.join("|"), "gi"),
   },
   {
     id: "rhetorical-question",
@@ -562,6 +584,11 @@ export const rules = [
     pattern: new RegExp(String.raw`${COMMENT_START}(?:${words(WHAT_COMMENT_STARTS)})\b`, "gmi"),
     scope: "comments",
   },
+  {
+    id: "mid-phrase-break",
+    message: "Line break in the middle of a phrase. Break where there is a pause",
+    check: midPhraseBreaks,
+  },
   {
     id: "long-sentence",
     message: "Long sentence. One idea per sentence",

package/src/shape.js

@@ -11,6 +11,86 @@ const TRAILER = /\s*(?:\*\/|-->|"""|''')\s*$/;
 const LIST_ITEM = /^(?:[-*+]|\d+[.)])\s+|^@\w+/;
 const SENTENCE_END = /[.!?]+(?:["')\]]+)?(?:\s+|$)/;
 
+/**
+ * Words that cannot end a thought.
+ * They point at whatever comes next,
+ * so a line that stops on one stopped in the middle of a phrase.
+ *
+ * Words that can stand at the end of a clause stay out, however often they
+ * also appear mid phrase. "Yes it can" and "give it to her" are ordinary,
+ * so can and her are not here.
+ */
+export const DANGLING_WORDS = [
+  "a",
+  "an",
+  "the",
+  "of",
+  "to",
+  "in",
+  "on",
+  "at",
+  "by",
+  "for",
+  "with",
+  "from",
+  "into",
+  "onto",
+  "upon",
+  "over",
+  "under",
+  "about",
+  "across",
+  "after",
+  "before",
+  "between",
+  "during",
+  "through",
+  "toward",
+  "towards",
+  "within",
+  "without",
+  "against",
+  "among",
+  "around",
+  "beyond",
+  "per",
+  "via",
+  "and",
+  "or",
+  "but",
+  "nor",
+  "that",
+  "which",
+  "who",
+  "whom",
+  "whose",
+  "if",
+  "when",
+  "while",
+  "because",
+  "although",
+  "though",
+  "unless",
+  "until",
+  "since",
+  "whether",
+  "is",
+  "are",
+  "was",
+  "were",
+  "has",
+  "have",
+  "had",
+  "its",
+  "their",
+  "your",
+  "our",
+  "my",
+  "every",
+];
+
+const DANGLING_END = new RegExp(String.raw`\b(${DANGLING_WORDS.join("|")})\s*$`, "i");
+
 export const WALL_WORDS = 120;
 export const WALL_SENTENCES = 7;
 export const LONG_SENTENCE_WORDS = 30;
@@ -23,8 +103,8 @@ export const UNIFORM_SENTENCE_VARIATION = 0.2;
 /**
  * Groups segments into paragraphs.
  *
- * Pass inComments: true for comment segments, so that
- * comment markers are removed before counting.
+ * Pass inComments: true for comment segments,
+ * so comment markers are removed before counting.
  *
  * Each paragraph has:
  *   line, column → where it starts
@@ -260,3 +340,56 @@ function preview(text) {
   const words = text.split(/\s+/, 6);
   return words.join(" ") + (words.length === 6 ? "..." : "");
 }
+
+/**
+ * Line breaks that land in the middle of a phrase.
+ *
+ * A break reads as a pause, so the line before it should be able to stop.
+ * A line ending on a word that points at the next one cannot.
+ *
+ * Only a break the next line carries on counts.
+ * A blank line, a heading, a table, a fence, or a new list item
+ * each end the thought by themselves, so the line before one is skipped.
+ */
+export function midPhraseBreaks(segments, scope) {
+  const inComments = scope === "comments";
+  const found = [];
+  let fence = false;
+  let previous = null;
+
+  for (let i = 0; i < segments.length; i++) {
+    const segment = segments[i];
+    const text = inComments ? strip(segment.text) : segment.text.trim();
+
+    if (text.startsWith("```") || text.startsWith("~~~")) {
+      fence = !fence;
+      previous = null;
+      continue;
+    }
+
+    if (fence || text.length === 0 || text.startsWith("#") || text.startsWith("|")) {
+      previous = null;
+      continue;
+    }
+
+    const carriesOn = previous !== null && segment.line === previous.line + 1;
+    if (carriesOn && !LIST_ITEM.test(text)) {
+      const match = DANGLING_END.exec(previous.text);
+      if (match) {
+        found.push({
+          line: previous.line,
+          column: previous.column + match.index,
+          text: match[1],
+        });
+      }
+    }
+
+    previous = {
+      text,
+      line: segment.line,
+      column: segment.column + segment.text.indexOf(text),
+    };
+  }
+
+  return found;
+}