@casomoltd/tooling 0.41.0 → 0.42.0

package/bin/buttondown.mjs

@@ -8,9 +8,11 @@
  *   buttondown [diff]              what live differs in: fields and tags
  *   buttondown apply               patch what differs
  *   buttondown check               gate the skins and settings (no key)
+ *   buttondown check <file.md>     gate one post's copy (no key)
  *   buttondown tokens              rewrite the web skin's TOKENS block
  *   buttondown emit <dir>          write each long field out as rendered
- *   buttondown draft <file.md>     create or replace a draft
+ *   buttondown draft <file.md>     gate the post, then create or replace
+ *                                  its draft
  *   buttondown upload <image>      upload an image, print its URL
  *   buttondown api <METHOD> <path> [json] [--send]
  *                                  any call, for setting up a manual test;
@@ -24,21 +26,48 @@ import {
   apply, assertApiPath, assertNotSending, client, draft, drift, emit,
   loadSettings, readKey, runCheck, upload, writeTokens,
 } from "../buttondown/engine.mjs";
+import {checkPost} from "../buttondown/post-check.mjs";
 
 /** The one flag: without it, `api` refuses any call that could send an
  *  email, and no other command takes it. */
 const SEND_FLAG = "--send";
 
-const USAGE = "usage: buttondown [diff|apply|check|tokens|emit <dir>|"
+const USAGE = "usage: buttondown [diff|apply|check [file.md]|tokens|emit <dir>|"
   + `draft <file.md>|upload <image>|api <METHOD> <path> [json] [${SEND_FLAG}]]`;
 
+/** The exit status for a command line that can't be run as written,
+ *  apart from 1, which every command uses for a refusal or a problem. */
+const EXIT_USAGE = 2;
+
+/** The indent `api` prints a response body with. */
+const JSON_INDENT = 2;
+
+/** `api` takes a method and a path, and optionally a JSON body. */
+const API_ARGS = ["method", "path", "json"];
+
 /** Each command with the fewest and most arguments it takes. */
 const ARGS = {
-  diff: [0, 0], apply: [0, 0], check: [0, 0], tokens: [0, 0],
-  emit: [1, 1], draft: [1, 1], upload: [1, 1], api: [2, 3],
+  diff: [0, 0], apply: [0, 0], check: [0, 1], tokens: [0, 0],
+  emit: [1, 1], draft: [1, 1], upload: [1, 1],
+  api: [API_ARGS.length - 1, API_ARGS.length],
 };
 
-const argv = process.argv.slice(2);
+/** Print a post's warnings and problems; true when it may go. */
+async function postPasses(file) {
+  const {problems, warnings} = await checkPost(file, {root: process.cwd()});
+  for (const w of warnings) console.warn(`buttondown check: warning: ${w}`);
+  if (problems.length) {
+    console.error(`buttondown check: ${file}:\n  ${problems.join("\n  ")}`);
+    return false;
+  }
+  console.log(`buttondown check: ${file} clean`);
+  return true;
+}
+
+/** `node` and this script come first in `process.argv`. */
+const OWN_ARGS_FROM = 2;
+
+const argv = process.argv.slice(OWN_ARGS_FROM);
 const send = argv.includes(SEND_FLAG);
 const [command = "diff", ...args] = argv.filter((a) => a !== SEND_FLAG);
 const [fewest, most] = ARGS[command] ?? [];
@@ -46,9 +75,12 @@ const badUsage = fewest === undefined || args.length < fewest
   || args.length > most || (send && command !== "api");
 if (badUsage) {
   console.error(USAGE);
-  process.exit(2);
+  process.exit(EXIT_USAGE);
 }
 
+const checksOnePost = command === "check" && args.length > 0;
+if (checksOnePost) process.exit(await postPasses(args[0]) ? 0 : 1);
+
 const settings = await loadSettings();
 const api = async () => client(settings, await readKey());
 
@@ -87,6 +119,7 @@ switch (command) {
     for (const field of await emit(settings, args[0])) console.log(field);
     break;
   case "draft":
+    if (!await postPasses(args[0])) process.exit(1);
     await draft(settings, await api(), args[0]);
     break;
   case "upload":
@@ -105,7 +138,8 @@ switch (command) {
       }
     }
     const res = await (await api()).call(method.toUpperCase(), url, payload);
-    console.log(res.status, res.data ? JSON.stringify(res.data, null, 2) : "");
+    const shown = res.data ? JSON.stringify(res.data, null, JSON_INDENT) : "";
+    console.log(res.status, shown);
     process.exit(res.ok ? 0 : 1);
   }
 }

package/bin/check-personal-data.mjs

@@ -36,6 +36,7 @@
  */
 import {execFileSync} from "node:child_process";
 import {existsSync, readFileSync, realpathSync, statSync} from "node:fs";
+import path from "node:path";
 import {pathToFileURL} from "node:url";
 import {CASOMO_CONFIG_KEYS, loadCasomoConfig} from "./utils.mjs";
 
@@ -202,9 +203,26 @@ function files(stagedOnly) {
       && statSync(f).isFile());
 }
 
-function denied(path) {
-  if (!existsSync(path)) return [];
-  return readFileSync(path, "utf8").split("\n")
+/** Where a repo keeps its local denylist unless its settings say. */
+export const DEFAULT_DENYLIST = ".personal-data-denylist";
+
+/** The rules and denied names a repo's `personalData` settings give, and the
+ *  denylist path they were read from (relative paths against `root`). */
+export function personalDataSettings(config, root = process.cwd()) {
+  const denylist = path.resolve(root, config.denylist ?? DEFAULT_DENYLIST);
+  return {
+    rules: [...identityRules(config.allowEmail), ...(config.rules ?? [])],
+    names: readDenylist(denylist),
+    denylist,
+    hasDenylist: existsSync(denylist),
+  };
+}
+
+/** The names on a local denylist, one per line; `#` starts a comment. No
+ *  file means no names. */
+export function readDenylist(file) {
+  if (!existsSync(file)) return [];
+  return readFileSync(file, "utf8").split("\n")
     .map((l) => l.trim()).filter((l) => l && !l.startsWith("#"));
 }
 
@@ -229,9 +247,8 @@ export function scan(fileList, read, rules, names) {
 
 async function main() {
   const config = await loadCasomoConfig(CASOMO_CONFIG_KEYS.personalData);
-  const denylist = config.denylist ?? ".personal-data-denylist";
-  const rules = [...identityRules(config.allowEmail), ...(config.rules ?? [])];
-  const names = denied(denylist);
+  const denylist = config.denylist ?? DEFAULT_DENYLIST;
+  const {rules, names} = personalDataSettings(config);
   const allow = (config.allow ?? []).map((a) =>
     a instanceof RegExp ? a : new RegExp(a));
 

package/buttondown/banned-words.json

@@ -0,0 +1,26 @@
+{
+  "words": [
+    "solution",
+    "solutions",
+    "innovative",
+    "cutting-edge",
+    "world-class",
+    "best-in-class",
+    "unparalleled",
+    "passionate",
+    "exciting",
+    "stakeholder",
+    "stakeholders",
+    "synergy",
+    "empower",
+    "empowers",
+    "unlock",
+    "unlocks",
+    "deep dive",
+    "reach out",
+    "circle back",
+    "touch base",
+    "let's connect"
+  ],
+  "inContext": ["leverage", "drive", "bandwidth"]
+}

package/buttondown/post-check.mjs

@@ -0,0 +1,317 @@
+/**
+ * The copy gate for one newsletter post, behind `buttondown check
+ * <file.md>`; `draft` runs it before it uploads anything. A problem fails
+ * the post, a warning is printed and doesn't. The checks and their
+ * thresholds are in `docs/buttondown.md`, *What `check` refuses in a post*.
+ */
+import {execFile} from "node:child_process";
+import {existsSync, readFileSync} from "node:fs";
+import path from "node:path";
+import {
+  findPersonalData, personalDataSettings,
+} from "../bin/check-personal-data.mjs";
+import {
+  CASOMO_CONFIG_KEYS, loadCasomoConfig, loadConfig,
+} from "../bin/utils.mjs";
+import {
+  DEFAULT_MAX_GRADE, DEFAULT_WORD_SUBS, scoreText,
+} from "../readability/index.mjs";
+
+/** The banned-word list: `words` are checked, `inContext` words are wrong
+ *  in one sense only, which a word match can't tell, so they are not. */
+export const BANNED = JSON.parse(readFileSync(
+  new URL("banned-words.json", import.meta.url), "utf8"));
+
+const FRONT_MATTER = /^---\n([\s\S]*?)\n---\n/;
+/** The line that opens a post's body names its subject. */
+const HEADING = "# ";
+
+/** An em dash, as the character or as any HTML entity for it. */
+const EM_DASH = /—|&mdash;|&#8212;|&#x2014;/i;
+
+/** The HTML elements a reader takes as a unit of their own. */
+const BLOCK_END = /<\/(?:p|li|h[1-6]|figcaption|td|th|div|blockquote)>/gi;
+/** A markdown line that stands alone: a heading, list item or table row. */
+const BLOCK_LINE = /^[ \t]*(?:#+ |[-*+] |\d+\. |\|)/;
+/** How a sentence ends, for the readability formulas. */
+const ENDS_SENTENCE = /[.!?:;]\s*$/;
+/** A block that isn't running prose: a heading, list, table, quote,
+ *  caption or emphasis line, image or HTML block. */
+const NOT_PARAGRAPH = /^(?:[#\-*+>|<!]|\d+\.)/;
+
+/** The names cspell looks for its config under, in its own order. */
+const CSPELL_CONFIGS = [
+  ".cspell.json", "cspell.json", ".cSpell.json", "cSpell.json",
+  "cspell.config.json", "cspell.config.mjs", "cspell.config.js",
+  "cspell.config.yaml", "cspell.yaml",
+];
+/** cspell's exit status when it found unknown words, and nothing worse. */
+const CSPELL_FOUND = 1;
+
+/** A link that is gone for good: the page was removed, not mislaid. */
+const NOT_FOUND = 404;
+const GONE_FOR_GOOD = 410;
+const GONE = new Set([NOT_FOUND, GONE_FOR_GOOD]);
+/** A status from the publisher's side, which may clear on a retry. */
+const SERVER_ERROR = 500;
+/** A refusal from the publisher (403, 429), which says nothing either way
+ *  about the page. */
+const CLIENT_ERROR = 400;
+/** How long one fetch of a link may take. Proposed here; no external
+ *  source. */
+const LINK_TIMEOUT_MS = 10_000;
+
+/**
+ * Every problem and warning for the post at `file`, checked as the repo
+ * at `root` configures it: `root` picks whose word lists, ceiling and
+ * personal-data rules apply, so it has no default. `fetchImpl` is the link
+ * fetcher, so a test runs without the network.
+ */
+export async function checkPost(file, {root, fetchImpl = fetch}) {
+  if (!root) throw new Error("checkPost needs the repo root to check against");
+  const raw = readFileSync(file, "utf8");
+  const post = parsePost(raw);
+  const config = await loadConfig("readability.config.mjs", root);
+  return merge([
+    readabilityProblems(post, {
+      maxGrade: config.maxGrade ?? DEFAULT_MAX_GRADE,
+      wordSubs: config.wordSubs ?? DEFAULT_WORD_SUBS,
+    }),
+    emDashProblems(post),
+    bannedProblems(post),
+    await personalDataProblems(raw, root),
+    await spellingProblems(raw, root),
+    await linkProblems(post, fetchImpl),
+  ]);
+}
+
+/** A post split into the fields the gate reads. Like `draft`, it refuses a
+ *  post whose body doesn't open on `# Subject`. */
+export function parsePost(raw) {
+  const m = raw.match(FRONT_MATTER);
+  const fields = Object.fromEntries((m?.[1] ?? "").split("\n")
+    .filter((line) => line.includes(":"))
+    .map((line) => {
+      const at = line.indexOf(":");
+      return [line.slice(0, at).trim(), line.slice(at + 1).trim()];
+    }));
+  const lines = (m ? raw.slice(m[0].length) : raw).trimStart().split("\n");
+  if (!lines[0]?.startsWith(HEADING)) {
+    throw new Error("after any front matter, the first line must be `# Subject`");
+  }
+  return {
+    subject: lines[0].slice(HEADING.length).trim(),
+    description: fields.description ?? "",
+    image: fields.image ?? "",
+    body: lines.slice(1).join("\n"),
+  };
+}
+
+/** The words a reader reads: markup, code and URLs taken out, link text
+ *  and alt text kept. */
+export function proseOf(markdown) {
+  const blocks = endBlocks(withoutCode(stripSpans(markdown, "<!--", "-->")));
+  return stripSpans(blocks, "<", ">")
+    .replace(/\]\([^)\s]*\)/g, " ")
+    .replace(/!?\[/g, " ")
+    .replace(/https?:\/\/\S+/g, " ")
+    .replace(/&[a-z]+;/g, " ")
+    .replace(/^[ \t]*(?:#+|>|[-*+]|\d+\.|\|)/gm, "")
+    .replace(/[*_|]/g, " ");
+}
+
+/** The body's paragraphs: the blank-line-separated blocks of running text.
+ *  The formulas score sentences, and a caption or list item dense with
+ *  figures and titles reads as a hard sentence when it isn't one. */
+export function paragraphsOf(body) {
+  return body.split(/\n[ \t]*\n/).map((block) => block.trim())
+    .filter((block) => block && !NOT_PARAGRAPH.test(block))
+    .join("\n\n");
+}
+
+/** Every http(s) URL the post links to or loads, once each; an example in
+ *  code is not a link. */
+export function linksOf(markdown) {
+  const text = withoutCode(markdown);
+  const found = [
+    ...text.matchAll(/\]\((https?:\/\/[^)\s]+)\)/g),
+    ...text.matchAll(/(?:href|src)="(https?:\/\/[^"]+)"/g),
+    ...text.matchAll(/<(https?:\/\/[^>\s]+)>/g),
+  ].map((m) => m[1]);
+  return [...new Set(found)];
+}
+
+/** Each banned word or phrase the prose uses, once. A typographic
+ *  apostrophe matches a straight one. */
+export function bannedIn(text, words = BANNED.words) {
+  const plain = text.replace(/’/g, "'");
+  return words.filter((w) => new RegExp(
+    `(?<![\\w-])${w.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(?![\\w-])`, "i",
+  ).test(plain));
+}
+
+const merge = (parts) => ({
+  problems: parts.flatMap((p) => p.problems ?? []),
+  warnings: parts.flatMap((p) => p.warnings ?? []),
+});
+
+/** `text` with every span from `open` to the next `close` taken out. A
+ *  scan rather than a lazy regex, which backtracks on an unclosed span. */
+function stripSpans(text, open, close) {
+  let out = "";
+  let at = 0;
+  for (let start = text.indexOf(open); start !== -1;
+    start = text.indexOf(open, at)) {
+    const end = text.indexOf(close, start + open.length);
+    if (end === -1) break;
+    out += `${text.slice(at, start)} `;
+    at = end + close.length;
+  }
+  return out + text.slice(at);
+}
+
+/** `text` with fenced and inline code taken out. */
+const withoutCode = (text) => stripSpans(stripSpans(text, "```", "```"), "`", "`");
+
+/** Each block ended as a sentence, so the formulas don't read a list or a
+ *  run of headings as one long sentence; a reader takes each on its own. */
+function endBlocks(text) {
+  return text.replace(BLOCK_END, ". ").split("\n")
+    .map((line) => BLOCK_LINE.test(line) && !ENDS_SENTENCE.test(line)
+      ? `${line}.` : line)
+    .join("\n");
+}
+
+function readabilityProblems(post, {maxGrade, wordSubs}) {
+  const result = scoreText(proseOf(paragraphsOf(post.body)), wordSubs);
+  if (result.unscored) {
+    return {warnings: [`readability: ${result.wordCount} words, too few to score`]};
+  }
+  return result.consensusGrade > maxGrade
+    ? {problems: [`readability: grade ${result.consensusGrade}, above ${maxGrade}`]}
+    : {};
+}
+
+/** Checked on the text as written, code aside, so an entity counts too. */
+function emDashProblems(post) {
+  const where = [
+    ["subject", post.subject], ["description", post.description],
+    ["body", withoutCode(post.body)],
+  ].filter(([, text]) => EM_DASH.test(text)).map(([field]) => field);
+  return where.length
+    ? {problems: [`em dash in the ${where.join(", ")}: use a full stop, colon, `
+      + "semicolon, comma or brackets"]}
+    : {};
+}
+
+function bannedProblems(post) {
+  const used = bannedIn(proseOf(
+    [post.subject, post.description, post.body].join("\n")));
+  return used.length ? {problems: [`banned words: ${used.join(", ")}`]} : {};
+}
+
+async function personalDataProblems(raw, root) {
+  const config = await loadCasomoConfig(CASOMO_CONFIG_KEYS.personalData, root);
+  const {rules, names, denylist, hasDenylist} = personalDataSettings(config, root);
+  const hits = findPersonalData(raw, rules, names);
+  return {
+    problems: hits.map((h) => `personal data, line ${h.line}: ${h.rule}`),
+    warnings: hasDenylist ? []
+      : [`personal data: no ${path.relative(root, denylist)}; name checking off`],
+  };
+}
+
+/** Unknown words the repo's cspell reports, read from stdin as markdown.
+ *  Any failure of cspell itself is a problem: a check that didn't run must
+ *  not read as clean. */
+async function spellingProblems(raw, root) {
+  const bin = path.join(root, "node_modules", ".bin", "cspell");
+  if (!existsSync(bin)) {
+    return {problems: ["spelling: cspell is not installed in this repo"]};
+  }
+  const config = spellConfig(root);
+  if (config.missing) {
+    return {problems: [`spelling: the spell script names ${config.missing}, `
+      + "which doesn't exist"]};
+  }
+  const run = await outputOf(bin, [
+    "lint", "--no-progress", "--no-summary", "--no-color",
+    ...(config.file ? ["--config", config.file] : []),
+    "--language-id", "markdown", "stdin",
+  ], {cwd: root, input: raw});
+  const unknown = [...run.stdout.matchAll(/Unknown word \(([^)]+)\)/g)]
+    .map((m) => m[1]);
+  if (run.code === 0) return {};
+  if (run.code === CSPELL_FOUND && unknown.length) {
+    return {problems: [`spelling: ${[...new Set(unknown)].join(", ")}`]};
+  }
+  const why = (run.stderr.trim().split("\n")[0] || run.stdout.trim()) || "no output";
+  return {problems: [`spelling: cspell failed (${run.code}): ${why}`]};
+}
+
+/** The cspell config the repo's own `spell` script names, so a post is
+ *  checked against the words its pages are; else the first file cspell's
+ *  own search would find. Read from stdin, cspell searches for neither. */
+function spellConfig(root) {
+  const pkg = path.join(root, "package.json");
+  const script = existsSync(pkg)
+    ? JSON.parse(readFileSync(pkg, "utf8")).scripts?.spell ?? "" : "";
+  const named = script.match(/(?:^|\s)(?:-c|--config)[\s=]+(\S+)/)?.[1];
+  if (named) {
+    const file = path.join(root, named.replace(/^["']|["']$/g, ""));
+    return existsSync(file) ? {file} : {missing: named};
+  }
+  return {file: CSPELL_CONFIGS.map((f) => path.join(root, f))
+    .find((f) => existsSync(f))};
+}
+
+/** What a command prints and how it exits, given `input` on stdin. A spawn
+ *  failure comes back as a non-numeric code, never as a throw. */
+const outputOf = (bin, args, {cwd, input}) => new Promise((resolve) => {
+  const child = execFile(bin, args, {cwd}, (error, stdout, stderr) => resolve({
+    code: error ? error.code ?? "error" : 0, stdout: String(stdout ?? ""),
+    stderr: String(stderr ?? ""),
+  }));
+  child.stdin.on("error", () => {});
+  child.stdin.end(input);
+});
+
+/** Every link in the body, and the share image, fetched. When every one
+ *  failed for a reason a retry might clear, the check didn't run, which is
+ *  said once rather than per link. */
+async function linkProblems(post, fetchImpl) {
+  const urls = [...new Set([
+    ...linksOf(post.body), ...(/^https?:\/\//.test(post.image) ? [post.image] : []),
+  ])];
+  const outcomes = await Promise.all(urls.map((url) => linkOutcome(url, fetchImpl)));
+  if (urls.length && outcomes.every((o) => o.transient)) {
+    return {warnings: [`links: none of ${urls.length} could be checked (network?)`]};
+  }
+  return merge(outcomes);
+}
+
+async function linkOutcome(url, fetchImpl) {
+  let r = await fetchStatus(url, fetchImpl);
+  if (transient(r)) r = await fetchStatus(url, fetchImpl);
+  if (GONE.has(r.status)) return {problems: [`link ${r.status}: ${url}`]};
+  if (transient(r)) {
+    return {transient: true,
+      warnings: [`link ${r.status ?? r.error} after a retry: ${url}`]};
+  }
+  return r.status >= CLIENT_ERROR ? {warnings: [`link ${r.status}: ${url}`]} : {};
+}
+
+const transient = (r) => r.error || r.status >= SERVER_ERROR;
+
+/** One link's status, or the error that stopped it. */
+async function fetchStatus(url, fetchImpl) {
+  try {
+    const res = await fetchImpl(url, {
+      redirect: "follow", signal: AbortSignal.timeout(LINK_TIMEOUT_MS),
+    });
+    await res.body?.cancel();
+    return {status: res.status};
+  } catch (error) {
+    return {error: error?.name ?? String(error)};
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@casomoltd/tooling",
-  "version": "0.41.0",
+  "version": "0.42.0",
   "description": "Casomo's shared dev config: ESLint, Prettier, commitlint, knip & jscpd presets, dev CLIs (git hooks, readability, screenshots, design specs), and the build-time data and helpers sites import (the Buttondown engine, the shared Organization)",
   "author": "Casomo Ltd",
   "license": "MIT",
@@ -28,7 +28,7 @@
     "check": "node bin/check-gates.mjs && node bin/verify-pack.mjs && npm run check:private-refs && npm run check:personal-data && npm run lint && npm run typecheck && npm run test && npm run lint:md",
     "lint": "eslint .",
     "typecheck": "tsc --noEmit -p hooks/tsconfig.json && tsc --noEmit -p identity/tsconfig.json",
-    "test": "node hooks/test.mts && node bin/check-gates.test.mjs && node bin/check-listed-files.test.mjs && node bin/check-personal-data.test.mjs && node bin/check-routes.test.mjs && node bin/casomo-config.test.mjs && node bin/buttondown.test.mjs && node commitlint/index.test.mjs && node readability/index.test.mjs && node identity/index.test.mjs",
+    "test": "node hooks/test.mts && node bin/check-gates.test.mjs && node bin/check-listed-files.test.mjs && node bin/check-personal-data.test.mjs && node bin/check-routes.test.mjs && node bin/casomo-config.test.mjs && node bin/buttondown.test.mjs && node buttondown/post-check.test.mjs && node commitlint/index.test.mjs && node readability/index.test.mjs && node identity/index.test.mjs",
     "lint:md": "remark --frail --quiet --no-stdout --rc-path bin/skills-lint.config.mjs skills agents docs",
     "build": "echo 'no-op: source-only repo, nothing to build'",
     "prepare": "husky || true",
@@ -55,6 +55,7 @@
       "types": "./buttondown/index.d.mts",
       "default": "./buttondown/index.mjs"
     },
+    "./banned-words": "./buttondown/banned-words.json",
     "./identity": {
       "types": "./identity/index.d.mts",
       "default": "./identity/index.mjs"