@roughen/cli 0.3.0

package/lib/site.mjs

@@ -0,0 +1,409 @@
+import { readFile, readdir, lstat } from 'node:fs/promises';
+import path from 'node:path';
+import { loadConfig, included } from './config.mjs';
+import { formatFor, lintSource } from './lint-file.mjs';
+import { parseSource, missedCopy } from './jsx.mjs';
+import { review } from './review.mjs';
+
+/**
+ * `roughen site`: which copy a Next.js app-router site ships, where its
+ * habits are, and a plan for revising it. Follows static and dynamic imports
+ * from every route file, resolves tsconfig/jsconfig paths, and never runs the
+ * site, a bundler or the network. Reads files; writes nothing.
+ */
+
+// Files Next.js renders for a route segment. route.* handlers and metadata files (sitemap, icons) aren't pages.
+const routeFile = /^(?:page|layout|template|not-found|error|global-error|loading|default)\.(?:[cm]?[jt]sx?|mdx?)$/;
+const resolvable = ['.tsx', '.ts', '.jsx', '.js', '.mjs', '.cjs', '.mts', '.cts', '.mdx', '.md'];
+// Directories that never hold shipped copy.
+const skippedDirs = new Set(['node_modules', 'dist', 'build', 'out', 'coverage', 'public', 'scripts', 'supabase', 'docs', 'test', 'tests', '__tests__', '__mocks__', 'e2e', 'patches', 'fixtures']);
+const skippedFiles = /\.(?:test|spec|stories|config|d)\.[cm]?[jt]sx?$|^(?:next-env\.d\.ts|middleware\.[jt]s|proxy\.[jt]s|instrumentation\.[jt]s)$/;
+// Route handlers and metadata files under app/: they answer requests, they aren't pages a reader reads.
+const handlerFile = /^(?:route|sitemap|robots|manifest|opengraph-image|twitter-image|icon|apple-icon)\.[cm]?[jt]sx?$/;
+
+/** tsconfig/jsconfig allow comments and trailing commas. */
+function parseJsonc(text) {
+  let out = '';
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+    if (char === '"') {
+      let j = i + 1;
+      for (; j < text.length && text[j] !== '"'; j++) if (text[j] === '\\') j++;
+      out += text.slice(i, j + 1); i = j; continue;
+    }
+    if (char === '/' && text[i + 1] === '/') { while (i < text.length && text[i] !== '\n') i++; out += '\n'; continue; }
+    if (char === '/' && text[i + 1] === '*') { const end = text.indexOf('*/', i + 2); i = end < 0 ? text.length : end + 1; continue; }
+    out += char;
+  }
+  return JSON.parse(out.replace(/,(\s*[}\]])/g, '$1'));
+}
+
+async function exists(file) {
+  try { return (await lstat(file)).isFile(); } catch { return false; }
+}
+
+/** Path aliases from tsconfig.json or jsconfig.json (one level of `extends`). */
+async function loadAliases(root) {
+  for (const name of ['tsconfig.json', 'jsconfig.json']) {
+    const file = path.join(root, name);
+    if (!await exists(file)) continue;
+    let config = parseJsonc(await readFile(file, 'utf8'));
+    let base = root;
+    if (typeof config.extends === 'string' && config.extends.startsWith('.')) {
+      const parentFile = path.resolve(root, config.extends.endsWith('.json') ? config.extends : `${config.extends}.json`);
+      if (await exists(parentFile)) {
+        const parent = parseJsonc(await readFile(parentFile, 'utf8'));
+        config = { ...parent, ...config, compilerOptions: { ...parent.compilerOptions, ...config.compilerOptions } };
+        if (!config.compilerOptions?.baseUrl && parent.compilerOptions?.baseUrl) base = path.dirname(parentFile);
+      }
+    }
+    const options = config.compilerOptions ?? {};
+    const baseUrl = options.baseUrl ? path.resolve(base, options.baseUrl) : root;
+    const paths = Object.entries(options.paths ?? {}).map(([pattern, targets]) => ({ pattern, targets: targets.map((target) => path.resolve(baseUrl, target)) }));
+    return { baseUrl: options.baseUrl ? baseUrl : null, paths };
+  }
+  return { baseUrl: null, paths: [] };
+}
+
+/** Resolves an import specifier to a file, or null for packages and anything missing. */
+async function resolveImport(from, specifier, aliases) {
+  const candidates = [];
+  if (specifier.startsWith('.')) candidates.push(path.resolve(path.dirname(from), specifier));
+  else {
+    for (const { pattern, targets } of aliases.paths) {
+      const star = pattern.indexOf('*');
+      const match = star < 0 ? specifier === pattern : specifier.startsWith(pattern.slice(0, star)) && specifier.endsWith(pattern.slice(star + 1));
+      if (!match) continue;
+      const rest = star < 0 ? '' : specifier.slice(star, specifier.length - (pattern.length - star - 1));
+      for (const target of targets) candidates.push(target.replace('*', rest));
+    }
+    if (aliases.baseUrl) candidates.push(path.join(aliases.baseUrl, specifier));
+  }
+  for (const base of candidates) {
+    // ESM-style TypeScript imports name the emitted .js file.
+    const stems = [base, ...(/\.[cm]?js$/.test(base) ? [base.replace(/\.([cm]?)js$/, '.$1ts'), base.replace(/\.js$/, '.tsx')] : [])];
+    for (const stem of stems) {
+      if (await exists(stem)) return stem;
+      for (const extension of resolvable) if (await exists(stem + extension)) return stem + extension;
+      for (const extension of resolvable) if (await exists(path.join(stem, `index${extension}`))) return path.join(stem, `index${extension}`);
+    }
+  }
+  return null;
+}
+
+/**
+ * Import specifiers in a module: static imports and re-exports, import(),
+ * require(), and template-literal import(`./cities/${slug}`), which a bundler
+ * turns into every file that can match. Type-only imports ship nothing.
+ */
+function importsOf(ast) {
+  const found = [];
+  const walk = (node) => {
+    if (!node || typeof node.type !== 'string') return;
+    if ((node.type === 'ImportDeclaration' || node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration') && node.source && node.importKind !== 'type' && node.exportKind !== 'type') {
+      found.push({ specifier: node.source.value });
+    } else if ((node.type === 'ImportExpression' || (node.type === 'CallExpression' && (node.callee.type === 'Import' || (node.callee.type === 'Identifier' && node.callee.name === 'require'))))) {
+      const argument = node.type === 'ImportExpression' ? node.source : node.arguments[0];
+      if (argument?.type === 'StringLiteral') found.push({ specifier: argument.value });
+      else if (argument?.type === 'TemplateLiteral') {
+        const quasis = argument.quasis.map((quasi) => quasi.value.cooked ?? quasi.value.raw);
+        if (argument.expressions.length === 0) found.push({ specifier: quasis[0] });
+        else if (quasis[0].includes('/')) found.push({ template: quasis });
+      }
+    }
+    for (const key of Object.keys(node)) {
+      if (key === 'loc' || key === 'start' || key === 'end' || key === 'extra' || key.endsWith('Comments')) continue;
+      const value = node[key];
+      if (Array.isArray(value)) for (const item of value) walk(item);
+      else if (value && typeof value === 'object') walk(value);
+    }
+  };
+  walk(ast.program);
+  return found;
+}
+
+/** Every file under `dir` a template import could name: the static prefix picks the directory, the rest must match. */
+async function resolveTemplate(from, quasis, aliases) {
+  const prefix = quasis[0];
+  const slash = prefix.lastIndexOf('/');
+  const directorySpecifier = prefix.slice(0, slash) || '.';
+  let directory = null;
+  if (directorySpecifier.startsWith('.')) directory = path.resolve(path.dirname(from), directorySpecifier);
+  else {
+    for (const { pattern, targets } of aliases.paths) {
+      const star = pattern.indexOf('*');
+      if (star >= 0 && directorySpecifier.startsWith(pattern.slice(0, star))) { directory = targets[0].replace('*', directorySpecifier.slice(star)); break; }
+    }
+  }
+  if (!directory) return [];
+  const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const matcher = new RegExp(`^${escape(prefix.slice(slash + 1))}${quasis.slice(1).map((quasi) => `.+${escape(quasi)}`).join('')}(?:${resolvable.map(escape).join('|')})?$`);
+  const files = [];
+  const walk = async (dir, relative) => {
+    let entries;
+    try { entries = await readdir(dir, { withFileTypes: true }); } catch { return; }
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+      if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
+      const next = relative ? `${relative}/${entry.name}` : entry.name;
+      if (entry.isDirectory()) await walk(path.join(dir, entry.name), next);
+      else if (entry.isFile() && matcher.test(next) && resolvable.includes(path.extname(entry.name))) files.push(path.join(dir, entry.name));
+    }
+  };
+  await walk(directory, '');
+  return files;
+}
+
+/** '/web-design/[area]' for app/web-design/[area]/page.jsx. Route groups and parallel slots don't appear in the URL. */
+export function routePath(appDir, file) {
+  const segments = path.relative(appDir, path.dirname(file)).split(path.sep).filter((segment) => segment && !/^\(.*\)$/.test(segment) && !segment.startsWith('@'));
+  return `/${segments.join('/')}`;
+}
+
+/** The site area a route belongs to: its first URL segment, 'home' for /, 'site' for root layouts. */
+function areaOf(entry) {
+  const first = entry.route.split('/')[1];
+  if (first) return first.replace(/^\[+\.{0,3}|\]+$/g, '') || first;
+  return entry.kind === 'page' ? 'home' : 'site';
+}
+
+async function walkFiles(dir, root, out = []) {
+  let entries;
+  try { entries = await readdir(dir, { withFileTypes: true }); } catch { return out; }
+  for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+    if (entry.name.startsWith('.') || entry.isSymbolicLink()) continue;
+    const file = path.join(dir, entry.name);
+    if (entry.isDirectory()) { if (!skippedDirs.has(entry.name)) await walkFiles(file, root, out); }
+    else if (entry.isFile() && formatFor(file) && !skippedFiles.test(entry.name)) out.push(file);
+  }
+  return out;
+}
+
+/** Findings split the way an editor sees them. */
+function tally(result, reviewed) {
+  const counts = { findings: 0, revise: 0, banned: 0, protected: 0 };
+  for (const finding of result.findings) {
+    if (finding.severity === 'info') continue;
+    if (finding.role === 'protected') { counts.protected++; continue; }
+    counts.findings++;
+    if (finding.ruleId.startsWith('voice/banned-')) counts.banned++;
+  }
+  counts.revise = reviewed.brief.items.reduce((sum, item) => sum + item.rewriteAtLeast, 0);
+  return counts;
+}
+
+const byRule = (findings) => {
+  const rules = {};
+  for (const finding of findings) if (finding.severity !== 'info') rules[finding.ruleId] = (rules[finding.ruleId] ?? 0) + 1;
+  return rules;
+};
+
+/**
+ * Groups of files for editors working in parallel. Every file lands in
+ * exactly one group: its route area if one area ships it, 'shared' if
+ * several do. Areas split into groups of about `groupWords` editable words,
+ * in path order so a directory stays together. Busiest groups first.
+ */
+export function workPlan(files, { groupWords = 6000 } = {}) {
+  const byArea = new Map();
+  for (const file of files) {
+    if (!file.editableWords) continue;
+    const area = file.areas.length === 1 ? file.areas[0] : 'shared';
+    byArea.set(area, [...(byArea.get(area) ?? []), file]);
+  }
+  const groups = [];
+  for (const [area, members] of [...byArea].sort(([a], [b]) => a.localeCompare(b))) {
+    const chunks = [];
+    let current = null;
+    for (const file of members.sort((a, b) => a.path.localeCompare(b.path))) {
+      if (!current || (current.words + file.editableWords > groupWords && current.files.length)) chunks.push(current = { words: 0, files: [] });
+      current.files.push(file);
+      current.words += file.editableWords;
+    }
+    chunks.forEach((chunk, index) => groups.push({
+      id: chunks.length > 1 ? `${area}-${index + 1}` : area,
+      area,
+      files: chunk.files.map((file) => file.path),
+      words: chunk.words,
+      findings: chunk.files.reduce((sum, file) => sum + file.findings, 0),
+      revise: chunk.files.reduce((sum, file) => sum + file.revise, 0),
+      banned: chunk.files.reduce((sum, file) => sum + file.banned, 0),
+    }));
+  }
+  return groups.sort((a, b) => b.findings - a.findings || b.words - a.words || a.id.localeCompare(b.id));
+}
+
+/**
+ * The site report. `configPath` overrides roughen.config discovery; without
+ * it each file uses its nearest config, like the linter.
+ */
+export async function planSite(input, { configPath, groupWords = 6000, maxExcerpts = 8 } = {}) {
+  const root = path.resolve(input);
+  let app = null;
+  for (const dir of [path.join(root, 'app'), path.join(root, 'src', 'app')]) {
+    try { if ((await lstat(dir)).isDirectory()) { app = dir; break; } } catch {}
+  }
+  if (!app) throw new Error(`No app/ or src/app/ directory in ${root}: roughen site reads Next.js app-router projects`);
+  const aliases = await loadAliases(root);
+  const all = await walkFiles(root, root);
+  const entries = all.filter((file) => file.startsWith(app + path.sep) && routeFile.test(path.basename(file)))
+    .map((file) => ({ file, route: routePath(app, file), kind: path.basename(file).split('.')[0] }));
+
+  // One parse per file, shared by import tracing, copy extraction and the missed-copy scan.
+  const parsed = new Map();
+  const parse = async (file) => {
+    if (!parsed.has(file)) {
+      const source = await readFile(file, 'utf8');
+      let ast = null; let error = null;
+      if (formatFor(file) === 'copy') { try { ast = parseSource(source, file); } catch (caught) { error = caught.message; } }
+      parsed.set(file, { source, ast, error });
+    }
+    return parsed.get(file);
+  };
+
+  const reach = new Map(); // file → Set of entry indexes
+  const unresolvedTemplates = [];
+  for (const [index, entry] of entries.entries()) {
+    const queue = [entry.file];
+    const seen = new Set();
+    while (queue.length) {
+      const file = queue.pop();
+      if (seen.has(file)) continue;
+      seen.add(file);
+      if (!reach.has(file)) reach.set(file, new Set());
+      reach.get(file).add(index);
+      const { ast } = await parse(file);
+      if (!ast) continue;
+      for (const found of importsOf(ast)) {
+        if (found.specifier) {
+          const target = await resolveImport(file, found.specifier, aliases);
+          if (target && target.startsWith(root + path.sep) && formatFor(target) && !seen.has(target)) queue.push(target);
+        } else {
+          const targets = await resolveTemplate(file, found.template, aliases);
+          if (!targets.length) unresolvedTemplates.push({ file: path.relative(root, file), template: found.template.join('${…}') });
+          for (const target of targets) if (!seen.has(target)) queue.push(target);
+        }
+      }
+    }
+  }
+
+  const relative = (file) => path.relative(root, file).split(path.sep).join('/');
+  const files = [];
+  const unreachable = [];
+  const missed = new Map();
+  const configs = new Map();
+  const configFor = async (file) => {
+    const loaded = await loadConfig(file, configPath);
+    configs.set(loaded.dir, true);
+    return loaded;
+  };
+  for (const file of [...new Set([...reach.keys(), ...all])].sort()) {
+    const { source, ast, error } = await parse(file);
+    const format = formatFor(file);
+    const loaded = await configFor(file);
+    const entryIndexes = reach.get(file);
+    const record = { path: relative(file), format: format === 'copy' ? 'copy' : format, routes: entryIndexes ? [...entryIndexes].map((index) => entries[index].route).filter((route, i, list) => list.indexOf(route) === i).sort() : [] };
+    if (error) { files.push({ ...record, error, words: { body: 0, short: 0, protected: 0 }, editableWords: 0, findings: 0, revise: 0, banned: 0, protected: 0, areas: [], rules: {} }); continue; }
+    if (!included(file, loaded)) { if (entryIndexes) files.push({ ...record, excluded: true, words: { body: 0, short: 0, protected: 0 }, editableWords: 0, findings: 0, revise: 0, banned: 0, protected: 0, areas: [], rules: {} }); continue; }
+    const result = lintSource(source, { file, format, config: loaded.config, ...(ast ? { ast } : {}) });
+    const words = result.copy?.words ?? { body: result.metrics.wordCount, short: 0, protected: 0 };
+    const total = words.body + words.short + words.protected;
+    if (!entryIndexes) {
+      if (total && !(file.startsWith(app + path.sep) && handlerFile.test(path.basename(file)))) unreachable.push({ path: relative(file), words: total });
+      continue;
+    }
+    const reviewed = review(source, { file, format, config: loaded.config, maxExcerpts, linted: result });
+    if (ast) {
+      for (const miss of missedCopy(source, ast, result.copy?.strings ?? [])) {
+        const entry = missed.get(miss.context) ?? { context: miss.context, words: 0, files: new Set() };
+        entry.words += miss.words; entry.files.add(relative(file));
+        missed.set(miss.context, entry);
+      }
+    }
+    const areas = [...new Set([...entryIndexes].map((index) => areaOf(entries[index])))].sort();
+    files.push({ ...record, areas, words, editableWords: words.body + words.short, ...tally(result, reviewed), rules: byRule(result.findings.filter((finding) => finding.role !== 'protected')), brief: reviewed.brief.text || null });
+  }
+
+  const shipped = files.filter((file) => !file.excluded && !file.error);
+  const fileByPath = new Map(shipped.map((file) => [file.path, file]));
+  const routes = entries.map((entry, index) => {
+    const members = [...reach].filter(([, set]) => set.has(index)).map(([file]) => fileByPath.get(relative(file))).filter(Boolean);
+    const sum = (key) => members.reduce((total, file) => total + (typeof file[key] === 'number' ? file[key] : 0), 0);
+    return { route: entry.route, kind: entry.kind, file: relative(entry.file), files: members.length, words: members.reduce((total, file) => total + file.words.body + file.words.short + file.words.protected, 0), editableWords: sum('editableWords'), findings: sum('findings'), revise: sum('revise'), banned: sum('banned') };
+  }).sort((a, b) => a.route.localeCompare(b.route) || a.kind.localeCompare(b.kind));
+
+  const totals = { files: shipped.length, words: { body: 0, short: 0, protected: 0 }, findings: 0, revise: 0, banned: 0, protected: 0, rules: {} };
+  for (const file of shipped) {
+    for (const role of ['body', 'short', 'protected']) totals.words[role] += file.words[role];
+    for (const key of ['findings', 'revise', 'banned', 'protected']) totals[key] += file[key];
+    for (const [rule, count] of Object.entries(file.rules)) totals.rules[rule] = (totals.rules[rule] ?? 0) + count;
+  }
+  return {
+    root,
+    app: relative(app),
+    configs: [...configs.keys()].map((dir) => relative(dir) || '.'),
+    routes,
+    files: shipped.sort((a, b) => b.findings - a.findings || b.editableWords - a.editableWords || a.path.localeCompare(b.path)),
+    skipped: files.filter((file) => file.excluded || file.error).map(({ path: file, excluded, error }) => ({ path: file, ...(excluded ? { excluded } : { error }) })),
+    unreachable: unreachable.sort((a, b) => b.words - a.words || a.path.localeCompare(b.path)),
+    unresolvedImports: unresolvedTemplates,
+    missedCopy: [...missed.values()].map((entry) => ({ context: entry.context, words: entry.words, files: entry.files.size })).sort((a, b) => b.words - a.words).slice(0, 20),
+    totals,
+    plan: workPlan(shipped, { groupWords }),
+  };
+}
+
+const number = (value) => value.toLocaleString('en-US');
+const count = (value, singular, plural = `${singular}s`) => `${number(value)} ${value === 1 ? singular : plural}`;
+
+/** The plan as Markdown: summary, routes, the work groups, unreachable copy. */
+export function renderPlan(site, { top = 25 } = {}) {
+  const { totals } = site;
+  const words = totals.words.body + totals.words.short + totals.words.protected;
+  const pages = site.routes.filter((route) => route.kind === 'page');
+  const lines = [
+    `# Roughen site plan: ${path.basename(site.root)}`,
+    '',
+    `${count(pages.length, 'page')} and ${count(site.routes.length - pages.length, 'other route file')} ship ${count(totals.files, 'file')} with copy: ${count(words, 'word')} (${number(totals.words.body)} body, ${number(totals.words.short)} short, ${number(totals.words.protected)} protected).`,
+    `Findings an editor can act on: ${number(totals.findings)}${totals.banned ? `, ${number(totals.banned)} of them banned characters or patterns` : ''}. The briefs ask for at least ${count(totals.revise, 'rewrite')}.${totals.protected ? ` ${count(totals.protected, 'more finding', 'more findings')} sit in protected strings.` : ''}`,
+    `${count(site.unreachable.length, 'file')} with copy ${site.unreachable.length === 1 ? "isn't" : "aren't"} reachable from any route${site.unreachable.length ? ` (${count(site.unreachable.reduce((sum, file) => sum + file.words, 0), 'word')})` : ''}.`,
+    '',
+    '## Findings by rule',
+    '',
+    ...Object.entries(totals.rules).sort((a, b) => b[1] - a[1]).map(([rule, count]) => `- ${rule}: ${number(count)}`),
+    '',
+    '## Routes',
+    '',
+    '| Route | File | Files | Words | Findings | Rewrites |',
+    '|---|---|---|---|---|---|',
+    ...site.routes.map((route) => `| ${route.route}${route.kind === 'page' ? '' : ` (${route.kind})`} | ${route.file} | ${route.files} | ${number(route.words)} | ${number(route.findings)} | ${number(route.revise)} |`),
+    '',
+    `## Work plan: ${site.plan.length} groups, no file in two`,
+    '',
+    ...site.plan.flatMap((group, index) => [
+      `${index + 1}. **${group.id}**: ${count(group.files.length, 'file')}, ${count(group.words, 'editable word')}, ${count(group.findings, 'finding')}, ${count(group.revise, 'rewrite')}`,
+      ...group.files.map((file) => {
+        const record = site.files.find((item) => item.path === file);
+        return `   - ${file} (${count(record.editableWords, 'word')}, ${count(record.findings, 'finding')})`;
+      }),
+    ]),
+    '',
+    `## Top files by findings`,
+    '',
+    ...site.files.filter((file) => file.findings).slice(0, top).map((file) => `- ${file.path}: ${count(file.findings, 'finding')}, ${count(file.editableWords, 'editable word')}, shipped by ${count(file.routes.length, 'route')}`),
+  ];
+  if (site.unreachable.length) lines.push('', '## Copy no route ships', '', ...site.unreachable.slice(0, top).map((file) => `- ${file.path} (${count(file.words, 'word')})`), ...(site.unreachable.length > top ? [`- …and ${site.unreachable.length - top} more`] : []));
+  if (site.missedCopy.length) lines.push('', '## Prose Roughen didn\'t read as copy', '', 'Strings that read as prose in places the copy reader skips. If one is copy, add its key to `copy.body` (or `copy.short`) in roughen.config.', '', ...site.missedCopy.slice(0, 10).map((entry) => `- ${entry.context}: ${count(entry.words, 'word')} in ${count(entry.files, 'file')}`));
+  if (site.unresolvedImports.length) lines.push('', '## Dynamic imports Roughen couldn\'t follow', '', ...site.unresolvedImports.map((item) => `- ${item.file}: import(\`${item.template}\`)`));
+  if (site.skipped.length) lines.push('', '## Skipped', '', ...site.skipped.map((item) => `- ${item.path}: ${item.excluded ? 'excluded by roughen.config' : `doesn't parse (${item.error})`}`));
+  return `${lines.join('\n')}\n`;
+}
+
+/** One brief per work group, for handing a group to an editor. */
+export function renderGroupBrief(site, group) {
+  const lines = [`# ${group.id}: ${count(group.files.length, 'file')}, ${count(group.words, 'editable word')}`, '', 'Edit only these files. Other editors own the other groups.', ''];
+  for (const file of group.files) {
+    const record = site.files.find((item) => item.path === file);
+    lines.push(`## ${file}`, '', record.brief ? record.brief : 'Nothing flagged. Read it once for habits Roughen doesn\'t measure, and leave it alone if it reads well.', '');
+  }
+  return `${lines.join('\n')}\n`;
+}