gcf-aif-ext 0.2.0

package/node_modules/cnlp-kit/src/cnlp.js

@@ -0,0 +1,534 @@
+// The mechanical half of cnlp-format.md. One implementation for every document kind: what
+// differs between kinds is declared in a profile document (profiles/*.md), not branched here.
+// `bin/cnlp.js check` runs it over a project's corpus; a project embedding the checker calls
+// `bodyIssues` directly. Grammar and lexicon only — no caller can judge whether a bullet says
+// something useful.
+
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+
+export const DENY = /\b(surface[sd]?|sharpen[s]?|weigh[s]?|leverage[sd]?|robust|sanity-check|ensure[sd]?)\b/i;
+const NUM = '(?:\\d+|one|two|three|four|five|six|seven|eight|nine|ten)';
+/** §5: a threshold is a digit. */
+export const SPELLED_THRESHOLD = /\b(at least|exactly|more than|fewer than|no more than|only) (one|two|three|four|five|six|seven|eight|nine|ten)\b/i;
+/** §5: a limit whose bound the phrase leaves open — the enforced half of the canonical form. */
+export const AMBIGUOUS_LIMIT = new RegExp(`\\b(?:no more than|at most|up to|not exceeding|not exceed|no fewer than|not less than)\\s+${NUM}\\b`, 'i');
+/** §5: an unquantified comparative claims a value the reader cannot check. */
+export const COMPARATIVE = /\b(better|faster|cleaner|significantly|substantially|flexible|scalable|extensible|where possible|if needed|as appropriate|should probably|may want to)\b/i;
+/** §5: the light-verb construction, which is a verb hiding inside an abstract noun. */
+const NOMINAL = 'decision|validation|verification|comparison|description|inspection|check|review|report|record|update|statement|selection|removal|migration|conversion|assessment|evaluation';
+// The noun closes the phrase or takes a preposition: `make a check pass` is a causative carrying a
+// real object, and reading it as a nominalization rejects a line that has nothing wrong with it.
+export const NOMINALIZATION = new RegExp(`\\b(?:make|makes|do|does|perform|performs|provide|provides|carry out|carries out|give|gives)\\s+(?:a|an|the)\\s+(?:${NOMINAL})\\b(?=\\s*(?:$|[,.;:)]|(?:of|on|for|to|about|against|with|in)\\b))`, 'i');
+/** §5: an agentless passive drops the actor an imperative document exists to address. */
+export const PASSIVE = /\b(?:is|are|was|were|be|been|being)\s+(?:not\s+)?([a-z]+ed)\b/i;
+/** §5: a participle naming a state of the subject, which is not an action with an actor to name. */
+const STATE_PARTICIPLE = /^(?:un[a-z]+ed|required|allowed|needed|based|limited|intended|expected)$/i;
+export const HARD_LIMIT = 250; // §4: 150 is the target, 250 is compound whatever it claims
+
+const FORMS = new Set(['scalar', 'bullet-list', 'numbered-list', 'record-list', 'keyed-block']);
+const KEY = /^([a-z][a-z0-9_]*):(.*)$/;
+
+/**
+ * Body split into lines, top-level `key:` sections and `##` headings, with fenced regions
+ * blanked — §3: a fence is opaque, its contents are data and never section keys.
+ * Leading frontmatter is stripped when present. Line numbers are 1-based within the body.
+ */
+export function parseBlocks(raw) {
+  const body = raw.replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, '');
+  let fenced = false;
+  const fences = []; // 1-based line numbers a fence covered: blanked, but not absent
+  const lines = body.split(/\r?\n/).map((l, i) => {
+    if (/^\s*```/.test(l)) { fenced = !fenced; fences.push(i + 1); return ''; }
+    if (fenced) { fences.push(i + 1); return ''; }
+    return l;
+  });
+  const sections = [];
+  const headings = [];
+  lines.forEach((l, i) => {
+    const key = l.match(KEY);
+    if (key) sections.push({ key: key[1], value: key[2].trim(), line: i + 1 });
+    if (/^##\s/.test(l)) headings.push({ text: l.trim(), line: i + 1 });
+  });
+  return { lines, sections, headings, fences: new Set(fences) };
+}
+
+/** Lines of a section: everything up to the next top-level key or `##` heading. */
+export function itemsOf(lines, at) {
+  const out = [];
+  for (let i = at; i < lines.length; i += 1) {
+    if (KEY.test(lines[i]) || /^##\s/.test(lines[i])) break;
+    if (lines[i].trim() !== '') out.push({ text: lines[i], line: i + 1 });
+  }
+  return out;
+}
+
+/** §3 record-list: `- key: value` opens a record, two-space-indented keys continue it. */
+export function readRecords(items) {
+  const records = [];
+  const stray = [];
+  const repeated = [];
+  for (const { text, line } of items) {
+    const open = text.match(/^- ([a-z][a-z0-9_]*): ?(.*)$/);
+    const sub = text.match(/^ {2}([a-z][a-z0-9_]*): ?(.*)$/);
+    if (open) records.push({ line, keys: [open[1]], values: { [open[1]]: open[2].trim() } });
+    else if (sub && records.length) {
+      const rec = records[records.length - 1];
+      // A second copy of a key would silently win: two readers, two interpretations.
+      if (rec.keys.includes(sub[1])) repeated.push({ key: sub[1], line });
+      rec.keys.push(sub[1]);
+      rec.values[sub[1]] = sub[2].trim();
+    } else stray.push({ text, line });
+  }
+  return { records, stray, repeated };
+}
+
+const listValues = (items) => items
+  .map((i) => i.text.replace(/^[-\d.]+ ?/, '').trim())
+  .map((t) => (t.includes(':') ? t.slice(0, t.indexOf(':')).trim() : t))
+  .filter((t) => t !== '');
+
+/**
+ * A value outside its domain is a typo that would otherwise pass silently — `required: ye`
+ * reading as `no` disables the check the profile meant to declare.
+ */
+function domain(key, field, value, allowed) {
+  if (!allowed.includes(value)) {
+    throw new Error(`profile section "${key}": ${field} is ${JSON.stringify(value)}, expected one of ${allowed.join(', ')}`);
+  }
+  return value;
+}
+
+/**
+ * §7: the closed vocabulary of a record key, written `status = covered, source, avoid` and
+ * separated by `;` for a second key. A value outside its vocabulary used to read as whichever
+ * value the reader assumed, which is the failure that costs most where a status decides whether
+ * a page exists at all.
+ */
+function readVocabularies(key, raw) {
+  const out = {};
+  for (const pair of (raw ?? '').split(';').map((p) => p.trim()).filter(Boolean)) {
+    const at = pair.indexOf('=');
+    if (at < 0) {
+      throw new Error(`profile section "${key}": record_values reads "<key> = <value>, <value>", found ${JSON.stringify(pair)}`);
+    }
+    const name = pair.slice(0, at).trim();
+    const values = pair.slice(at + 1).split(',').map((v) => v.trim()).filter(Boolean);
+    if (!name || values.length === 0) {
+      throw new Error(`profile section "${key}": record_values names no key or no value in ${JSON.stringify(pair)}`);
+    }
+    out[name] = values;
+  }
+  return out;
+}
+
+/** A profile document (profiles/*.md) read into the values the checker needs. */
+export function readProfile(raw) {
+  const { lines, sections } = parseBlocks(raw);
+  const block = (key) => {
+    const at = sections.find((s) => s.key === key);
+    return at ? { value: at.value, items: itemsOf(lines, at.line) } : null;
+  };
+  const names = (key) => {
+    const b = block(key);
+    if (!b) return [];
+    const out = listValues(b.items);
+    return out.length === 1 && out[0] === 'none' ? [] : out;
+  };
+  const declared = readRecords(block('sections')?.items ?? []).records.map((r) => ({
+    key: r.values.key,
+    form: domain(r.values.key, 'form', r.values.form, [...FORMS]),
+    required: domain(r.values.key, 'required', r.values.required, ['yes', 'no']) === 'yes',
+    heading: (r.values.heading ?? '').replace(/^"|"$/g, ''),
+    recordKeys: (r.values.record_keys ?? '').split(',').map((s) => s.trim()).filter(Boolean),
+    optionalRecordKeys: (r.values.optional_record_keys ?? '').split(',').map((s) => s.trim()).filter(Boolean),
+    recordValues: readVocabularies(r.values.key, r.values.record_values),
+  }));
+  return {
+    format: block('format')?.value ?? '',
+    mood: block('mood')?.value ?? '',
+    headings: names('headings').map((h) => (h.startsWith('##') ? h : `## ${h}`)),
+    frontmatterFields: names('frontmatter_fields'),
+    custom: names('custom_sections'),
+    sections: declared,
+  };
+}
+
+/** This package's own root: where the standard and the profiles that ship with it live. */
+export const PACKAGE_ROOT = new URL('../', import.meta.url);
+
+const cache = new Map();
+
+/**
+ * Absolute path of a CNL-P document: the standard, or `profiles/<name>.md`. `roots` is searched
+ * in order and the first root holding the file wins, so a project's own profile directory
+ * shadows the one that ships here. With no roots given it resolves inside this package, which
+ * is what `cnlp print` serves.
+ */
+export function resolveDoc(name = 'format', { roots = [] } = {}) {
+  // The name reaches this from a command line, so it names a document and never a path:
+  // `profiles/${name}.md` with a `..` in it walks out of the package and reads anything.
+  if (!/^[a-z][a-z0-9-]*$/.test(name)) {
+    throw new Error(`not a CNL-P document name: ${JSON.stringify(name)}`);
+  }
+  const rel = name === 'format' ? 'cnlp-format.md' : `profiles/${name}.md`;
+  const search = [...roots, PACKAGE_ROOT];
+  for (const root of search) {
+    const abs = fileURLToPath(new URL(rel, root));
+    if (existsSync(abs)) return abs;
+  }
+  // Nothing holds it: report the packaged location, which is where the caller expected it.
+  return fileURLToPath(new URL(rel, PACKAGE_ROOT));
+}
+
+/**
+ * Load a profile from a path. The cache is keyed on that path, so a project's own `skill` and
+ * the packaged `skill` are two entries rather than whichever was read first.
+ */
+export async function loadProfileAt(abs) {
+  if (!cache.has(abs)) cache.set(abs, readProfile(await readFile(abs, 'utf8')));
+  return cache.get(abs);
+}
+
+/** Load a profile by name, resolved through `roots` and then this package. */
+export async function loadProfile(name, opts) {
+  return loadProfileAt(resolveDoc(name, opts));
+}
+
+/**
+ * Body conformance with the profile and with the shared lexicon. Returns [{ line, message }]
+ * sorted by line; the caller decides the severity. A line number of 0 means the issue is
+ * about the document as a whole.
+ */
+export function bodyIssues(raw, profile) {
+  const { lines, sections, headings, fences } = parseBlocks(raw);
+  const issues = [];
+  const at = (line, message) => issues.push({ line, message });
+  const byKey = new Map(profile.sections.map((s) => [s.key, s]));
+  const order = profile.sections.map((s) => s.key);
+
+  const seen = headings.map((h) => h.text);
+  for (const want of profile.headings) {
+    if (!seen.includes(want)) at(0, `missing heading "${want}"`);
+  }
+  for (const h of headings) {
+    if (!profile.headings.includes(h.text)) at(h.line, `unknown heading "${h.text}" — the profile declares ${profile.headings.join(', ') || 'none'}`);
+    else if (seen.indexOf(h.text) !== seen.lastIndexOf(h.text)) at(h.line, `heading "${h.text}" appears more than 1 time`);
+  }
+  const headingOrder = seen.filter((t) => profile.headings.includes(t));
+  if (headingOrder.join() !== [...new Set(headingOrder)].sort((a, b) => profile.headings.indexOf(a) - profile.headings.indexOf(b)).join()) {
+    at(0, `headings out of order — the profile declares ${profile.headings.join(', ')}`);
+  }
+
+  for (const s of profile.sections) {
+    if (s.required && !sections.some((f) => f.key === s.key)) at(0, `missing required block "${s.key}:"`);
+  }
+
+  const declared = new Set();
+  for (const { key, line } of sections) {
+    if (declared.has(key)) at(line, `block "${key}:" appears more than 1 time — a fact lives in exactly 1 place`);
+    declared.add(key);
+    if (byKey.has(key) || profile.custom.includes(key)) continue;
+    if (profile.frontmatterFields.includes(key)) at(line, `"${key}:" is a frontmatter field, not a body block`);
+    else at(line, `unknown block "${key}:" — reuse a name the profile declares`);
+  }
+
+  const found = sections.map((s) => s.key).filter((k) => byKey.has(k));
+  const sorted = [...found].sort((a, b) => order.indexOf(a) - order.indexOf(b));
+  if (found.join() !== sorted.join()) at(0, `blocks out of order — the profile declares ${order.join(', ')}`);
+
+  for (const { key, value, line } of sections) {
+    const spec = byKey.get(key);
+    if (!spec) continue; // a custom section declares no form
+    const items = itemsOf(lines, line);
+    if (spec.heading) {
+      const under = [...headings].reverse().find((h) => h.line < line);
+      if (!under) at(line, `"${key}:" belongs under "${spec.heading}", found before any heading`);
+      else if (under.text !== spec.heading) at(line, `"${key}:" belongs under "${spec.heading}", found under "${under.text}"`);
+    }
+    if (spec.form === 'scalar') {
+      if (!value) at(line, `"${key}:" is a scalar — its value goes on the key line`);
+      continue;
+    }
+    if (value) at(line, `"${key}:" carries a value on the key line but its form is ${spec.form}`);
+    else if (items.length === 0) {
+      // A fence is opaque content, not absence: its lines are blanked, so ask the parser.
+      if (!coversFence(fences, line, lines)) {
+        at(line, `"${key}:" is empty — a required block states "- none" with the reason, an optional one is deleted`);
+      }
+      continue;
+    }
+    // "- none: reason" is the declared way to say a required block has nothing, in any form.
+    // It is the whole content of a required block: anything else makes it 2 statements.
+    const none = items.find((i) => /^- none\b/.test(i.text));
+    if (none) {
+      if (!spec.required) at(none.line, `"${key}:" is optional and has nothing to say — delete the block instead of stating "- none"`);
+      else if (items.length !== 1) at(none.line, `"${key}:" states "- none" alongside other items — the sentinel is the whole content or it is not used`);
+      else if (!/^- none: \S/.test(none.text)) at(none.line, `"${key}:" states "- none" without the reason, which is what the reader needs`);
+      continue;
+    }
+    issues.push(...formIssues(key, spec, items));
+  }
+
+  issues.push(...lexiconIssues(lines, profile.mood));
+  return issues.sort((a, b) => a.line - b.line);
+}
+
+/** Whether the region under a key held a fenced block — §3: a fence is opaque, not empty. */
+function coversFence(fences, at, lines) {
+  for (let i = at; i < lines.length; i += 1) {
+    if (KEY.test(lines[i]) || /^##\s/.test(lines[i])) return false;
+    if (fences.has(i + 1)) return true;
+  }
+  return false;
+}
+
+/** §7: a value the profile closed to a vocabulary, held to it. A key the profile left open passes. */
+function vocabularyIssues(spec, key, name, value, line) {
+  const allowed = spec.recordValues?.[name];
+  if (!allowed || allowed.includes(value)) return [];
+  return [{ line, message: `"${name}:" is ${JSON.stringify(value)} in "${key}:" — the profile declares ${allowed.join(', ')}` }];
+}
+
+function formIssues(key, spec, items) {
+  const issues = [];
+  const at = (line, message) => issues.push({ line, message });
+  if (spec.form === 'numbered-list') {
+    const bad = items.find((i) => !/^\d+\. /.test(i.text));
+    if (bad) at(bad.line, `"${key}:" is a numbered-list — every item opens with "N. "`);
+    else {
+      // The numbers are the order: a gap or a repeat makes 2 readings of the same sequence.
+      items.forEach((i, n) => {
+        const got = Number(i.text.match(/^(\d+)\./)[1]);
+        if (got !== n + 1) at(i.line, `"${key}:" step ${got} is out of sequence — the steps run 1..${items.length}`);
+      });
+    }
+    return issues;
+  }
+  if (spec.form === 'bullet-list') {
+    const bad = items.find((i) => !/^- /.test(i.text));
+    if (bad) at(bad.line, `"${key}:" is a bullet-list — every item opens with "- "`);
+    return issues;
+  }
+  if (spec.form === 'keyed-block') {
+    const bad = items.find((i) => !/^ {2}[a-z][a-z0-9_]*:/.test(i.text));
+    if (bad) at(bad.line, `"${key}:" is a keyed-block — its sub-keys are two-space-indented`);
+    const allowed = [...spec.recordKeys, ...spec.optionalRecordKeys];
+    const present = [];
+    for (const i of items) {
+      const [sub, ...rest] = i.text.trim().split(':');
+      if (present.includes(sub)) at(i.line, `"${key}:" carries "${sub}:" more than 1 time`);
+      else if (!allowed.includes(sub)) at(i.line, `"${sub}" is not a "${key}:" sub-key — the profile declares ${allowed.join(', ')}`);
+      else issues.push(...vocabularyIssues(spec, key, sub, rest.join(':').trim(), i.line));
+      present.push(sub);
+    }
+    for (const want of spec.recordKeys) {
+      if (!present.includes(want)) at(items[0].line, `"${key}:" is missing "${want}:"`);
+    }
+    return issues;
+  }
+  // record-list
+  const { records, stray, repeated } = readRecords(items);
+  for (const r of repeated) at(r.line, `a "${key}:" record carries "${r.key}:" more than 1 time`);
+  for (const s of stray) at(s.line, `"${key}:" is a record-list — a record opens with "- ${spec.recordKeys[0] ?? 'key'}: …" then two-space-indented keys`);
+  const allowed = [...spec.recordKeys, ...spec.optionalRecordKeys];
+  for (const rec of records) {
+    if (spec.recordKeys.length && rec.keys[0] !== spec.recordKeys[0]) {
+      at(rec.line, `a "${key}:" record opens with "- ${spec.recordKeys[0]}: <value>"`);
+    }
+    for (const want of spec.recordKeys) {
+      if (!rec.keys.includes(want)) at(rec.line, `a "${key}:" record is missing "${want}:"`);
+    }
+    for (const k of rec.keys) {
+      if (!allowed.includes(k)) at(rec.line, `"${k}" is not a "${key}:" key — the profile declares ${allowed.join(', ')}`);
+      else issues.push(...vocabularyIssues(spec, key, k, rec.values[k], rec.line));
+    }
+  }
+  return issues;
+}
+
+/** The keys a record may carry, required ones first, which is the order a record writes them in. */
+const allowedKeys = (spec) => [...spec.recordKeys, ...spec.optionalRecordKeys];
+
+function recordLines(spec, rec, indent) {
+  const allowed = allowedKeys(spec);
+  for (const k of Object.keys(rec)) {
+    if (!allowed.includes(k)) throw new Error(`"${k}" is not a "${spec.key}:" key — the profile declares ${allowed.join(', ') || 'none'}`);
+  }
+  for (const want of spec.recordKeys) {
+    if (rec[want] === undefined) throw new Error(`a "${spec.key}:" record is missing "${want}:"`);
+  }
+  return allowed.filter((k) => rec[k] !== undefined).map((k) => `${indent}${k}: ${rec[k]}`);
+}
+
+function sectionLines(spec, value) {
+  const { key, form } = spec;
+  const wrong = (want) => { throw new Error(`"${key}:" is a ${form}, so its value is ${want}`); };
+  if (form === 'scalar') {
+    if (typeof value !== 'string') wrong('a string');
+    return [`${key}: ${value}`];
+  }
+  if (form === 'bullet-list' || form === 'numbered-list') {
+    if (!Array.isArray(value) || value.some((v) => typeof v !== 'string')) wrong('an array of strings');
+    return [`${key}:`, ...value.map((v, i) => `${form === 'numbered-list' ? `${i + 1}. ` : '- '}${v}`)];
+  }
+  if (form === 'keyed-block') {
+    if (!value || typeof value !== 'object' || Array.isArray(value)) wrong('an object');
+    return [`${key}:`, ...recordLines(spec, value, '  ')];
+  }
+  if (!Array.isArray(value) || value.some((v) => !v || typeof v !== 'object' || Array.isArray(v))) wrong('an array of objects');
+  const records = value.map((rec) => {
+    const [first, ...rest] = recordLines(spec, rec, '  ');
+    return [`- ${first.trim()}`, ...rest];
+  });
+  // §3: a blank line separates records that run to 2 lines or more, where the only other boundary
+  // is a `- ` a reader has to scan for. A list of one-line records stays tight.
+  const spaced = records.length > 1 && records.every((r) => r.length > 1);
+  return [`${key}:`, ...records.flatMap((r, i) => (spaced && i > 0 ? ['', ...r] : r))];
+}
+
+/**
+ * A body written from data, in the order and the forms the profile declares. The profile owns
+ * the shape, so this branches on no document kind: hand it a docset index or an ADR and the
+ * difference is which profile it read.
+ *
+ * A block the data does not carry is left out rather than filled with a placeholder. A skeleton
+ * that conformed by inventing content would claim a completeness nobody wrote, and `cnlp check`
+ * is then the list of what the author still owes — which is the same honesty `status: source`
+ * gives a catalog record with no page yet.
+ */
+export function renderBody(data, profile) {
+  const declared = new Set(profile.sections.map((s) => s.key));
+  for (const key of Object.keys(data)) {
+    if (declared.has(key) || profile.frontmatterFields.includes(key) || profile.custom.includes(key)) continue;
+    throw new Error(`"${key}" is not a block this profile declares — it declares ${[...declared].join(', ')}`);
+  }
+  const out = [];
+  const front = profile.frontmatterFields.filter((f) => data[f] !== undefined);
+  if (front.length) out.push('---', ...front.map((f) => `${f}: ${data[f]}`), '---', '');
+  let under = '';
+  for (const spec of profile.sections) {
+    if (data[spec.key] === undefined) continue;
+    if (spec.heading && spec.heading !== under) { out.push(spec.heading, ''); under = spec.heading; }
+    out.push(...sectionLines(spec, data[spec.key]), '');
+  }
+  return `${out.join('\n').trimEnd()}\n`;
+}
+
+/**
+ * What moved between a document and the data a fresh read of its sources produced. A record is
+ * identified by the first key the profile declares for its block, so this branches on no document
+ * kind: the profile names what identifies a record, and the data names which keys are worth
+ * comparing — a key the data leaves out is a key this does not police.
+ *
+ * A record the data carries a key for and the document does not is reported rather than skipped.
+ * A comparison covering fewer records than it claims is the failure the comparison exists to catch,
+ * which is the same shape as a catalog whose absent value reads as a present one.
+ */
+export function recordDrift(raw, data, profile) {
+  const { lines, sections } = parseBlocks(raw);
+  const out = [];
+  const at = (block, id, change) => out.push({ block, id, change });
+  for (const spec of profile.sections) {
+    if (spec.form !== 'record-list' || data[spec.key] === undefined) continue;
+    const idKey = spec.recordKeys[0];
+    if (!idKey) continue;
+    const block = sections.find((s) => s.key === spec.key);
+    const have = new Map();
+    if (block) {
+      for (const rec of readRecords(itemsOf(lines, block.line)).records) have.set(rec.values[idKey], rec.values);
+    }
+    const want = new Map(data[spec.key].map((rec) => [String(rec[idKey]), rec]));
+    for (const [id, rec] of want) {
+      const got = have.get(id);
+      if (!got) { at(spec.key, id, 'added — the document has no record for it'); continue; }
+      for (const key of Object.keys(rec)) {
+        if (key === idKey) continue;
+        if (got[key] === undefined) at(spec.key, id, `carries no "${key}:", so a change to it goes unreported`);
+        else if (String(got[key]) !== String(rec[key])) at(spec.key, id, `"${key}:" moved`);
+      }
+    }
+    for (const id of have.keys()) if (!want.has(id)) at(spec.key, id, 'removed — the source states it no longer');
+  }
+  return out;
+}
+
+/** §9: the 3 keys that carry a reference. A name anywhere else is a mention, not a reference. */
+const REFERENCE = /^\s*-?\s*(page|by|docset):\s*(\S+)\s*$/;
+
+/**
+ * §9 references a document makes, each with the block it sits in, so a caller can hold a
+ * profile's own meaning for one — an `overrides:` `page:` is not a `see_also:` `page:`. Fenced
+ * regions are already blank here, so a `page:` line inside an example is not a reference.
+ */
+export function pageRefs(raw) {
+  const { lines, sections } = parseBlocks(raw);
+  const out = [];
+  for (const { key, line } of sections) {
+    for (const item of itemsOf(lines, line)) {
+      const ref = item.text.match(REFERENCE);
+      if (ref) out.push({ block: key, key: ref[1], target: ref[2], line: item.line });
+    }
+  }
+  return out;
+}
+
+/** §5: quoted or backticked text is data, not a claim, so the lexicon does not police it. */
+const unquoted = (l) => l.replace(/`[^`]*`|"[^"]*"|'[^']*'/g, ' ');
+
+/** Delimiters that open and close on the same line. A backtick is its own closer. */
+const UNCLOSED = [
+  ['backtick', /`/g, null],
+  ['parenthesis', /\(/g, /\)/g],
+  ['bracket', /\[/g, /\]/g],
+];
+
+/**
+ * §5 lexicon. One rule set for every profile, except the active voice: `mood` decides whether the
+ * document has an actor to name, so a caller reaching this directly passes the profile's mood or
+ * gets every rule that holds whatever the register is.
+ */
+export function lexiconIssues(lines, mood = '') {
+  const issues = [];
+  lines.forEach((l, i) => {
+    const line = i + 1;
+    const text = unquoted(l);
+    // Every line, not only a bullet: a scalar or a record sub-key holds one idea too.
+    if (l.length > HARD_LIMIT) {
+      issues.push({ line, message: `${l.length} chars exceeds the ${HARD_LIMIT} hard limit — it is holding more than one idea` });
+    }
+    if (/^- never\b/.test(l)) issues.push({ line, message: 'a prohibition opens with "do not", not "never"' });
+    // A converted document is where these 2 turn up: a mechanical split reads an ordinal as a
+    // sentence and leaves `- 4` behind, and cutting a line to the hard limit lands mid-span.
+    const item = l.match(/^\s*(?:[-*]|\d+\.)\s+(.*)$/);
+    if (item && !/[a-z]/i.test(item[1])) issues.push({ line, message: 'the item carries no word — it states no claim' });
+    for (const [name, open, close] of UNCLOSED) {
+      const n = (l.match(open) || []).length;
+      if (close ? n !== (l.match(close) || []).length : n % 2) {
+        issues.push({ line, message: `unclosed ${name} — 1 idea per line and no wrapping, so this line was cut` });
+      }
+    }
+    const deny = text.match(DENY);
+    if (deny) issues.push({ line, message: `deny-list word "${deny[0]}"` });
+    const spelled = text.match(SPELLED_THRESHOLD);
+    if (spelled) issues.push({ line, message: `"${spelled[0]}" — a threshold is a digit` });
+    const limit = text.match(AMBIGUOUS_LIMIT);
+    if (limit) issues.push({ line, message: `"${limit[0]}" leaves its bound open — state the limit as "<subject> <= <value> <unit>"` });
+    const comp = text.match(COMPARATIVE);
+    if (comp) issues.push({ line, message: `unquantified comparative "${comp[0]}" — name the property and its bound` });
+    // Not the corpus this ships with, which is clean: like the unclosed-span checks above, this is
+    // what a document arriving from prose carries, where the actor was never on the line.
+    const nom = text.match(NOMINALIZATION);
+    if (nom) issues.push({ line, message: `"${nom[0]}" hides a verb in a noun — write the verb` });
+    if (mood === 'imperative') {
+      const passive = text.match(PASSIVE);
+      if (passive && !STATE_PARTICIPLE.test(passive[1])) {
+        issues.push({ line, message: `"${passive[0]}" is passive and names no actor — an imperative document says who acts` });
+      }
+    }
+  });
+  return issues;
+}
+
+export { FORMS };