@maestroagora/agora 1.2.1 → 1.3.0

package/scripts/voice/gates.mjs

@@ -0,0 +1,244 @@
+// Corpus admission.
+//
+// Word counts come from authorship-attribution research on literary and
+// journalistic corpora, not from a controlled study of generative voice
+// synthesis. Every count below is a floor rather than a certificate, and the
+// four structural rules on top of it are governance defaults chosen so the
+// build has something checkable to enforce.
+
+import { coreFeatureVector, measure } from "./features.mjs";
+import { round } from "./pipeline.mjs";
+
+export const CERTIFICATION_FLOOR = 5000;
+export const PRODUCTION_MINIMUM = 10000;
+export const PREFERRED_TIER = 20000;
+export const MIN_DOCUMENTS = 10;
+export const MAX_SINGLE_DOCUMENT_SHARE = 0.25;
+export const REGISTER_MIN_WORDS = 2500;
+export const REGISTER_MIN_DOCUMENTS = 3;
+export const MAX_REGISTER_VARIANCE_SHARE = 0.3;
+export const MAX_LEAVE_ONE_OUT_SHIFT = 0.2;
+export const HETEROGENEITY_FAILURE_SHARE = 1 / 3;
+// The register-variance rule asks how much of a feature's spread register
+// explains. When a feature barely moves across the whole corpus there is no
+// meaningful spread for register to explain, and the ratio becomes unstable:
+// an arbitrarily small between-register difference divides an arbitrarily small
+// total and reports a high share. A feature whose relative spread is below this
+// value is treated as stable without evaluating the share. Governance default.
+export const NEGLIGIBLE_RELATIVE_SPREAD = 0.05;
+
+/** The confidence tier the clean word count permits. */
+export function admissionTier(cleanWords) {
+  if (cleanWords < CERTIFICATION_FLOOR) {
+    return {
+      certified: false,
+      confidence: null,
+      disposition: "refused",
+      reason: `${cleanWords} clean words is below the ${CERTIFICATION_FLOOR}-word certification floor. Below the stable region, stylometric estimates are unreliable rather than merely noisy, so no profile is certified.`,
+    };
+  }
+  if (cleanWords < PRODUCTION_MINIMUM) {
+    return {
+      certified: true,
+      confidence: "low",
+      disposition: "built with restricted features",
+      reason: `${cleanWords} clean words sits between the ${CERTIFICATION_FLOOR}-word floor and the ${PRODUCTION_MINIMUM}-word production minimum. Only features that stay stable on short texts are reported.`,
+    };
+  }
+  if (cleanWords < PREFERRED_TIER) {
+    return {
+      certified: true,
+      confidence: "medium",
+      disposition: "production minimum met",
+      reason: `${cleanWords} clean words meets the ${PRODUCTION_MINIMUM}-word production minimum for a persistent profile.`,
+    };
+  }
+  return {
+    certified: true,
+    confidence: "production",
+    disposition: "preferred tier",
+    reason: `${cleanWords} clean words reaches the ${PREFERRED_TIER}-word preferred tier, which is a governance default chosen as an engineering target and not an empirical threshold.`,
+  };
+}
+
+function documentIndependence(documents, cleanWords) {
+  const largest = documents.reduce(
+    (winner, document) => (document.clean_words > (winner?.clean_words ?? -1) ? document : winner),
+    null,
+  );
+  const share = largest && cleanWords ? largest.clean_words / cleanWords : 0;
+  const failures = [];
+  if (documents.length < MIN_DOCUMENTS) {
+    failures.push(
+      `only ${documents.length} independently composed documents; the governance default is at least ${MIN_DOCUMENTS}, because cross-document stability is the thing being measured`,
+    );
+  }
+  if (share > MAX_SINGLE_DOCUMENT_SHARE) {
+    failures.push(
+      `${largest.source} supplies ${round(share * 100, 1)} percent of the clean tokens; the governance default caps any single document at ${MAX_SINGLE_DOCUMENT_SHARE * 100} percent`,
+    );
+  }
+  return {
+    documents: documents.length,
+    largest_document_share_percent: round(share * 100, 1),
+    passed: failures.length === 0,
+    failures,
+  };
+}
+
+function registerSubprofiles(documents) {
+  const groups = new Map();
+  for (const document of documents) {
+    const register = document.register || "unlabelled";
+    if (!groups.has(register)) groups.set(register, []);
+    groups.get(register).push(document);
+  }
+  return [...groups.entries()]
+    .sort((left, right) => left[0].localeCompare(right[0]))
+    .map(([register, members]) => {
+      const cleanWords = members.reduce((total, member) => total + member.clean_words, 0);
+      const qualifies = cleanWords >= REGISTER_MIN_WORDS && members.length >= REGISTER_MIN_DOCUMENTS;
+      return {
+        register,
+        documents: members.length,
+        clean_words: cleanWords,
+        numeric: qualifies,
+        note: qualifies
+          ? null
+          : `below the governance default of ${REGISTER_MIN_WORDS} clean words across ${REGISTER_MIN_DOCUMENTS} independent documents; observations here stay qualitative and every number is provisional`,
+        measured: qualifies ? measure(members.map((member) => member.text).join("\n\n")) : null,
+      };
+    });
+}
+
+function variance(values) {
+  if (values.length < 2) return 0;
+  const average = values.reduce((total, value) => total + value, 0) / values.length;
+  return values.reduce((total, value) => total + (value - average) ** 2, 0) / (values.length - 1);
+}
+
+/**
+ * A candidate global feature fails persistence when register accounts for more
+ * than 30 percent of its document-level variance, or when deleting one document
+ * moves its pooled estimate by more than 20 percent. Both are governance
+ * defaults. A failing feature moves into a register override or is dropped.
+ */
+function featureStability(documents, pooled) {
+  const perDocument = documents.map((document) => ({
+    register: document.register || "unlabelled",
+    vector: coreFeatureVector(measure(document.text)),
+    words: document.clean_words,
+  }));
+  const pooledVector = coreFeatureVector(pooled);
+  const results = [];
+
+  for (const [name, pooledValue] of Object.entries(pooledVector)) {
+    if (pooledValue === null) continue;
+    const values = perDocument.map((entry) => entry.vector[name]).filter((value) => value !== null);
+    if (values.length < 2) {
+      results.push({ feature: name, stable: false, reason: "too few documents supplied this feature" });
+      continue;
+    }
+
+    const total = variance(values);
+    const registers = new Map();
+    for (const entry of perDocument) {
+      if (entry.vector[name] === null) continue;
+      if (!registers.has(entry.register)) registers.set(entry.register, []);
+      registers.get(entry.register).push(entry.vector[name]);
+    }
+    let within = 0;
+    let weight = 0;
+    for (const group of registers.values()) {
+      if (group.length < 2) continue;
+      within += variance(group) * (group.length - 1);
+      weight += group.length - 1;
+    }
+    const withinAverage = weight === 0 ? total : within / weight;
+    const registerShare = total === 0 ? 0 : Math.max(0, (total - withinAverage) / total);
+    const relativeSpread = pooledValue === 0 ? 0 : Math.sqrt(total) / Math.abs(pooledValue);
+    const negligibleSpread = relativeSpread < NEGLIGIBLE_RELATIVE_SPREAD;
+
+    let worstShift = 0;
+    for (let index = 0; index < values.length; index += 1) {
+      const withoutOne = values.filter((_, position) => position !== index);
+      const average = withoutOne.reduce((sum, value) => sum + value, 0) / withoutOne.length;
+      const shift = pooledValue === 0 ? 0 : Math.abs(average - pooledValue) / Math.abs(pooledValue);
+      worstShift = Math.max(worstShift, shift);
+    }
+
+    const reasons = [];
+    if (registers.size > 1 && !negligibleSpread && registerShare > MAX_REGISTER_VARIANCE_SHARE) {
+      reasons.push(
+        `register accounts for ${round(registerShare * 100, 1)} percent of document-level variance, above the ${MAX_REGISTER_VARIANCE_SHARE * 100} percent governance default`,
+      );
+    }
+    if (worstShift > MAX_LEAVE_ONE_OUT_SHIFT) {
+      reasons.push(
+        `deleting one document moves the pooled estimate by ${round(worstShift * 100, 1)} percent, above the ${MAX_LEAVE_ONE_OUT_SHIFT * 100} percent governance default`,
+      );
+    }
+    results.push({
+      feature: name,
+      stable: reasons.length === 0,
+      register_variance_share: negligibleSpread ? null : round(registerShare, 3),
+      relative_spread: round(relativeSpread, 4),
+      worst_leave_one_out_shift: round(worstShift, 3),
+      negligible_spread: negligibleSpread,
+      reason: reasons.join("; ") || null,
+    });
+  }
+  return results;
+}
+
+/**
+ * Mark the corpus not profileable as one voice when more than a third of the
+ * proposed core features fail the stability rule, or when the clean corpus
+ * falls below the certification floor after exclusions. Offer to build two
+ * profiles rather than averaging two registers into a voice that belongs to
+ * nobody. Governance default.
+ */
+function heterogeneityStop(stability, tier, registers) {
+  const evaluated = stability.length;
+  const failed = stability.filter((entry) => !entry.stable).length;
+  const share = evaluated === 0 ? 1 : failed / evaluated;
+  const reasons = [];
+  if (evaluated > 0 && share > HETEROGENEITY_FAILURE_SHARE) {
+    reasons.push(
+      `${failed} of ${evaluated} core features fail the stability rule (${round(share * 100, 1)} percent, above the one-third governance default)`,
+    );
+  }
+  if (!tier.certified) reasons.push("the clean corpus falls below the certification floor after exclusions");
+  const namedRegisters = registers.filter((entry) => entry.register !== "unlabelled");
+  return {
+    stopped: reasons.length > 0,
+    failed_features: failed,
+    evaluated_features: evaluated,
+    reasons,
+    remedy:
+      reasons.length > 0 && namedRegisters.length > 1
+        ? `Build one profile per register (${namedRegisters.map((entry) => entry.register).join(", ")}) rather than averaging them into a voice that belongs to nobody.`
+        : reasons.length > 0
+          ? "Supply more independent documents, or label registers with --register so two profiles can be built instead of one average."
+          : null,
+  };
+}
+
+/** Run every admission gate over a cleaned corpus. */
+export function runGates(documents, pooled) {
+  const cleanWords = documents.reduce((total, document) => total + document.clean_words, 0);
+  const tier = admissionTier(cleanWords);
+  const independence = documentIndependence(documents, cleanWords);
+  const registers = registerSubprofiles(documents);
+  const stability = tier.certified ? featureStability(documents, pooled) : [];
+  const heterogeneity = heterogeneityStop(stability, tier, registers);
+  return {
+    clean_words: cleanWords,
+    tier,
+    independence,
+    registers,
+    stability,
+    heterogeneity,
+    certified: tier.certified && !heterogeneity.stopped,
+  };
+}

package/scripts/voice/ingest.mjs

@@ -0,0 +1,226 @@
+// Corpus ingestion and cleaning.
+//
+// Every threshold in the admission rules counts clean author-controlled words:
+// the author's own prose after quotations, forwarded text, copied source
+// material, boilerplate, templates, legal disclaimers, automatic signatures,
+// and house-written headlines are removed. Cleaning therefore runs before any
+// counting, and each removal is recorded so the profile can report what left.
+
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename, dirname, extname, join, relative, resolve } from "node:path";
+
+import { countTypography, normalize, tokenize } from "./pipeline.mjs";
+
+export const TEXT_EXTENSIONS = new Set([".md", ".markdown", ".txt", ".text", ".html", ".htm"]);
+export const REFUSED_EXTENSIONS = new Set([".docx", ".doc", ".pdf", ".rtf", ".odt", ".pages", ".epub"]);
+const MAX_FETCH_BYTES = 5_000_000;
+const FETCH_TIMEOUT_MS = 20_000;
+
+/**
+ * Binary document formats are refused rather than parsed. Extracting prose from
+ * them needs a dependency this package does not ship, and a partial extraction
+ * would silently change the word counts every admission threshold depends on.
+ */
+export function refusalMessage(source, extension) {
+  return `${source}: ${extension} is not supported. Export it to Markdown, plain text, or HTML and rerun. Voice thresholds count clean words, and a partial extraction from a binary format would move every one of them without saying so.`;
+}
+
+function stripHtml(html) {
+  const withoutHead = html
+    .replace(/<!--[\s\S]*?-->/g, " ")
+    .replace(/<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?<\/\1>/gi, " ")
+    .replace(/<(nav|header|footer|aside|form)\b[^>]*>[\s\S]*?<\/\1>/gi, " ");
+  const blockquotes = (withoutHead.match(/<blockquote\b/gi) || []).length;
+  const headings = (withoutHead.match(/<h[1-6]\b/gi) || []).length;
+  const body = withoutHead
+    .replace(/<blockquote\b[^>]*>[\s\S]*?<\/blockquote>/gi, " ")
+    .replace(/<h[1-6]\b[^>]*>[\s\S]*?<\/h[1-6]>/gi, " ")
+    .replace(/<\/(p|div|li|tr|section|article|br)>/gi, "\n\n")
+    .replace(/<br\s*\/?>/gi, "\n\n")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/gi, " ")
+    .replace(/&amp;/gi, "&")
+    .replace(/&lt;/gi, "<")
+    .replace(/&gt;/gi, ">")
+    .replace(/&quot;/gi, '"')
+    .replace(/&#39;|&apos;/gi, "'");
+  return { text: body, removed: { blockquotes, headings } };
+}
+
+function stripMarkdown(markdown, options) {
+  const removed = { frontmatter: 0, code_blocks: 0, blockquotes: 0, headings: 0, tables: 0, signatures: 0 };
+  let text = markdown;
+
+  text = text.replace(/^---\n[\s\S]*?\n---\n/, () => {
+    removed.frontmatter += 1;
+    return "";
+  });
+  text = text.replace(/^```[\s\S]*?^```$/gm, () => {
+    removed.code_blocks += 1;
+    return "\n\n";
+  });
+  text = text.replace(/^(?: {4}|\t).*$/gm, "");
+  text = text.replace(/`[^`\n]+`/g, " ");
+
+  const lines = text.split("\n");
+  const kept = [];
+  for (const line of lines) {
+    if (/^\s*>/.test(line)) {
+      removed.blockquotes += 1;
+      continue;
+    }
+    if (/^\s{0,3}#{1,6}\s/.test(line)) {
+      removed.headings += 1;
+      if (!options.keepHeadings) continue;
+    }
+    if (/^\s*\|.*\|\s*$/.test(line) || /^\s*\|?[\s:-]*-{3,}[\s:|-]*$/.test(line)) {
+      removed.tables += 1;
+      continue;
+    }
+    if (/^\s*(--\s*$|Sent from my |Get Outlook for )/.test(line)) {
+      removed.signatures += 1;
+      continue;
+    }
+    kept.push(line);
+  }
+
+  return {
+    text: kept
+      .join("\n")
+      .replace(/!\[[^\]]*\]\([^)]*\)/g, " ")
+      .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1")
+      .replace(/^\s*[-*+]\s+/gm, "")
+      .replace(/^\s*\d+\.\s+/gm, "")
+      .replace(/[*_]{1,3}([^*_]+)[*_]{1,3}/g, "$1")
+      .replace(/^\s*(?:[-*_]\s*){3,}$/gm, ""),
+    removed,
+  };
+}
+
+/** Clean one document and count what the cleaning removed. */
+export function cleanDocument(raw, { source, format, keepHeadings = false }) {
+  const typography = countTypography(raw);
+  const rawWords = tokenize(raw).length;
+  const html = format === "html";
+  const stage = html ? stripHtml(raw) : stripMarkdown(raw, { keepHeadings });
+  const cleanText = normalize(stage.text).replace(/\n{3,}/g, "\n\n").trim();
+  const tokens = tokenize(cleanText);
+  return {
+    source,
+    format,
+    text: cleanText,
+    tokens,
+    clean_words: tokens.length,
+    raw_words: rawWords,
+    removed: stage.removed,
+    typography,
+  };
+}
+
+function formatFor(name) {
+  const extension = extname(name).toLowerCase();
+  if (extension === ".html" || extension === ".htm") return "html";
+  return "markdown";
+}
+
+/**
+ * A short, stable label for a document. Relative to the working directory when
+ * the file sits underneath it, and the parent folder plus the file name when it
+ * does not, so a profile never carries a long absolute path from the machine
+ * that built it.
+ */
+function labelFor(path, cwd) {
+  const relativePath = relative(cwd, path).replaceAll("\\", "/");
+  if (relativePath !== "" && !relativePath.startsWith("..")) return relativePath;
+  return `${basename(dirname(path))}/${basename(path)}`;
+}
+
+async function walk(directory) {
+  const found = [];
+  for (const entry of (await readdir(directory, { withFileTypes: true })).sort((left, right) =>
+    left.name.localeCompare(right.name),
+  )) {
+    if (entry.name.startsWith(".")) continue;
+    const path = join(directory, entry.name);
+    if (entry.isDirectory()) found.push(...(await walk(path)));
+    else if (entry.isFile()) found.push(path);
+  }
+  return found;
+}
+
+async function fetchDocument(url) {
+  const parsed = new URL(url);
+  if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
+    throw new Error(`${url}: only http and https URLs can be fetched`);
+  }
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const response = await fetch(url, { redirect: "follow", signal: controller.signal });
+    if (!response.ok) throw new Error(`${url}: fetch returned ${response.status}`);
+    const type = (response.headers.get("content-type") || "").toLowerCase();
+    if (type.includes("pdf") || type.includes("officedocument") || type.includes("msword")) {
+      throw new Error(refusalMessage(url, type.split(";")[0]));
+    }
+    const body = await response.text();
+    if (body.length > MAX_FETCH_BYTES) {
+      throw new Error(`${url}: response exceeds the ${MAX_FETCH_BYTES} byte ceiling`);
+    }
+    return { raw: body, format: type.includes("text/html") ? "html" : "markdown" };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+/**
+ * Resolve every --from argument into cleaned documents. Directories expand to
+ * their readable text files; unsupported formats are refused by name so the
+ * user learns which document was skipped and why.
+ */
+export async function collectCorpus(sources, { keepHeadings = false, cwd = process.cwd() } = {}) {
+  const documents = [];
+  const refused = [];
+  const targets = [];
+
+  for (const source of sources) {
+    if (/^https?:\/\//i.test(source)) {
+      targets.push({ kind: "url", value: source, label: source });
+      continue;
+    }
+    const path = resolve(cwd, source);
+    const info = await stat(path).catch(() => null);
+    if (!info) throw new Error(`${source}: no such file or directory`);
+    if (info.isDirectory()) {
+      for (const file of await walk(path)) {
+        targets.push({ kind: "file", value: file, label: labelFor(file, cwd) });
+      }
+    } else {
+      targets.push({ kind: "file", value: path, label: labelFor(path, cwd) });
+    }
+  }
+
+  for (const target of targets) {
+    if (target.kind === "file") {
+      const extension = extname(target.value).toLowerCase();
+      if (REFUSED_EXTENSIONS.has(extension)) {
+        refused.push(refusalMessage(target.label, extension));
+        continue;
+      }
+      if (!TEXT_EXTENSIONS.has(extension)) continue;
+      const raw = await readFile(target.value, "utf8");
+      documents.push(cleanDocument(raw, { source: target.label, format: formatFor(target.value), keepHeadings }));
+      continue;
+    }
+    try {
+      const fetched = await fetchDocument(target.value);
+      documents.push(
+        cleanDocument(fetched.raw, { source: target.label, format: fetched.format, keepHeadings }),
+      );
+    } catch (error) {
+      refused.push(error.message);
+    }
+  }
+
+  documents.sort((left, right) => left.source.localeCompare(right.source));
+  return { documents, refused };
+}

package/scripts/voice/lexicon.mjs

@@ -0,0 +1,162 @@
+// Frozen word lists for the agora-voice measurement pipeline.
+//
+// Every list here is part of the pipeline contract. Changing a list changes the
+// numbers a profile reports, so a change must also raise LEXICON_VERSION in
+// pipeline.mjs. A profile built under one lexicon version is not comparable to
+// a draft measured under another.
+
+export const ABBREVIATIONS = new Set([
+  "a.m", "p.m", "approx", "apr", "aug", "ave", "b.c", "a.d", "c.f", "capt",
+  "co", "corp", "dec", "dept", "dr", "e.g", "est", "etc", "feb", "fig", "fri",
+  "gen", "gov", "i.e", "inc", "jan", "jr", "jul", "jun", "lt", "ltd", "mar",
+  "messrs", "mon", "mr", "mrs", "ms", "mt", "no", "nov", "oct", "p.s", "ph.d",
+  "pp", "prof", "rev", "sat", "sep", "sept", "sgt", "sr", "st", "sun", "thu",
+  "tue", "u.k", "u.s", "u.s.a", "v.s", "vol", "vs", "wed",
+]);
+
+// Function words, grouped by class. Rates are reported per thousand tokens.
+export const FUNCTION_WORDS = {
+  articles: ["a", "an", "the"],
+  prepositions: [
+    "about", "above", "across", "after", "against", "along", "among", "around",
+    "at", "before", "behind", "below", "beneath", "beside", "between", "beyond",
+    "by", "despite", "down", "during", "except", "for", "from", "in", "inside",
+    "into", "near", "of", "off", "on", "onto", "outside", "over", "past",
+    "since", "through", "throughout", "to", "toward", "towards", "under",
+    "until", "up", "upon", "with", "within", "without",
+  ],
+  auxiliaries: [
+    "am", "are", "be", "been", "being", "can", "could", "did", "do", "does",
+    "had", "has", "have", "is", "may", "might", "must", "shall", "should",
+    "was", "were", "will", "would",
+  ],
+  conjunctions: [
+    "after", "although", "and", "as", "because", "before", "but", "either",
+    "for", "if", "neither", "nor", "once", "or", "since", "so", "than", "that",
+    "though", "unless", "until", "when", "whenever", "where", "whereas",
+    "wherever", "whether", "while", "yet",
+  ],
+  pronouns: [
+    "he", "her", "hers", "herself", "him", "himself", "his", "i", "it", "its",
+    "itself", "me", "mine", "my", "myself", "our", "ours", "ourselves", "she",
+    "their", "theirs", "them", "themselves", "they", "us", "we", "what",
+    "which", "who", "whom", "whose", "you", "your", "yours", "yourself",
+    "yourselves",
+  ],
+};
+
+export const PERSON_CLASSES = {
+  first_singular: ["i", "me", "my", "mine", "myself"],
+  first_plural: ["we", "us", "our", "ours", "ourselves"],
+  second: ["you", "your", "yours", "yourself", "yourselves"],
+  third_singular: ["he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself"],
+  third_plural: ["they", "them", "their", "theirs", "themselves"],
+};
+
+export const HEDGES = [
+  "apparently", "arguably", "broadly", "generally", "largely", "likely",
+  "mainly", "maybe", "mostly", "often", "partly", "perhaps", "possibly",
+  "presumably", "probably", "quite", "rather", "relatively", "roughly",
+  "seemingly", "somewhat", "sometimes", "suggests", "tends", "typically",
+  "usually",
+];
+
+export const BOOSTERS = [
+  "absolutely", "always", "certainly", "clearly", "completely", "definitely",
+  "entirely", "essential", "every", "exactly", "extremely", "highly",
+  "incredibly", "indeed", "never", "obviously", "particularly", "precisely",
+  "really", "significantly", "strongly", "surely", "totally", "truly",
+  "undoubtedly", "utterly", "very",
+];
+
+export const MODALS = [
+  "can", "could", "may", "might", "must", "ought", "shall", "should", "will",
+  "would",
+];
+
+// Sentence-opening classes, checked in the order the features module applies.
+export const OPENING_CLASSES = {
+  coordinator: ["and", "but", "or", "nor", "so", "yet", "for"],
+  subordinator: [
+    "after", "although", "as", "because", "before", "if", "once", "since",
+    "though", "unless", "until", "when", "whenever", "where", "whereas",
+    "wherever", "whether", "while",
+  ],
+  question_word: ["how", "what", "when", "where", "which", "who", "whom", "whose", "why"],
+  discourse_marker: [
+    "additionally", "also", "anyway", "besides", "consequently", "conversely",
+    "furthermore", "however", "importantly", "instead", "meanwhile", "moreover",
+    "nevertheless", "nonetheless", "notably", "otherwise", "similarly",
+    "still", "therefore", "thus", "ultimately",
+  ],
+  subject_pronoun: ["he", "i", "it", "she", "they", "we", "you"],
+  determiner: ["a", "an", "the", "this", "that", "these", "those", "each", "every", "no", "some", "any"],
+  expletive: ["there", "here"],
+  adverbial: [
+    "again", "almost", "already", "back", "even", "eventually", "everywhere",
+    "finally", "first", "immediately", "later", "lately", "now", "often",
+    "once", "only", "originally", "perhaps", "recently", "sometimes", "soon",
+    "suddenly", "then", "today", "tomorrow", "tonight", "usually", "yesterday",
+  ],
+  preposition: FUNCTION_WORDS.prepositions,
+};
+
+// Contractible pairs. The contraction rate counts contracted forms against the
+// eligible contexts where the expanded form appears, so the two columns are
+// measured on the same opportunity set rather than on raw frequency.
+export const CONTRACTION_PAIRS = [
+  { contracted: ["can't"], expanded: [["can", "not"], ["cannot"]] },
+  { contracted: ["don't"], expanded: [["do", "not"]] },
+  { contracted: ["doesn't"], expanded: [["does", "not"]] },
+  { contracted: ["didn't"], expanded: [["did", "not"]] },
+  { contracted: ["isn't"], expanded: [["is", "not"]] },
+  { contracted: ["aren't"], expanded: [["are", "not"]] },
+  { contracted: ["wasn't"], expanded: [["was", "not"]] },
+  { contracted: ["weren't"], expanded: [["were", "not"]] },
+  { contracted: ["won't"], expanded: [["will", "not"]] },
+  { contracted: ["wouldn't"], expanded: [["would", "not"]] },
+  { contracted: ["shouldn't"], expanded: [["should", "not"]] },
+  { contracted: ["couldn't"], expanded: [["could", "not"]] },
+  { contracted: ["haven't"], expanded: [["have", "not"]] },
+  { contracted: ["hasn't"], expanded: [["has", "not"]] },
+  { contracted: ["hadn't"], expanded: [["had", "not"]] },
+  { contracted: ["it's"], expanded: [["it", "is"], ["it", "has"]] },
+  { contracted: ["that's"], expanded: [["that", "is"]] },
+  { contracted: ["there's"], expanded: [["there", "is"], ["there", "has"]] },
+  { contracted: ["what's"], expanded: [["what", "is"]] },
+  { contracted: ["here's"], expanded: [["here", "is"]] },
+  { contracted: ["i'm"], expanded: [["i", "am"]] },
+  { contracted: ["i've"], expanded: [["i", "have"]] },
+  { contracted: ["i'll"], expanded: [["i", "will"]] },
+  { contracted: ["i'd"], expanded: [["i", "would"], ["i", "had"]] },
+  { contracted: ["you're"], expanded: [["you", "are"]] },
+  { contracted: ["you've"], expanded: [["you", "have"]] },
+  { contracted: ["you'll"], expanded: [["you", "will"]] },
+  { contracted: ["we're"], expanded: [["we", "are"]] },
+  { contracted: ["we've"], expanded: [["we", "have"]] },
+  { contracted: ["we'll"], expanded: [["we", "will"]] },
+  { contracted: ["they're"], expanded: [["they", "are"]] },
+  { contracted: ["they've"], expanded: [["they", "have"]] },
+  { contracted: ["they'll"], expanded: [["they", "will"]] },
+  { contracted: ["let's"], expanded: [["let", "us"]] },
+];
+
+// The generic AI-vocabulary list the tell gate bans. A profile's owned list can
+// suppress the ban for a measured word, and only for the words it measured.
+// Sourced from the vocabulary section of the canonical reference.
+export const GENERIC_AI_VOCABULARY = [
+  "bespoke", "bolster", "breathtaking", "comprehensive", "craft", "curated",
+  "cutting-edge", "delve", "elevate", "embark", "empower", "enhance",
+  "ecosystem", "essential", "facilitate", "forefront", "forge", "foster",
+  "game-changer", "groundbreaking", "harness", "holistic", "innovative",
+  "intricate", "invaluable", "journey", "landscape", "leverage", "meticulous",
+  "multifaceted", "navigate", "nuanced", "paramount", "pivotal", "powerhouse",
+  "profound", "realm", "revolutionary", "robust", "seamless", "showcase",
+  "spearhead", "state-of-the-art", "streamline", "tapestry", "testament",
+  "trailblazer", "transformative", "underscore", "unleash", "unlock",
+  "unparalleled", "unprecedented", "vibrant", "vital", "world-class",
+];
+
+export function classSet(words) {
+  return new Set(words.map((word) => word.toLowerCase()));
+}