chaffjs 0.18.0 → 0.20.0

package/src/structure/reference-text.ts

@@ -0,0 +1,114 @@
+import type { StructureNode } from "../plugin.ts";
+import type { StructureIssue } from "./issues.ts";
+
+/**
+ * 参照に添えた言葉と、参照先の中身。
+ * 名前: 参照のすぐ後ろの括弧か引用符の中（「第2章「料金」」「第5条（秘密保持）」"Section 2 (Pricing)"）。参照先の見出しと、どちらかが
+ * どちらかを含めば合う。合わなければ、番号か名前のどちらかが古い。
+ * 話題: 参照のすぐ後ろの「で述べた」「に定める」"for" に続く語（「第2章で述べた料金体系」"see Section 2 for pricing"）。
+ * 参照先の見出しにも本文にも無ければ言う。
+ * どちらも、参照先がちょうど一つに決まり、見出しを持つときだけ見る。
+ */
+export type ReferenceWords = {
+  /** 括弧の中に書いても名前ではない語（参照、上記、above）。 */
+  readonly asides: readonly string[];
+  /** 参照と話題のあいだの言い回し（で述べた、for）。 */
+  readonly topicLinks: readonly string[];
+  /** 話題にならない語（詳細、内容、details）。どの節にも当てはまる。 */
+  readonly vagueTopics: readonly string[];
+};
+
+export type ReferenceInput = {
+  readonly source: string;
+  readonly references: readonly StructureNode[];
+  readonly targetOf: (reference: StructureNode) => StructureNode | undefined;
+  readonly words: ReferenceWords;
+};
+
+const OPENERS: Readonly<Record<string, string>> = { "(": ")", "（": "）", "「": "」", "『": "』", '"': '"', "“": "”", "‘": "’" };
+const MAX_NAME = 30;
+const DIGIT = /\p{N}/u;
+const LETTER = /\p{L}/u;
+const SPACE_BEFORE = /^[ \t\u3000]?/u;
+
+const normalized = (text: string): string =>
+  text
+    .normalize("NFKC")
+    .toLowerCase()
+    .replace(/[\s・·]/gu, "");
+
+const headingOf = (node: StructureNode): string => String(node.attrs["heading"] ?? "");
+
+/** 参照のすぐ後ろの、括弧か引用符の中。 */
+const nameAfter = (source: string, end: number): { name: string; at: number } | undefined => {
+  const gap = SPACE_BEFORE.exec(source.slice(end))?.[0].length ?? 0;
+  const opener = source.charAt(end + gap);
+  const closer = OPENERS[opener];
+  if (closer === undefined) return undefined;
+  const from = end + gap + 1;
+  const close = source.slice(from, from + MAX_NAME + 1).indexOf(closer) + from;
+  if (close < from) return undefined;
+  return { name: source.slice(from, close).trim(), at: from };
+};
+
+/** 項や号の札（(a)、(ii)、（イ））。名前ではなく、参照の続き。 */
+const ITEM_LABEL = /^(?:[a-z]{1,2}|[ivxlcdm]{1,5}|.)$/iu;
+
+const isName = (name: string, words: ReferenceWords): boolean =>
+  LETTER.test(name) &&
+  !DIGIT.test(name) &&
+  !ITEM_LABEL.test(name.normalize("NFKC")) &&
+  !words.asides.some((aside) => normalized(name).includes(normalized(aside)));
+
+const agrees = (name: string, heading: string): boolean => {
+  const [a, b] = [normalized(name), normalized(heading)];
+  return a.includes(b) || b.includes(a);
+};
+
+/** 名前を添えた参照で、名前が参照先の見出しと合わない。 */
+export const titleMismatches = (input: ReferenceInput): StructureIssue[] =>
+  input.references.flatMap((reference) => {
+    const target = input.targetOf(reference);
+    const heading = target === undefined ? "" : headingOf(target);
+    const found = nameAfter(input.source, reference.span.end);
+    if (heading === "" || found === undefined || !isName(found.name, input.words) || agrees(found.name, heading)) return [];
+    return [{ offset: found.at, values: { label: String(reference.attrs["label"] ?? ""), name: found.name, heading } }];
+  });
+
+/** 話題の語: 漢字、片仮名、英字、数字の続き。日本語はここで切れる（「料金体系は」の は）。英語は三語まで。 */
+const JA_TOPIC = /^[\p{Script=Han}\p{Script=Katakana}ー々]{2,20}/u;
+const EN_TOPIC = /^(?:the |a |an )?([a-z][a-z-]{2,}(?: [a-z][a-z-]{2,}){0,2})/iu;
+const STEM = 5;
+const LATIN_TOPIC = /^[a-z -]+$/iu;
+
+const topicAfter = (source: string, end: number, links: readonly string[]): string | undefined => {
+  const after = source.slice(end, end + 60);
+  const link = links.find((candidate) => after.replace(SPACE_BEFORE, "").toLowerCase().startsWith(candidate.toLowerCase()));
+  if (link === undefined) return undefined;
+  const rest = after.replace(SPACE_BEFORE, "").slice(link.length).replace(SPACE_BEFORE, "");
+  return JA_TOPIC.exec(rest)?.[0] ?? EN_TOPIC.exec(rest)?.[1];
+};
+
+/** 話題が参照先にあるか。英語の語は、語の頭から五文字で探す（pricing と price は同じ語の形違い、metadata の data は別の語）。 */
+const mentions = (text: string, topic: string): boolean => {
+  const lower = text.normalize("NFKC").toLowerCase();
+  const words = topic.toLowerCase().split(" ");
+  if (!LATIN_TOPIC.test(topic)) return lower.includes(topic.normalize("NFKC").toLowerCase());
+  return words.every((word) => new RegExp(`(?<![a-z])${word.slice(0, STEM).replace(/[^a-z]/gu, "")}`, "u").test(lower));
+};
+
+/** 話題のどれかの語が、どの節にも当てはまる語（further caveats の further、料金の詳細の 詳細）。 */
+const isVague = (topic: string, words: ReferenceWords): boolean => {
+  const parts = topic.toLowerCase().split(" ");
+  return words.vagueTopics.some((word) => parts.includes(word.toLowerCase()) || normalized(topic).endsWith(normalized(word)));
+};
+
+/** 参照先の見出しにも本文にも無い話題。 */
+export const topicMisses = (input: ReferenceInput): StructureIssue[] =>
+  input.references.flatMap((reference) => {
+    const target = input.targetOf(reference);
+    const topic = topicAfter(input.source, reference.span.end, input.words.topicLinks);
+    if (target === undefined || headingOf(target) === "" || topic === undefined || isVague(topic, input.words)) return [];
+    if (mentions(input.source.slice(target.span.start, target.span.end), topic)) return [];
+    return [{ offset: reference.span.start, values: { label: String(reference.attrs["label"] ?? ""), topic, heading: headingOf(target) } }];
+  });

package/src/structure/universal.ts

@@ -5,7 +5,12 @@ import { numberInSentence } from "./number-in-sentence.ts";
  * 「4.2 設定」「3.1.4 Scope」のような通し番号。数字と点だけで書くので言語を問わない。
  * 各部分を 3 桁・深さを 6 までに抑え、後戻りが爆発しない形にしてある。
  */
-const DOTTED = /^[ \t]{0,3}(?<number>\d{1,3}(?:\.\d{1,3}){0,5})(?<closing>\.)?[ \t\u3000]+(?<rest>\S.*)$/u;
+const DOTTED = /^[ \t]{0,3}(?<number>\d{1,3}(?:\.\d{1,3}){0,5})(?<closing>[.．])?[ \t\u3000]+(?<rest>\S.*)$/u;
+/**
+ * 見出しで、点で閉じた番号に題を詰めて書いたもの（「7.委託」「7．委託」「2.Overview」）。後ろが数字なら小数（「1.5万人」）、
+ * 小文字なら版（「2.x 系」）なので読まない。
+ */
+const TIGHT_HEADING = /^[ \t]{0,3}(?<number>\d{1,3}(?:\.\d{1,3}){0,5})(?<closing>[.．])(?<rest>[^\s\d\p{Ll}.．].*)$/u;
 
 /**
  * RFC のようなテキストの仕様書（Markdown でないもの）の章見出し「5.  Security Considerations」。Markdown では、行内のコードを伏せた空白が
@@ -18,6 +23,9 @@ const MAX_TITLE = 80;
 
 const isTopSection = (line: string, rest: string): boolean => TOP_SECTION.test(line) && rest.length <= MAX_TITLE && !SENTENCE_END.test(rest);
 
+const dottedGroups = (line: string, context: NumberingContext): Readonly<Record<string, string | undefined>> | undefined =>
+  DOTTED.exec(line)?.groups ?? (context.isHeading ? TIGHT_HEADING.exec(line)?.groups : undefined);
+
 /** closedByDot: 番号を点で閉じたか（「5. 」）。「1.5 万人」の 1.5 は閉じていない。 */
 type DottedLine = NumberedLine & { readonly closedByDot: boolean };
 
@@ -26,7 +34,7 @@ type DottedLine = NumberedLine & { readonly closedByDot: boolean };
  * 「4.2 」のように点を含むものは、.txt の仕様書でも見出しとして書かれるので本文でも読む。
  */
 export const dottedNumber = (line: string, context: NumberingContext, plainText = false): DottedLine | undefined => {
-  const groups = DOTTED.exec(line)?.groups;
+  const groups = dottedGroups(line, context);
   const number = groups?.["number"];
   if (number === undefined) return undefined;
   const depth = number.split(".").length;

package/src/structure-shape/baseline-load.ts

@@ -0,0 +1,18 @@
+import { readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parse } from "yaml";
+import { parseStructureBaseline, type StructureBaseline } from "./baseline.ts";
+
+const BASELINE_FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "structure-baseline.yaml");
+
+const readBaseline = (): StructureBaseline => {
+  try {
+    return parseStructureBaseline(parse(readFileSync(BASELINE_FILE, "utf8")));
+  } catch (error) {
+    throw new Error(`cannot read the structure baseline ${BASELINE_FILE}`, { cause: error });
+  }
+};
+
+/** The human baseline shipped with chaff (structure-baseline.yaml), by language. */
+export const STRUCTURE_BASELINE: StructureBaseline = readBaseline();

package/src/structure-shape/baseline.ts

@@ -0,0 +1,62 @@
+import { FEATURE_IDS, type FeatureId } from "./features.ts";
+
+/** The percentiles kept for each measure: every five from 5 to 95. */
+export const BASELINE_STEPS: readonly number[] = Array.from({ length: 19 }, (_, index) => (index + 1) * 5);
+
+const PERCENT = 100;
+
+/** The value at percentile `step` by nearest rank: the smallest value at least `step`% of the documents do not exceed. */
+const nearestRank = (sorted: readonly number[], step: number): number => sorted[Math.max(0, Math.ceil((step / PERCENT) * sorted.length) - 1)] ?? 0;
+
+/** A set of values as its percentiles at BASELINE_STEPS, keyed by step. Empty when there are no values. */
+export const percentilesOf = (values: readonly number[]): Record<number, number> => {
+  if (values.length === 0) return {};
+  const sorted = values.toSorted((left, right) => left - right);
+  return Object.fromEntries(BASELINE_STEPS.map((step) => [step, nearestRank(sorted, step)]));
+};
+
+/** One measure's human distribution: how many documents had a value, and its percentiles. */
+export type FeatureBaseline = { readonly measured: number; readonly percentiles: ReadonlyMap<number, number> };
+
+/** One language's baseline: how many articles were read, and each measure's distribution. */
+export type LanguageBaseline = { readonly articles: number; readonly features: Readonly<Partial<Record<FeatureId, FeatureBaseline>>> };
+
+export type StructureBaseline = Readonly<Record<string, LanguageBaseline>>;
+
+const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
+
+const isFiniteNumber = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value);
+
+const isCount = (value: unknown): value is number => isFiniteNumber(value) && value >= 0;
+
+/** Every step must be one of BASELINE_STEPS with a finite value, or the measure is not compared at all. */
+const percentilesFrom = (raw: unknown): ReadonlyMap<number, number> => {
+  const entries = Object.entries(isRecord(raw) ? raw : {});
+  const read = entries.flatMap(([step, value]) => (BASELINE_STEPS.includes(Number(step)) && isFiniteNumber(value) ? [[Number(step), value] as const] : []));
+  return read.length === entries.length ? new Map(read) : new Map();
+};
+
+const featureFrom = (raw: unknown): FeatureBaseline | undefined => {
+  if (!isRecord(raw) || !isCount(raw["measured"])) return undefined;
+  const percentiles = percentilesFrom(raw["percentiles"]);
+  return percentiles.size === 0 ? undefined : { measured: raw["measured"], percentiles };
+};
+
+const languageFrom = (raw: unknown): LanguageBaseline | undefined => {
+  if (!isRecord(raw) || !isCount(raw["articles"]) || !isRecord(raw["features"])) return undefined;
+  const features = raw["features"];
+  const read = FEATURE_IDS.flatMap((id) => {
+    const feature = featureFrom(features[id]);
+    return feature === undefined ? [] : [[id, feature] as const];
+  });
+  return { articles: raw["articles"], features: Object.fromEntries(read) };
+};
+
+/** The baseline file as data. A language or a measure that does not parse is left out, and is then reported as not compared. */
+export const parseStructureBaseline = (raw: unknown): StructureBaseline =>
+  Object.fromEntries(
+    Object.entries(isRecord(raw) ? raw : {}).flatMap(([language, value]) => {
+      const parsed = languageFrom(value);
+      return parsed === undefined ? [] : [[language, parsed] as const];
+    }),
+  );

package/src/structure-shape/features.ts

@@ -0,0 +1,189 @@
+import { lengthOf } from "../measure.ts";
+import type { Lexicon, ProseDocument, Section, Sentence } from "../plugin.ts";
+import { coefficientOfVariation } from "../detectors/structure.ts";
+import { boldLabelItems } from "../detectors/bold-label.ts";
+import { MIN_DOCUMENT_LENGTH } from "../detectors/signals.ts";
+import { bookendOf, headingFormOf, headingWords, proConPairs } from "./heading-words.ts";
+import { restatementPercent } from "./restatement.ts";
+
+/** The structure measures, in the order they are shown. */
+export const FEATURE_IDS = [
+  "heading-density",
+  "short-sections",
+  "section-uniformity",
+  "heading-forms",
+  "three-subsections",
+  "bookends",
+  "closing-restatement",
+  "three-item-lists",
+  "bold-labels",
+  "emoji-headings",
+  "pro-con",
+] as const;
+
+export type FeatureId = (typeof FEATURE_IDS)[number];
+
+/** Which way a value moves away from human writing: higher (more headings) or lower (more uniform sections). */
+export type Direction = "high" | "low";
+
+export const DIRECTION: Readonly<Record<FeatureId, Direction>> = {
+  "heading-density": "high",
+  "short-sections": "high",
+  "section-uniformity": "low",
+  "heading-forms": "high",
+  "three-subsections": "high",
+  bookends: "high",
+  "closing-restatement": "high",
+  "three-item-lists": "high",
+  "bold-labels": "high",
+  "emoji-headings": "high",
+  "pro-con": "high",
+};
+
+/** Why a measure has no value for this document. */
+export type NotMeasured = "too-short" | "too-few-sections" | "too-few-headings" | "no-closing" | "too-few-lists";
+
+/** One measure: its value, or why it has none, and what it found (the forms, the pairs) to show beside the number. */
+export type FeatureValue = {
+  readonly id: FeatureId;
+  readonly value: number | undefined;
+  readonly notMeasured?: NotMeasured;
+  readonly detail?: string;
+  /** The count behind a rate: the section headings behind the heading density. */
+  readonly count?: number;
+};
+
+const PER = 1000;
+const PERCENT = 100;
+const MIN_SECTIONS = 5;
+const MIN_SHORT_SECTIONS = 3;
+const MIN_HEADINGS = 4;
+const MIN_BOOKEND_HEADINGS = 2;
+const MIN_LISTS = 3;
+/** One or two paragraphs: a section that is a heading over a paragraph, the shape of a heading every paragraph or two. */
+const SHORT_SECTION_PARAGRAPHS = 2;
+const THREE = 3;
+
+const EMOJI = /\p{Extended_Pictographic}/u;
+
+const measured = (id: FeatureId, value: number, detail?: string): FeatureValue => ({
+  id,
+  value,
+  ...(detail === undefined || detail === "" ? {} : { detail }),
+});
+const notMeasured = (id: FeatureId, why: NotMeasured): FeatureValue => ({ id, value: undefined, notMeasured: why });
+
+const roundTo1 = (value: number): number => Math.round(value * 10) / 10;
+const percentOf = (part: number, whole: number): number => Math.round((part / whole) * PERCENT);
+
+const lengthOfAll = (sentences: readonly Sentence[], doc: ProseDocument): number =>
+  sentences.reduce((sum, sentence) => sum + lengthOf(sentence, doc.lengthUnit), 0);
+
+/** The sections under a heading, without the title: the first heading when it is a top-level one that comes before any other. */
+export const headedSections = (sections: readonly Section[]): Section[] => {
+  const headed = sections.filter((section) => section.depth > 0);
+  return headed[0]?.depth === 1 ? headed.slice(1) : headed;
+};
+
+const headingDensity = (doc: ProseDocument, headed: readonly Section[]): FeatureValue => {
+  const length = lengthOfAll(doc.sentences, doc);
+  if (length < MIN_DOCUMENT_LENGTH[doc.lengthUnit]) return notMeasured("heading-density", "too-short");
+  return { ...measured("heading-density", roundTo1((headed.length / length) * PER)), count: headed.length };
+};
+
+const paragraphsIn = (doc: ProseDocument, section: Section): number =>
+  doc.paragraphs.filter((paragraph) => paragraph.span.start >= section.span.start && paragraph.span.start < section.span.end).length;
+
+const shortSections = (doc: ProseDocument, headed: readonly Section[]): FeatureValue => {
+  const counts = headed.filter((section) => section.sentences.length > 0).map((section) => paragraphsIn(doc, section));
+  if (counts.length < MIN_SHORT_SECTIONS) return notMeasured("short-sections", "too-few-sections");
+  const short = counts.filter((count) => count <= SHORT_SECTION_PARAGRAPHS).length;
+  const mean = roundTo1(counts.reduce((sum, count) => sum + count, 0) / counts.length);
+  return measured("short-sections", percentOf(short, counts.length), String(mean));
+};
+
+const sectionUniformity = (doc: ProseDocument): FeatureValue => {
+  const lengths = doc.sections.map((section) => lengthOfAll(section.sentences, doc)).filter((length) => length > 0);
+  const cv = lengths.length < MIN_SECTIONS ? undefined : coefficientOfVariation(lengths);
+  return cv === undefined ? notMeasured("section-uniformity", "too-few-sections") : measured("section-uniformity", Math.round(cv * PERCENT));
+};
+
+const lexiconOf = (doc: ProseDocument, id: string): Lexicon => doc.lexicons[id] ?? [];
+
+/** The most common of the forms found, by its pattern: what the headings were cast in. */
+const mostCommon = (forms: readonly string[]): string | undefined => {
+  const counts = forms.reduce((tally, form) => tally.set(form, (tally.get(form) ?? 0) + 1), new Map<string, number>());
+  return [...counts.entries()].toSorted((left, right) => right[1] - left[1])[0]?.[0];
+};
+
+const headingForms = (doc: ProseDocument, words: readonly string[]): FeatureValue => {
+  const bookends = lexiconOf(doc, "bookend-heading");
+  const body = words.filter((heading) => bookendOf(heading, bookends) === undefined);
+  if (body.length < MIN_HEADINGS) return notMeasured("heading-forms", "too-few-headings");
+  const forms = body.flatMap((heading) => headingFormOf(heading, lexiconOf(doc, "heading-form"))?.pattern ?? []);
+  return measured("heading-forms", percentOf(forms.length, body.length), mostCommon(forms));
+};
+
+/** How many headings directly under each heading: the next deeper level, up to the next heading as high as itself. */
+export const childCounts = (headed: readonly Section[]): number[] =>
+  headed.map((section, index) => {
+    const after = headed.slice(index + 1);
+    const end = after.findIndex((other) => other.depth <= section.depth);
+    return (end < 0 ? after : after.slice(0, end)).filter((other) => other.depth === section.depth + 1).length;
+  });
+
+/** Headings split into exactly three subheadings: the outline's own rule of three (課題が 3 つ、対策が 3 つ). */
+const threeSubsections = (headed: readonly Section[]): FeatureValue =>
+  measured("three-subsections", childCounts(headed).filter((count) => count === THREE).length);
+
+const bookends = (doc: ProseDocument, words: readonly string[]): FeatureValue => {
+  if (words.length < MIN_BOOKEND_HEADINGS) return notMeasured("bookends", "too-few-headings");
+  const sides = new Set(words.map((heading) => bookendOf(heading, lexiconOf(doc, "bookend-heading"))).filter((side) => side !== undefined));
+  return measured("bookends", sides.size, [...sides].join(","));
+};
+
+const sentencesText = (sentences: readonly Sentence[]): string => sentences.map((sentence) => sentence.text).join("\n");
+
+/** The last closing bookend (まとめ, Conclusion) that has text, compared with everything written before it. */
+const closingRestatement = (doc: ProseDocument, headed: readonly Section[]): FeatureValue => {
+  const lexicon = lexiconOf(doc, "bookend-heading");
+  const closing = headed.findLast((section) => section.sentences.length > 0 && bookendOf(headingWords(section.heading), lexicon) === "after");
+  if (closing === undefined) return notMeasured("closing-restatement", "no-closing");
+  const body = doc.sentences.filter((sentence) => sentence.span.start < closing.span.start);
+  const percent = restatementPercent(sentencesText(closing.sentences), sentencesText(body), doc.lengthUnit);
+  return percent === undefined ? notMeasured("closing-restatement", "no-closing") : measured("closing-restatement", percent, closing.heading);
+};
+
+const threeItemLists = (doc: ProseDocument): FeatureValue => {
+  if (doc.lists.length < MIN_LISTS) return notMeasured("three-item-lists", "too-few-lists");
+  const threes = doc.lists.filter((list) => list.items.length === THREE).length;
+  return measured("three-item-lists", percentOf(threes, doc.lists.length), `${String(threes)}/${String(doc.lists.length)}`);
+};
+
+const emojiHeadings = (doc: ProseDocument): FeatureValue =>
+  measured("emoji-headings", doc.sections.filter((section) => section.depth > 0 && EMOJI.test(section.heading)).length);
+
+const proCon = (doc: ProseDocument, words: readonly string[]): FeatureValue => {
+  if (words.length < MIN_BOOKEND_HEADINGS) return notMeasured("pro-con", "too-few-headings");
+  const pairs = proConPairs(words, lexiconOf(doc, "pro-con-heading"));
+  return measured("pro-con", pairs.length, pairs.join(", "));
+};
+
+/** Every structure measure of one document, in FEATURE_IDS order. Reads only the document: no baseline, no judgement. */
+export const structureFeaturesOf = (doc: ProseDocument): FeatureValue[] => {
+  const headed = headedSections(doc.sections);
+  const words = headed.map((section) => headingWords(section.heading));
+  return [
+    headingDensity(doc, headed),
+    shortSections(doc, headed),
+    sectionUniformity(doc),
+    headingForms(doc, words),
+    threeSubsections(headed),
+    bookends(doc, words),
+    closingRestatement(doc, headed),
+    threeItemLists(doc),
+    measured("bold-labels", boldLabelItems(doc).length),
+    emojiHeadings(doc),
+    proCon(doc, words),
+  ];
+};

package/src/structure-shape/heading-words.ts

@@ -0,0 +1,71 @@
+import type { Lexicon, LexiconEntry } from "../plugin.ts";
+import { escapeRegExp } from "../orthography.ts";
+
+/**
+ * A leading number ("1.", "2-3 ", "１．", "第2章"): numbering, not the heading's words. The number must end in a mark or
+ * a space, so the year in "2024年の振り返り" stays.
+ */
+const LEADING_NUMBER = /^第?[\d０-９]+(?:[.\-．][\d０-９]+)*(?:[章節.)．、:：]|(?=\s))\s*/u;
+
+/** A leading decoration: an emoji or a symbol, with its joiners and variation selectors. */
+const LEADING_DECORATION = /^[\p{So}\p{Sk}\p{Extended_Pictographic}\u{FE0F}\u{200D}]+\s*/u;
+
+const CLOSING_MARKS: ReadonlySet<string> = new Set(" \t?？!！。.:：");
+
+/** The text without the closing marks at its end, scanned from the end rather than by a pattern that backtracks. */
+const withoutClosingMarks = (text: string): string => {
+  const characters = [...text];
+  const kept = characters.findLastIndex((character) => !CLOSING_MARKS.has(character));
+  return characters.slice(0, kept + 1).join("");
+};
+
+/** A heading as the lexicons are matched against it: lower case, without its number, decoration and closing marks. */
+export const headingWords = (heading: string): string => {
+  const trimmed = heading.trim().toLowerCase();
+  const stripped = trimmed.replace(LEADING_DECORATION, "").replace(LEADING_NUMBER, "").replace(LEADING_DECORATION, "");
+  return withoutClosingMarks(stripped).trim();
+};
+
+const LATIN_LETTER = /^[a-z]$/u;
+
+/** A Latin letter on the outer side of a match means the match is part of a longer word ("key" in "keyboard"). */
+const breaksWord = (outside: string | undefined, inside: string | undefined): boolean =>
+  outside !== undefined && inside !== undefined && LATIN_LETTER.test(outside) && LATIN_LETTER.test(inside);
+
+export const startsWithWord = (words: string, phrase: string): boolean => words.startsWith(phrase) && !breaksWord(words[phrase.length], phrase.at(-1));
+
+export const endsWithWord = (words: string, phrase: string): boolean => words.endsWith(phrase) && !breaksWord(words.at(-phrase.length - 1), phrase[0]);
+
+/** Whether the phrase stands in the words as a whole word (any place for Japanese, between word edges for Latin text). */
+export const containsWord = (words: string, phrase: string): boolean => {
+  if (phrase.length === 0) return false;
+  const starts = [...words.matchAll(new RegExp(`(?=${escapeRegExp(phrase)})`, "gu"))].map((match) => match.index);
+  return starts.some((index) => !breaksWord(words[index - 1], phrase[0]) && !breaksWord(words[index + phrase.length], phrase.at(-1)));
+};
+
+const matchesAt = (words: string, entry: LexiconEntry): boolean => {
+  const phrase = entry.pattern.toLowerCase();
+  return entry.position === "before" ? startsWithWord(words, phrase) : endsWithWord(words, phrase);
+};
+
+/** The lexicon entry the heading's words take the form of (longest first), or undefined. Unpositioned entries stand at the end. */
+export const headingFormOf = (words: string, lexicon: Lexicon): LexiconEntry | undefined =>
+  lexicon.toSorted((left, right) => right.pattern.length - left.pattern.length).find((entry) => matchesAt(words, entry));
+
+/** Which bookend the heading is: "before" (an opening such as はじめに), "after" (a closing such as まとめ), or undefined. */
+export const bookendOf = (words: string, lexicon: Lexicon): "before" | "after" | undefined =>
+  lexicon.find((entry) => startsWithWord(words, entry.pattern.toLowerCase()))?.position;
+
+/** The heading's words without any occurrence of the counterpart, so デメリット is not read as メリット. */
+const withoutCounterpart = (words: string, counterpart: string): string => (counterpart === "" ? words : words.replaceAll(counterpart, " "));
+
+/** The pairs (good side, bad side) that both stand in different headings: a symmetric pros / cons layout. */
+export const proConPairs = (headings: readonly string[], lexicon: Lexicon): string[] =>
+  lexicon.flatMap((entry) => {
+    const pro = entry.pattern.toLowerCase();
+    const con = (entry.instead_of ?? "").toLowerCase();
+    if (con === "") return [];
+    const proAt = headings.findIndex((words) => containsWord(withoutCounterpart(words, con), pro));
+    const conAt = headings.findIndex((words, index) => index !== proAt && containsWord(words, con));
+    return proAt >= 0 && conAt >= 0 ? [`${entry.pattern}/${entry.instead_of ?? ""}`] : [];
+  });

package/src/structure-shape/of-document.ts

@@ -0,0 +1,7 @@
+import type { ProseDocument } from "../plugin.ts";
+import { STRUCTURE_BASELINE } from "./baseline-load.ts";
+import { structureFeaturesOf } from "./features.ts";
+import { structureScoreOf, type StructureScore } from "./score.ts";
+
+/** A document's structure measures against the shipped human baseline of its language. */
+export const structureOf = (doc: ProseDocument): StructureScore => structureScoreOf(structureFeaturesOf(doc), STRUCTURE_BASELINE[doc.language]);

package/src/structure-shape/restatement.ts

@@ -0,0 +1,41 @@
+import type { LengthUnit } from "../plugin.ts";
+
+/** Japanese is compared in runs of this many characters; four is past most particle-and-ending runs (ことが, します). */
+const CHAR_GRAM = 4;
+
+/** Punctuation, symbols and spaces end a run: a gram across them is shape, not content. */
+const BREAK = /[\s\p{P}\p{S}]/u;
+
+const WORD = /[\p{L}\p{N}]+/gu;
+
+/** Runs of code points, so a character outside the Basic Multilingual Plane counts as one. */
+const charGrams = (text: string): string[] =>
+  text.split(BREAK).flatMap((run) => {
+    const characters = [...run];
+    return Array.from({ length: Math.max(0, characters.length - CHAR_GRAM + 1) }, (_, start) => characters.slice(start, start + CHAR_GRAM).join(""));
+  });
+
+const wordGrams = (text: string): string[] => {
+  const words = (text.toLowerCase().match(WORD) ?? []).filter((word) => word.length > 0);
+  return words.slice(1).map((word, index) => `${words[index] ?? ""} ${word}`);
+};
+
+/** The distinct grams of a text: four-character runs for a document measured in characters, word pairs otherwise. */
+export const gramsOf = (text: string, unit: LengthUnit): ReadonlySet<string> => new Set(unit === "char" ? charGrams(text) : wordGrams(text));
+
+/** Fewer grams than this in the closing section and the share says more about its length than about what it says. */
+export const MIN_CLOSING_GRAMS = 10;
+
+const PERCENT = 100;
+
+/**
+ * The share of the closing section's grams already written in the body, as a rounded percentage: a まとめ that only
+ * restates scores high. Undefined when the closing is too short to measure.
+ */
+export const restatementPercent = (closing: string, body: string, unit: LengthUnit): number | undefined => {
+  const closingGrams = gramsOf(closing, unit);
+  if (closingGrams.size < MIN_CLOSING_GRAMS) return undefined;
+  const bodyGrams = gramsOf(body, unit);
+  const repeated = [...closingGrams].filter((gram) => bodyGrams.has(gram)).length;
+  return Math.round((repeated / closingGrams.size) * PERCENT);
+};

package/src/structure-shape/score.ts

@@ -0,0 +1,65 @@
+import { DIRECTION, type Direction, type FeatureValue } from "./features.ts";
+import type { FeatureBaseline, LanguageBaseline } from "./baseline.ts";
+
+/** A high measure counts past this percentile of human articles; a low one (uniform sections) below 100 minus it. */
+export const BEYOND_STEP = 90;
+const MEDIAN_STEP = 50;
+const ALL = 100;
+
+/** One measure set against the human articles. */
+export type Placement = {
+  readonly feature: FeatureValue;
+  readonly direction: Direction;
+  /** Roughly what share of human articles the value is past, away from human writing (0 when it is not past any). */
+  readonly pastShare: number | undefined;
+  /** The value past which the measure counts: the human p90, or p10 for a low measure. */
+  readonly limit: number | undefined;
+  readonly median: number | undefined;
+  /** Whether the value is past the limit: one point of the structure score. */
+  readonly beyond: boolean;
+};
+
+/** The structure score: how many measures are past the human p90, out of how many could be compared. */
+export type StructureScore = {
+  readonly placements: readonly Placement[];
+  readonly score: number;
+  readonly compared: number;
+  /** How many human articles the baseline read, or undefined when there is no baseline for the language. */
+  readonly articles: number | undefined;
+};
+
+const stepsOf = (baseline: FeatureBaseline): [number, number][] => [...baseline.percentiles.entries()].toSorted((left, right) => left[0] - right[0]);
+
+/** For a high measure: the highest step whose value is below this one. For a low one: the share whose value is above it. */
+export const pastShareOf = (value: number, baseline: FeatureBaseline, direction: Direction): number => {
+  const steps = stepsOf(baseline);
+  if (direction === "high") return steps.findLast(([, at]) => at < value)?.[0] ?? 0;
+  const above = steps.find(([, at]) => at > value)?.[0];
+  return above === undefined ? 0 : ALL - above;
+};
+
+const limitStep = (direction: Direction): number => (direction === "high" ? BEYOND_STEP : ALL - BEYOND_STEP);
+
+export const isBeyond = (value: number, limit: number, direction: Direction): boolean => (direction === "high" ? value > limit : value < limit);
+
+const placementOf = (feature: FeatureValue, baseline: FeatureBaseline | undefined): Placement => {
+  const direction = DIRECTION[feature.id];
+  const limit = baseline?.percentiles.get(limitStep(direction));
+  const median = baseline?.percentiles.get(MEDIAN_STEP);
+  if (feature.value === undefined || baseline === undefined || limit === undefined) {
+    return { feature, direction, pastShare: undefined, limit, median, beyond: false };
+  }
+  const beyond = isBeyond(feature.value, limit, direction);
+  return { feature, direction, pastShare: pastShareOf(feature.value, baseline, direction), limit, median, beyond };
+};
+
+/** Each measure placed against the language's human baseline, and the count of those past the human p90. Never an opaque number. */
+export const structureScoreOf = (features: readonly FeatureValue[], baseline: LanguageBaseline | undefined): StructureScore => {
+  const placements = features.map((feature) => placementOf(feature, baseline?.features[feature.id]));
+  return {
+    placements,
+    score: placements.filter((placement) => placement.beyond).length,
+    compared: placements.filter((placement) => placement.pastShare !== undefined).length,
+    articles: baseline?.articles,
+  };
+};