chaffjs 0.17.0 → 0.19.0

package/src/number-name.ts

@@ -1,4 +1,4 @@
-import { hyphenGroups, isHyphen } from "./orthography.ts";
+import { HYPHEN_CHARS, hyphenGroups, isHyphen } from "./orthography.ts";
 import type { Span, Token } from "./plugin.ts";
 
 /**
@@ -6,8 +6,9 @@ import type { Span, Token } from "./plugin.ts";
  * 「〒100-8916 東京都」の郵便番号、「紀尾井町1-3 東京ガーデンテラス」の番地は、後ろの空白が番号と題・項目の区切りで、「3回」「3 回」のような空け方の好みではない。
  * 空け方を数える材料にしない。
  *
- * 形で名前と読むのは五つだけ。0 で始まる組を含むハイフンつなぎの番号、※・〒や鉤括弧のすぐ後ろの番号、市・区・町まで下りた地名のすぐ後ろのハイフンつなぎの番号、
- * 文頭の階層つきの節番号（2.1、3.1.2）、続き番号になった行の頭の番号（白書の注）。文頭でも「223 言語」「1ターン」は数量なので、階層の無い数は文頭というだけでは名前と読まない。
+ * 形で名前と読むのは六つだけ。0 で始まる組を含むハイフンつなぎの番号、※・〒や鉤括弧のすぐ後ろの番号、市・区・町まで下りた地名のすぐ後ろのハイフンつなぎの番号、
+ * 文頭の階層つきの節番号（2.1、3.1.2）、続き番号になった行や番号の並ぶ箇条書きの頭の番号（白書の注、関連コード）、「問3」のように番号の前に書く語のすぐ後ろの番号。
+ * 文頭でも「223 言語」「1ターン」は数量なので、階層の無い数は文頭というだけでは名前と読まない。
  * そのうえで、すぐ後ろ（空白 1 つまで）の語が数につく語（助数詞・数・助詞・助動詞）なら数量として数え続ける（「3-5 日」「26.7 万行」）。
  * 品詞が無ければ判断せず、名前とは読まない。
  */
@@ -36,12 +37,14 @@ const isHyphenIdentifier = (digits: string): boolean => {
 const DIGIT_OR_POINT = /[\d.]/u;
 const isRunChar = (char: string | undefined): boolean => DIGIT_OR_POINT.test(char ?? "") || isHyphen(char);
 
-/** at を含む、数字・小数点・ハイフン（全角の「－」なども）の並び。at が並びの中に無ければ undefined。 */
+/** at を含む、数字・小数点・ハイフン（全角の「－」なども）の並び。at が並びの中に無ければ undefined。並びは本文ほど長くなれるので、1 字ずつ呼び直さずに進む。 */
 export const digitRunAround = (text: string, at: number): Span | undefined => {
   if (!isRunChar(text[at])) return undefined;
-  const startAt = (index: number): number => (index > 0 && isRunChar(text[index - 1]) ? startAt(index - 1) : index);
-  const endAt = (index: number): number => (index < text.length && isRunChar(text[index]) ? endAt(index + 1) : index);
-  return { start: startAt(at), end: endAt(at) };
+  let start = at;
+  while (start > 0 && isRunChar(text[start - 1])) start -= 1;
+  let end = at;
+  while (end < text.length && isRunChar(text[end])) end += 1;
+  return { start, end };
 };
 
 /** 並びの後ろ（空白 1 つまで）で始まる語。 */
@@ -63,23 +66,123 @@ const CONJUNCTIONS = new Set(["CCONJ", "SCONJ"]);
  */
 const NUMBERED_LINE = /^[ \t]*(?:[-*+][ \t]+)?(?<number>\d{1,3})[ \t]+\S/gmu;
 
-export const sequenceLabelStarts = (text: string): ReadonlySet<number> => {
+const noteSequenceStarts = (text: string): number[] => {
   const lines = [...text.matchAll(NUMBERED_LINE)].map((match) => {
     const number = match.groups?.["number"] ?? "";
     return { start: match.index + match[0].indexOf(number), value: Number(number) };
   });
   const continues = (index: number): boolean =>
     lines[index - 1]?.value === (lines[index]?.value ?? 0) - 1 || lines[index + 1]?.value === (lines[index]?.value ?? 0) + 1;
-  return new Set(lines.flatMap((line, index) => (continues(index) ? [line.start] : [])));
+  return lines.flatMap((line, index) => (continues(index) ? [line.start] : []));
 };
 
+/**
+ * 箇条書きの項目の頭の番号（- 1122 医療費控除…、- 3-1個人情報…）。番号の後ろに「.」を書く番号付きの箇条書き（1. 概要）は、番号が印なので入れない。
+ * 階層は「.」かハイフンでつなぐ。
+ */
+const LIST_ITEM_NUMBER = new RegExp(`^[ \\t]*[-*+][ \\t]+(?<number>\\d+(?:[.${HYPHEN_CHARS}]\\d+)*)(?![\\d.．])`, "gmu");
+const LEVEL_SEPARATOR = new RegExp(`[.${HYPHEN_CHARS}]`, "u");
+
+/** 番号の並び以外で、番号の付いた項目と数える最少の数。二つだけでは「3 言語」「5 ユーザー」のような数量の並びと見分けがつかない。 */
+const MIN_LIST_ITEMS = 3;
+
+/** 階層の無い番号だけの並びを、コードの並びと読む最少の桁。「- 3 ユーザー」「- 5 チーム」「- 8 アカウント」は数量の並び。 */
+const MIN_CODE_DIGITS = 3;
+
+/** コードの並びで、次の項目までに増える数の上限。コードは近い番号を並べ（1122、1124、1126）、数量の並びは飛ぶ（100、200、300）。 */
+const MAX_CODE_STEP = 9;
+
+type ListItem = {
+  readonly start: number;
+  readonly lineStart: number;
+  readonly lineEnd: number;
+  readonly number: string;
+  readonly levels: readonly number[];
+};
+
+const listItems = (text: string): ListItem[] =>
+  [...text.matchAll(LIST_ITEM_NUMBER)].map((match) => {
+    const number = match.groups?.["number"] ?? "";
+    const lineEnd = text.indexOf("\n", match.index);
+    return {
+      start: match.index + match[0].length - number.length,
+      lineStart: match.index,
+      lineEnd: lineEnd === -1 ? text.length : lineEnd,
+      number,
+      levels: number.split(LEVEL_SEPARATOR).map(Number),
+    };
+  });
+
+/** 目次の順で earlier が later より前か（2-19 → 3 → 3-1 → 3-1-1）。 */
+const comesBefore = (earlier: readonly number[], later: readonly number[]): boolean => {
+  const differs = earlier.findIndex((level, index) => level !== later[index]);
+  if (differs === -1) return earlier.length < later.length;
+  return (earlier[differs] ?? 0) < (later[differs] ?? -1);
+};
+
+/** 空行だけを挟んで次の項目が続き、番号が増えているか。 */
+const followsItem = (text: string, previous: ListItem, item: ListItem): boolean =>
+  text.slice(previous.lineEnd, item.lineStart).trim() === "" && comesBefore(previous.levels, item.levels);
+
+/**
+ * 前の項目と同じ見出しの下に次の番号が続くか（3 → 3-1、2-18 → 2-19）。並びの番号は増えていくので、頭が同じなら後ろは段を持つ。
+ * 「5 → 8-10」は範囲で、目次の段ではない。
+ */
+const isOutlineStep = (previous: ListItem | undefined, item: ListItem): boolean => previous !== undefined && previous.levels[0] === item.levels[0];
+
+/** 同じ桁数（MIN_CODE_DIGITS 以上）で、MAX_CODE_STEP までずつ増えていくコードの並び（1122、1124、1126）。 */
+const isCodeRun = (run: readonly ListItem[]): boolean => {
+  const width = run[0]?.number.length ?? 0;
+  const closeToPrevious = (item: ListItem, index: number): boolean => index === 0 || (item.levels[0] ?? 0) - (run[index - 1]?.levels[0] ?? 0) <= MAX_CODE_STEP;
+  return width >= MIN_CODE_DIGITS && run.every((item, index) => item.number.length === width && closeToPrevious(item, index));
+};
+
+/**
+ * 番号の並びとして読める項目の並び。目次の段を含む並び（2-19、3、3-1）か、コードの並び（1122、1124）。
+ * 階層の無い数が増えていくだけの並び（3、5、8 や 100、200、300）は、数量を並べたものと見分けがつかない。
+ */
+const isLabelRun = (run: readonly ListItem[]): boolean =>
+  run.length >= MIN_LIST_ITEMS && (run.some((item, index) => isOutlineStep(run[index - 1], item)) || isCodeRun(run));
+
+/**
+ * 箇条書きの項目の頭に番号を置き、下へ番号が増えていく並び（国税庁の関連コード 1122、1124、…、ガイドラインの目次 2-19、3、3-1）。
+ * 番号は飛んでもよいが、isLabelRun の並びだけを番号と読む。
+ */
+const numberedListStarts = (text: string): number[] =>
+  listItems(text)
+    .reduce<ListItem[][]>((runs, item) => {
+      const run = runs.at(-1);
+      const previous = run?.at(-1);
+      if (run !== undefined && previous !== undefined && followsItem(text, previous, item)) run.push(item);
+      else runs.push([item]);
+      return runs;
+    }, [])
+    .filter(isLabelRun)
+    .flatMap((run) => run.map((item) => item.start));
+
+export const sequenceLabelStarts = (text: string): ReadonlySet<number> => new Set([...noteSequenceStarts(text), ...numberedListStarts(text)]);
+
 const isGeoUnit = (token: Token): boolean => token.features?.["NameType"] === "GeoUnit";
 const isPlaceWord = (token: Token): boolean => isGeoUnit(token) || token.features?.["NameType"] === "Geo";
 
-/** end で終わる、間を空けずに続く地名と地名の単位（東京都千代田区紀尾井町）。近いほうから。 */
+/** 終わりの位置ごとの、その位置で終わる最初の地名の語。Map は NaN を NaN の鍵で引けるが、位置の NaN はどこにも等しくないので入れない。 */
+const placesByEnd = (tokens: readonly Token[]): ReadonlyMap<number, Token> => {
+  const byEnd = new Map<number, Token>();
+  tokens.forEach((token) => {
+    if (isPlaceWord(token) && !Number.isNaN(token.span.end) && !byEnd.has(token.span.end)) byEnd.set(token.span.end, token);
+  });
+  return byEnd;
+};
+
+/**
+ * end で終わる、間を空けずに続く地名と地名の単位（東京都千代田区紀尾井町）。近いほうから。
+ * 連なりは本文ほど長くなれるので、語を 1 度だけ読む。同じ語に戻れば（幅の無い語。adapter が壊れているとき）そこで止める。
+ */
 export const placeChainBefore = (tokens: readonly Token[], end: number): Token[] => {
-  const last = tokens.find((token) => token.span.end === end && isPlaceWord(token));
-  return last === undefined ? [] : [last, ...placeChainBefore(tokens, last.span.start)];
+  const byEnd = placesByEnd(tokens);
+  const chain = new Set<Token>();
+  for (let last = byEnd.get(end); last !== undefined && !chain.has(last); last = byEnd.get(last.span.start)) chain.add(last);
+  return [...chain];
 };
 
 /**
@@ -97,22 +200,46 @@ const isNameShaped = (text: string, run: Span): boolean => {
   return isHyphenIdentifier(digits) || LABEL_LEAD.test(before) || (LINE_HEAD.test(before) && SECTION_NUMBER.test(digits));
 };
 
+/** 「問3」「図2」: 番号のすぐ前（空白を挟まない）の語が、番号の前に書いて名前を付ける語。「質問3」の「問」は語ではない。 */
+const followsLabelWord = (run: Span, tokens: readonly Token[], base: number, labelWords: ReadonlySet<string>): boolean =>
+  tokens.some((token) => token.span.end === base + run.start && labelWords.has(token.surface));
+
+/**
+ * 文書全体で一度だけ読むもの。sequence は sequenceLabelStarts の結果、topUnits は都道府県の単位、
+ * labelWords は番号の前に書いて名前を付ける語（語彙表 numbered-label の before）。
+ */
+export type NameContext = {
+  readonly sequence?: ReadonlySet<number>;
+  readonly topUnits?: ReadonlySet<string>;
+  readonly labelWords?: ReadonlySet<string>;
+};
+
 /**
- * text の run が名前として書かれた数か。tokens は文書全体の座標で、base は text の先頭の位置。sequence は sequenceLabelStarts の結果、
- * topUnits は都道府県の単位。
+ * text の run が名前として書かれた数か。tokens は文書全体の座標で、base は text の先頭の位置。
  * 後ろの語が読めない（品詞が無い、語の切れ目が合わない）ときは、数量として数え続ける。
  */
-export const isNumberName = (
-  text: string,
-  run: Span,
-  tokens: readonly Token[] | undefined,
-  base: number,
-  sequence: ReadonlySet<number> = new Set(),
-  topUnits: ReadonlySet<string> = new Set(),
-): boolean => {
-  const inSequence = sequence.has(base + run.start);
-  if (tokens === undefined || !(inSequence || isNameShaped(text, run) || isAddressNumber(text, run, tokens, base, topUnits))) return false;
+export const isNumberName = (text: string, run: Span, tokens: readonly Token[] | undefined, base: number, context: NameContext = {}): boolean => {
+  const inSequence = context.sequence?.has(base + run.start) ?? false;
+  if (tokens === undefined) return false;
+  const shaped =
+    inSequence ||
+    isNameShaped(text, run) ||
+    isAddressNumber(text, run, tokens, base, context.topUnits ?? new Set()) ||
+    followsLabelWord(run, tokens, base, context.labelWords ?? new Set());
+  if (!shaped) return false;
   const next = wordAfter(tokens, text, run, base);
   if (next === undefined) return false;
   return !isBoundToNumber(next) || (inSequence && CONJUNCTIONS.has(next.pos));
 };
+
+/** 「第」と番号の後ろに書く区切りの語（第1節、第 2 章）。 */
+const DIVISION_LABEL = /第 ?\d+ ?(?<unit>[^\s\d第]+)$/u;
+
+/**
+ * before が「第1節」「第4章第2節」のように、「第」と番号と区切りの語（語彙表 numbered-label の after）で終わるか。
+ * その後ろの空白は題との区切りで、空け方の好みではない（第1節 AI の…）。
+ */
+export const endsWithDivisionLabel = (before: string, units: ReadonlySet<string>): boolean => {
+  const unit = DIVISION_LABEL.exec(before)?.groups?.["unit"];
+  return unit !== undefined && units.has(unit);
+};

package/src/orthography.ts

@@ -140,7 +140,9 @@ const isOrdinalRun = (chars: readonly string[], index: number): boolean => {
 const MIN_CODE_GROUPS = 3;
 
 // 通達の「5－1－1」は全角の「－」でつなぐ。ハイフン「‐」やマイナス「−」も同じに読む。長音「ー」は仮名の一部なので入れない。
-const HYPHEN = /[-‐‑−－]/u;
+/** 正規表現の文字クラスに入れるハイフンの字。- は範囲の記号にならないよう \ を付ける。 */
+export const HYPHEN_CHARS = "\\-‐‑−－";
+const HYPHEN = new RegExp(`[${HYPHEN_CHARS}]`, "u");
 
 /** 数字をつなぐハイフンか。半角の - のほか、全角・和文のハイフンも。 */
 export const isHyphen = (char: string | undefined): boolean => char !== undefined && HYPHEN.test(char);
@@ -150,13 +152,25 @@ export const hyphenGroups = (text: string): string[] => text.split(HYPHEN).filte
 
 const isCodeChar = (char: string | undefined): boolean => DIGIT.test(char ?? "") || isHyphen(char);
 
-const isCode = (chars: readonly string[], digit: number): boolean => {
+/** digit を含む、isPart の字の並び。 */
+const runAround = (chars: readonly string[], digit: number, isPart: (char: string | undefined) => boolean): string => {
   let [first, last] = [digit, digit];
-  while (first > 0 && isCodeChar(chars[first - 1])) first -= 1;
-  while (last < chars.length - 1 && isCodeChar(chars[last + 1])) last += 1;
-  return hyphenGroups(chars.slice(first, last + 1).join("")).length >= MIN_CODE_GROUPS;
+  while (first > 0 && isPart(chars[first - 1])) first -= 1;
+  while (last < chars.length - 1 && isPart(chars[last + 1])) last += 1;
+  return chars.slice(first, last + 1).join("");
 };
 
+const isDottedChar = (char: string | undefined): boolean => DIGIT.test(char ?? "") || char === ".";
+
+/** 「1.0.0」「3.1.2」のように . でつないだ 3 組以上の数字は、版や項目の番号で数量ではない。2 組（「1.5 倍」）は小数なので数える。 */
+const isDottedCode = (chars: readonly string[], digit: number): boolean =>
+  runAround(chars, digit, isDottedChar)
+    .split(".")
+    .filter((group) => group !== "").length >= MIN_CODE_GROUPS;
+
+const isCode = (chars: readonly string[], digit: number): boolean =>
+  hyphenGroups(runAround(chars, digit, isCodeChar)).length >= MIN_CODE_GROUPS || isDottedCode(chars, digit);
+
 /** 日本語との境目にある数字が、数えない書き方（「第3条」の番地、「073-489-5909」の符号）か。 */
 const isUncountedNumber = (chars: readonly string[], index: number): boolean => {
   const [left, right] = [chars[index], chars[index + 1]];
@@ -166,8 +180,7 @@ const isUncountedNumber = (chars: readonly string[], index: number): boolean =>
   return left === ORDINAL_PREFIX || (isJapanese(left) && isCode(chars, next));
 };
 
-const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "offset"> | undefined => {
-  if (isUncountedNumber(chars, index)) return undefined;
+const candidateAt = (chars: readonly string[], index: number): Omit<Boundary, "offset"> | undefined => {
   const [left, right, after] = [chars[index], chars[index + 1], chars[index + 2]];
   const touching = latinBeside(left, right, "before-digit") ?? leftRunBeside(chars, index, right);
   if (touching !== undefined) return { kind: touching, spaced: false };
@@ -176,6 +189,12 @@ const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "of
   return spaced === undefined ? undefined : { kind: spaced, spaced: true };
 };
 
+// isUncountedNumber は数字の並びを端まで読む。境目の候補でだけ聞けば、並びの字ごとではなく並びごとに一度で済む。
+const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "offset"> | undefined => {
+  const candidate = candidateAt(chars, index);
+  return candidate === undefined || isUncountedNumber(chars, index) ? undefined : candidate;
+};
+
 /**
  * 文の中の境目を全部。offset は境目の位置（左の字の直後、UTF-16）。配列を作り直さずに一度なめる。
  * text は本文でないものを空白で覆った文、written は覆う前の同じ範囲。覆ってできた空白（リンクの「[」など）は
@@ -194,13 +213,30 @@ export const latinBoundaries = (text: string, written: string = text): Boundary[
 };
 
 /**
- * 混ざっているとき、そろえるべき少数派。種類（英字・数字の前・数字の後ろ）ごとに数える。
+ * 二通りの書き方（true と false）が混ざっているとき、そろえるべき少数派。どちらか一方しか無ければ undefined。
  * 同数なら、文書が先に使った書き方をその文書の書き方とみなし、後から出た書き方を少数派にする。
  */
-export const minorityStyle = (boundaries: readonly { readonly spaced: boolean }[]): boolean | undefined => {
-  const spaced = boundaries.filter((boundary) => boundary.spaced).length;
-  const touching = boundaries.length - spaced;
-  if (spaced === 0 || touching === 0) return undefined;
-  if (spaced !== touching) return spaced < touching;
-  return !(boundaries[0]?.spaced ?? false);
+export const minorityOf = (written: readonly boolean[]): boolean | undefined => {
+  const yes = written.filter(Boolean).length;
+  const no = written.length - yes;
+  if (yes === 0 || no === 0) return undefined;
+  if (yes !== no) return yes < no;
+  return !(written[0] ?? false);
 };
+
+const PERCENT = 100;
+
+/**
+ * 二通りに書き分けたもの（side が true か false）のうち、少ないほうのもの。少ないほうが全体の limitPercent パーセントを超えるなら、
+ * 文書が使い分けていると見て空。どちらか一方しか無くても空。
+ */
+export const minorityWithin = <T>(items: readonly T[], side: (item: T) => boolean, limitPercent: number): T[] => {
+  const minority = minorityOf(items.map(side));
+  if (minority === undefined) return [];
+  const odd = items.filter((item) => side(item) === minority);
+  return odd.length * PERCENT > limitPercent * items.length ? [] : odd;
+};
+
+/** 空けるか詰めるかの少数派。種類（英字・数字の前・数字の後ろ）ごとに数える。 */
+export const minorityStyle = (boundaries: readonly { readonly spaced: boolean }[]): boolean | undefined =>
+  minorityOf(boundaries.map((boundary) => boundary.spaced));

package/src/outline/render.ts

@@ -0,0 +1,85 @@
+import type { LengthUnit } from "../plugin.ts";
+import type { Outline, OutlineEntry, Shape } from "./shape.ts";
+import type { OutlineText } from "./text.ts";
+
+/** One file's outline as `chaff outline` shows it. */
+export type DocumentOutline = { readonly path: string; readonly language: string; readonly outline: Outline };
+
+type Measure = { readonly name: string; readonly value: (shape: Shape, unit: LengthUnit) => string };
+
+/** The shape's measures in the order they are shown. */
+const measuresOf = (text: OutlineText): readonly Measure[] => [
+  { name: text.headings, value: (shape) => String(shape.headings) },
+  { name: text.averageSection, value: (shape, unit) => text.length(shape.averageSectionLength, unit) },
+  { name: text.lists, value: (shape) => `${String(shape.listPercent)}%` },
+  { name: text.bold, value: (shape) => String(shape.bold) },
+];
+
+/** The shape's measures on one line: 見出し 6、節の平均 120 字、… / headings 6, average section 80 words, … */
+export const shapeMeasures = (shape: Shape, unit: LengthUnit, text: OutlineText): string =>
+  measuresOf(text)
+    .map((measure) => text.measure(measure.name, measure.value(shape, unit)))
+    .join(text.separator);
+
+const shapeLine = (document: DocumentOutline, text: OutlineText): string =>
+  text.shapeOf(document.path, shapeMeasures(document.outline.shape, document.outline.unit, text));
+
+const headingOf = (entry: OutlineEntry, text: OutlineText): string => (entry.depth === 0 ? text.lead : `${"#".repeat(entry.depth)} ${entry.heading}`);
+
+/** A heading indented by its depth, where it is, and how long its own text is. */
+const entryLine = (entry: OutlineEntry, document: DocumentOutline, text: OutlineText): string => {
+  const indent = "  ".repeat(Math.max(entry.depth - 1, 0));
+  const length = text.length(entry.length, document.outline.unit);
+  return `  ${indent}${headingOf(entry, text)}  (${document.path}:${String(entry.line)})  ${length}`;
+};
+
+const outlineBlock = (document: DocumentOutline, text: OutlineText): string[] => [
+  shapeLine(document, text),
+  "",
+  ...document.outline.entries.map((entry) => entryLine(entry, document, text)),
+];
+
+/** Each measure before and after, so a restructure shows as numbers that moved. */
+const changeBlock = (before: DocumentOutline, after: DocumentOutline, text: OutlineText): string[] => [
+  "",
+  text.changed(before.path, after.path),
+  ...measuresOf(text).map(
+    (measure) => `  ${measure.name}: ${measure.value(before.outline.shape, before.outline.unit)} → ${measure.value(after.outline.shape, after.outline.unit)}`,
+  ),
+];
+
+/** For a person: each file's shape and outline, then, for two files, how each measure moved. */
+export const renderOutlineFriendly = (documents: readonly DocumentOutline[], text: OutlineText): string => {
+  const [before, after] = documents;
+  const outlines = documents.flatMap((document, index) => [...(index > 0 ? [""] : []), ...outlineBlock(document, text)]);
+  return [...outlines, ...(before !== undefined && after !== undefined ? changeBlock(before, after, text) : [])].join("\n");
+};
+
+const COMPACT_UNIT: Readonly<Record<LengthUnit, string>> = { char: "chars", word: "words" };
+
+const compactEntry = (entry: OutlineEntry, document: DocumentOutline): string => {
+  const heading = entry.depth === 0 ? "lead" : `h${String(entry.depth)} ${entry.heading}`;
+  return `${document.path}:${String(entry.line)}: ${heading} (${String(entry.length)} ${COMPACT_UNIT[document.outline.unit]})`;
+};
+
+/** For grep: one section per line, then each file's shape. */
+export const renderOutlineCompact = (documents: readonly DocumentOutline[], text: OutlineText): string =>
+  [
+    ...documents.flatMap((document) => document.outline.entries.map((entry) => compactEntry(entry, document))),
+    ...documents.map((document) => shapeLine(document, text)),
+  ].join("\n");
+
+const outlineJson = (document: DocumentOutline): Readonly<Record<string, unknown>> => ({
+  path: document.path,
+  language: document.language,
+  unit: document.outline.unit,
+  shape: document.outline.shape,
+  outline: document.outline.entries,
+});
+
+/** For an AI: one file's outline and shape, or `before` and `after` for two. */
+export const renderOutlineJson = (documents: readonly DocumentOutline[]): string => {
+  const [before, after] = documents;
+  const body = before !== undefined && after !== undefined ? { before: outlineJson(before), after: outlineJson(after) } : documents.map(outlineJson)[0];
+  return JSON.stringify(body, null, 2);
+};

package/src/outline/shape.ts

@@ -0,0 +1,72 @@
+import { lengthOf } from "../measure.ts";
+import { lineStarts, placeOf } from "../position.ts";
+import type { LengthUnit, ProseDocument, Section, Sentence } from "../plugin.ts";
+import { coversOffset, spanIndex } from "../compare/spans.ts";
+
+/** One section: its heading's depth (0 for the text before the first heading), its words, its line, and its own length. */
+export type OutlineEntry = { readonly depth: number; readonly heading: string; readonly line: number; readonly length: number };
+
+/** The measures a restructure moves. Lengths are in the document's unit: characters for Japanese, words for English. */
+export type Shape = {
+  readonly headings: number;
+  /** The mean length of the sections that have text, rounded. */
+  readonly averageSectionLength: number;
+  /** The share of the body text inside list items, as a rounded percentage. */
+  readonly listPercent: number;
+  /** Bold spans (Markdown `**…**`). */
+  readonly bold: number;
+};
+
+export type Outline = { readonly unit: LengthUnit; readonly entries: readonly OutlineEntry[]; readonly shape: Shape };
+
+const PERCENT = 100;
+
+const lengthOfAll = (sentences: readonly Sentence[], unit: LengthUnit): number => sentences.reduce((sum, sentence) => sum + lengthOf(sentence, unit), 0);
+
+/** The text before the first heading is listed only when there is some: a title line alone has nothing above it. */
+const isListed = (entry: OutlineEntry): boolean => entry.depth > 0 || entry.length > 0;
+
+/** Where each heading starts, keyed by where it ends: a section begins after its heading, past a Setext underline. */
+type HeadingStarts = ReadonlyMap<number, number>;
+
+const headingStartsOf = (doc: ProseDocument): HeadingStarts => new Map((doc.markup?.headings ?? []).map((heading) => [heading.end, heading.start]));
+
+/** A heading's first line; for the text above the first heading, the line its text starts on, below any front matter. */
+const startOf = (section: Section, headingStarts: HeadingStarts): number =>
+  section.depth === 0 ? (section.sentences[0]?.span.start ?? section.span.start) : (headingStarts.get(section.span.start) ?? section.span.start);
+
+type Place = { readonly lineOf: (offset: number) => number; readonly headingStarts: HeadingStarts };
+
+const entryOf = (section: Section, unit: LengthUnit, place: Place): OutlineEntry => ({
+  depth: section.depth,
+  heading: section.heading,
+  line: place.lineOf(startOf(section, place.headingStarts)),
+  length: lengthOfAll(section.sentences, unit),
+});
+
+const averageOf = (lengths: readonly number[]): number =>
+  lengths.length === 0 ? 0 : Math.round(lengths.reduce((sum, length) => sum + length, 0) / lengths.length);
+
+const listPercentOf = (doc: ProseDocument): number => {
+  const lists = spanIndex(doc.listSpans);
+  const total = lengthOfAll(doc.sentences, doc.lengthUnit);
+  const inLists = lengthOfAll(
+    doc.sentences.filter((sentence) => coversOffset(lists, sentence.span.start)),
+    doc.lengthUnit,
+  );
+  return total === 0 ? 0 : Math.round((inLists / total) * PERCENT);
+};
+
+/** The document's outline and its shape, measured from the same reading lint uses. */
+export const outlineOf = (doc: ProseDocument): Outline => {
+  const starts = lineStarts(doc.source);
+  const place: Place = { lineOf: (offset) => placeOf(starts, offset).line, headingStarts: headingStartsOf(doc) };
+  const entries = doc.sections.map((section) => entryOf(section, doc.lengthUnit, place)).filter(isListed);
+  const shape: Shape = {
+    headings: entries.filter((entry) => entry.depth > 0).length,
+    averageSectionLength: averageOf(entries.map((entry) => entry.length).filter((length) => length > 0)),
+    listPercent: listPercentOf(doc),
+    bold: doc.sections.reduce((sum, section) => sum + section.strongCount, 0),
+  };
+  return { unit: doc.lengthUnit, entries, shape };
+};

package/src/outline/text.ts

@@ -0,0 +1,50 @@
+import type { Texts } from "../ui.ts";
+import type { LengthUnit } from "../plugin.ts";
+import { counted } from "../render/plural.ts";
+
+export type OutlineText = {
+  readonly usage: string;
+  /** How a length is written: 120 字 / 120 words. */
+  readonly length: (count: number, unit: LengthUnit) => string;
+  /** The text before the first heading, in place of a heading. */
+  readonly lead: string;
+  readonly headings: string;
+  readonly averageSection: string;
+  readonly lists: string;
+  readonly bold: string;
+  readonly separator: string;
+  /** The line that names the file and gives its shape. */
+  readonly shapeOf: (path: string, measures: string) => string;
+  readonly measure: (name: string, value: string) => string;
+  /** The heading of the before/after block. */
+  readonly changed: (before: string, after: string) => string;
+};
+
+export const OUTLINE_TEXT: Texts<OutlineText> = {
+  ja: {
+    usage: "使い方: chaff outline <file> [<書き直した後>] [--compact | --json] [--language ja|en|…] [--genre <ジャンル>]",
+    length: (count, unit) => (unit === "char" ? `${String(count)} 字` : `${String(count)} 語`),
+    lead: "（最初の見出しの前）",
+    headings: "見出し",
+    averageSection: "節の平均",
+    lists: "箇条書き",
+    bold: "太字",
+    separator: "、",
+    shapeOf: (path, measures) => `${path} の構成: ${measures}`,
+    measure: (name, value) => `${name} ${value}`,
+    changed: (before, after) => `構成の変化（${before} → ${after}）`,
+  },
+  en: {
+    usage: "usage: chaff outline <file> [<rewritten>] [--compact | --json] [--language ja|en|…] [--genre <genre>]",
+    length: (count, unit) => (unit === "char" ? counted(count, "character") : counted(count, "word")),
+    lead: "(before the first heading)",
+    headings: "headings",
+    averageSection: "average section",
+    lists: "in lists",
+    bold: "bold",
+    separator: ", ",
+    shapeOf: (path, measures) => `${path} outline: ${measures}`,
+    measure: (name, value) => `${name} ${value}`,
+    changed: (before, after) => `How the shape changed (${before} → ${after})`,
+  },
+};

package/src/page-furniture.ts

@@ -1,5 +1,7 @@
 import type { Span } from "./plugin.ts";
 import { NO_OUTLINE, type Outline } from "./structure/build.ts";
+import { markdownFigures, textFigures } from "./text-figures.ts";
+import type { MarkdownNode } from "./markdown-node.ts";
 
 /** 改ページから、飾りの行を探しにいく行数。飾りと改ページのあいだには空行が数行入る。 */
 const REACH = 3;
@@ -41,5 +43,12 @@ export const pageFurniture = (source: string): Span[] => {
   });
 };
 
-/** テキストの文書の外形。見出しもコードも無く、ページの飾りだけを覆う。木を作る入口はどれもこれを使う。 */
-export const textOutline = (source: string): Outline => ({ ...NO_OUTLINE, opaque: pageFurniture(source) });
+/** テキストの文書の外形。見出しもコードも無く、ページの飾り・線で描いた図・メールの引用した返信だけを覆う。木を作る入口はどれもこれを使う。 */
+export const textOutline = (source: string, replyQuotes: readonly Span[] = []): Outline => ({
+  ...NO_OUTLINE,
+  opaque: [...pageFurniture(source), ...textFigures(source), ...replyQuotes],
+});
+
+/** 紙面の形で本文でないもの: テキストの文書のページの飾り（Markdown には改ページが無い）と、線で描いた図。 */
+export const layoutMasks = (root: MarkdownNode, source: string, markdown: boolean): Span[] =>
+  markdown ? markdownFigures(root, source) : [...pageFurniture(source), ...textFigures(source)];

package/src/plain-source.ts

@@ -1,5 +1,6 @@
 /**
  * The Markdown parser drops a leading BOM and reads a lone "\r" as a line break; the rest of chaff counts "\n" only.
  * Reading the file this way keeps every offset, line and column on the text the writer sees.
+ * Every leading BOM goes, so that running this twice (on reading, then in buildDocument) gives the same text as once.
  */
-export const plainSource = (raw: string): string => raw.replace(/^\uFEFF/u, "").replace(/\r\n?/gu, "\n");
+export const plainSource = (raw: string): string => raw.replace(/^\uFEFF+/u, "").replace(/\r\n?/gu, "\n");