chaffjs 0.17.0 → 0.18.0

package/src/document.ts

@@ -1,31 +1,28 @@
-import { fromMarkdown } from "mdast-util-from-markdown";
-import { gfmTable } from "micromark-extension-gfm-table";
-import { gfmTableFromMarkdown } from "mdast-util-gfm-table";
-import { frontmatter } from "micromark-extension-frontmatter";
-import { frontmatterFromMarkdown } from "mdast-util-frontmatter";
+import { headingReadersOf, sectionsOf } from "./document-sections.ts";
 import { atxHeadingText, headingText } from "./heading-text.ts";
 import { hasTitle } from "./heading-title.ts";
 import { maskSpans } from "./mask.ts";
 import { spansWithin, unmaskedSoftBreaks } from "./soft-break.ts";
 import { segmentJoined } from "./joined-view.ts";
-import { lineParagraphs } from "./line-paragraphs.ts";
+import { documentLineParagraphs } from "./line-paragraphs.ts";
+import { subheadingPieces } from "./subheading-line.ts";
 import { speakerLabels } from "./speaker-labels.ts";
 import { buildTree, type Outline } from "./structure/build.ts";
 import { isMarkdownPath } from "./structure/markdown-path.ts";
-import { pageFurniture, textOutline } from "./page-furniture.ts";
+import { layoutMasks, textOutline } from "./page-furniture.ts";
 import { tokenizedLexicons } from "./lexicon-tokens.ts";
+import { plainSource } from "./plain-source.ts";
 import { inPageAnchors, isInPageNavigation, isNavigationList, type InPageAnchors } from "./in-page-nav.ts";
-import type { BulletList, LanguageAdapter, Paragraph, ProseDocument, Section, Sentence, Span, StructureNode, DocumentProfile, Token } from "./plugin.ts";
-
-type Place = { readonly offset?: number | undefined };
-type Node = {
-  readonly type: string;
-  readonly url?: string | undefined;
-  readonly value?: string | undefined;
-  readonly identifier?: string | undefined;
-  readonly position?: { readonly start: Place; readonly end: Place } | undefined;
-  readonly children?: readonly Node[] | undefined;
-};
+import { eachPreOrder } from "./tree-walk.ts";
+import { alertReader } from "./template-syntax.ts";
+import { opaqueSpans, parse, readMarkdown, spansOfType } from "./markdown-read.ts";
+import { spanOf, type MarkdownNode as Node } from "./markdown-node.ts";
+import { emailParts, emailVocabulary } from "./email-parts.ts";
+import { cutTextSpans } from "./span-cut.ts";
+import { markdownFigures } from "./text-figures.ts";
+import { documentMarkup } from "./markup.ts";
+import { BARE_URL } from "./bare-url.ts";
+import type { BulletList, LanguageAdapter, Markup, Paragraph, ProseDocument, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
 
 /**
  * 本文として数えないもの。
@@ -36,22 +33,8 @@ type Node = {
  */
 const NOT_PROSE = new Set(["code", "inlineCode", "html", "yaml", "toml", "table", "blockquote", "thematicBreak", "definition", "image", "imageReference"]);
 
-const spanOf = (node: Node): Span | undefined => {
-  const start = node.position?.start.offset;
-  const end = node.position?.end.offset;
-  return start === undefined || end === undefined ? undefined : { start, end };
-};
-
-const walk = (node: Node, visit: (node: Node) => void): void => {
-  visit(node);
-  (node.children ?? []).forEach((child) => walk(child, visit));
-};
-
-const parse = (source: string): Node =>
-  fromMarkdown(source, { extensions: [gfmTable(), frontmatter(["yaml"])], mdastExtensions: [gfmTableFromMarkdown(), frontmatterFromMarkdown(["yaml"])] });
-
 /** link は `[text](url)` の外側だけを覆う。表示される文字は本文なので残す。 */
-const linkChrome = (node: Node): Span[] => {
+export const linkChrome = (node: Node): Span[] => {
   const whole = spanOf(node);
   const first = node.children?.[0];
   const last = node.children?.at(-1);
@@ -88,27 +71,26 @@ const EMPHASIS = new Set(["strong", "emphasis", "delete"]);
  * `:::note` `:::` のようなディレクティブ。Zenn・Docusaurus・VitePress が使う記法で、
  * 標準の Markdown には無いため段落として解析される。囲みの指定であって文章ではない。
  */
-const DIRECTIVE = /^:::[^\n]*/gmu;
-
-/**
- * `https://…` をそのまま書いた URL。GFM の autolink 拡張を入れていないので mdast では
- * ただのテキストになり、本文として残る。残すと、見出しと URL の中の識別子が一致して
- * 「見出しの繰り返し」と読まれる。表示される文字も本文ではない。
- */
-const BARE_URL = /https?:\/\/[^\s)<>"'\]]+/gu;
+const DIRECTIVE = /^[ \t]*:::[^\n]*/gmu;
 
 const matchSpans = (source: string, pattern: RegExp): Span[] =>
   [...source.matchAll(pattern)].map((match) => ({ start: match.index, end: match.index + match[0].length }));
 
 const directiveSpans = (source: string): Span[] => [...matchSpans(source, DIRECTIVE), ...matchSpans(source, BARE_URL)];
 
-const collectMasks = (root: Node, source: string, anchors: InPageAnchors): Span[] => {
-  const spans: Span[] = [...directiveSpans(source)];
-  walk(root, (node) => {
+const collectMasks = (root: Node, source: string, anchors: InPageAnchors, syntax: readonly Span[]): Span[] => {
+  const spans: Span[] = [...directiveSpans(source), ...syntax];
+  const alertOf = alertReader(source);
+  eachPreOrder(root, (node) => {
     if (node.type === "link" || node.type === "linkReference") {
       spans.push(...linkChrome(node));
       return;
     }
+    const alert = alertOf(node);
+    if (alert !== undefined) {
+      spans.push(...alert);
+      return;
+    }
     if (!NOT_PROSE.has(node.type) && node.type !== "heading" && !isInPageNavigation(node, anchors)) return;
     const span = spanOf(node);
     if (span !== undefined) spans.push(span);
@@ -118,7 +100,7 @@ const collectMasks = (root: Node, source: string, anchors: InPageAnchors): Span[
 
 const emphasisSpans = (root: Node, source: string): Span[] => {
   const spans: Span[] = [];
-  walk(root, (node) => {
+  eachPreOrder(root, (node) => {
     if (EMPHASIS.has(node.type)) spans.push(...emphasisChrome(node, source));
   });
   return spans;
@@ -126,7 +108,7 @@ const emphasisSpans = (root: Node, source: string): Span[] => {
 
 const textOf = (node: Node, source: string): string => {
   const parts: string[] = [];
-  walk(node, (child) => {
+  eachPreOrder(node, (child) => {
     if (child.type !== "text" && child.type !== "inlineCode") return;
     const span = spanOf(child);
     if (span !== undefined) parts.push(source.slice(span.start, span.end));
@@ -136,17 +118,25 @@ const textOf = (node: Node, source: string): string => {
 
 export type Heading = { readonly depth: number; readonly text: string; readonly start: number; readonly end: number };
 
-const headingsOf = (root: Node, source: string): Heading[] => {
+/**
+ * `neutral` is the source with the template syntax blanked: `## Usage {% if x %}` is headed "Usage". Whether a
+ * heading has a title is read from the source, since `## {{ product }}` names something.
+ * 引用した返信（outside）の中の見出しは、ほかの人の文書の見出しなので数えない。
+ */
+const headingsOf = (root: Node, source: string, neutral: string, outside: readonly Span[] = []): Heading[] => {
   const found: Heading[] = [];
-  walk(root, (node) => {
+  eachPreOrder(root, (node) => {
     if (node.type !== "heading") return;
     const span = spanOf(node);
     const depth = typeof node === "object" && "depth" in node && typeof node.depth === "number" ? node.depth : 1;
-    if (span === undefined) return;
+    if (span === undefined || startsInside(span, outside)) return;
     // ATX（行頭が #）なら閉じの # も外す。setext の見出しの末尾の # は言葉なので残す。
     const read = source.startsWith("#", span.start) ? atxHeadingText : headingText;
-    const text = read(textOf(node, source));
-    if (hasTitle(text)) found.push({ depth, text, start: span.start, end: span.end });
+    const written = textOf(node, source);
+    const shown = textOf(node, neutral);
+    // A blanked tag leaves a run of spaces where the reader sees one gap.
+    const text = read(shown === written ? written : shown.replace(/ {2,}/gu, " "));
+    if (hasTitle(read(written))) found.push({ depth, text, start: span.start, end: span.end });
   });
   return found;
 };
@@ -162,7 +152,7 @@ const startsInside = (span: Span, regions: readonly Span[]): boolean => regions.
  */
 const strongSpans = (root: Node, masked: readonly Span[]): Span[] => {
   const found: Span[] = [];
-  walk(root, (node) => {
+  eachPreOrder(root, (node) => {
     if (node.type !== "strong") return;
     const span = spanOf(node);
     if (span !== undefined && !startsInside(span, masked)) found.push(span);
@@ -170,18 +160,6 @@ const strongSpans = (root: Node, masked: readonly Span[]): Span[] => {
   return found;
 };
 
-const within = (span: Span, from: number, to: number): boolean => span.start >= from && span.start < to;
-
-const spansOfType = (root: Node, type: string, keep: (node: Node) => boolean = () => true): Span[] => {
-  const found: Span[] = [];
-  walk(root, (node) => {
-    if (node.type !== type || !keep(node)) return;
-    const span = spanOf(node);
-    if (span !== undefined) found.push(span);
-  });
-  return found;
-};
-
 /**
  * 文の分割は**段落ごと**に行う。
  *
@@ -193,6 +171,10 @@ const shift = (span: Span, by: number): Span => ({ start: by + span.start, end:
 
 const breaksWithin = (breaks: readonly Span[], paragraph: Span): Span[] => spansWithin(breaks, paragraph).map((span) => shift(span, -paragraph.start));
 
+/** 段落を、中の小見出しの行（「（経済再生）」）の後ろで切った片。片ごとに分割すれば、小見出しが次の文に入らない。 */
+const piecesOf = (prose: string, paragraphs: readonly Span[]): Span[] =>
+  paragraphs.flatMap((paragraph) => subheadingPieces(prose.slice(paragraph.start, paragraph.end)).map((piece) => shift(piece, paragraph.start)));
+
 const sentencesOf = (prose: string, paragraphs: readonly Span[], adapter: LanguageAdapter, softBreaks: readonly Span[]): Sentence[] =>
   paragraphs.flatMap((paragraph) =>
     segmentJoined(prose.slice(paragraph.start, paragraph.end), breaksWithin(softBreaks, paragraph), (text) => adapter.segment(text))
@@ -202,38 +184,10 @@ const sentencesOf = (prose: string, paragraphs: readonly Span[], adapter: Langua
         text: sentence.text,
         ...(sentence.tokens === undefined ? {} : { tokens: sentence.tokens.map((token) => ({ ...token, span: shift(token.span, paragraph.start) })) }),
         ...(sentence.wrapBreaks === undefined ? {} : { wrapBreaks: sentence.wrapBreaks.map((span) => shift(span, paragraph.start)) }),
+        ...(sentence.embeddedLanguage === undefined ? {} : { embeddedLanguage: sentence.embeddedLanguage }),
       })),
   );
 
-/** 見出しの語。品詞を読んでいない文書（文が tokens を持たない）では分けない。 */
-const headingTokensOf = (adapter: LanguageAdapter, tagged: boolean, heading: string): { headingTokens?: readonly Token[] } =>
-  tagged && heading !== "" ? { headingTokens: adapter.segment(heading).sentences.flatMap((sentence) => sentence.tokens ?? []) } : {};
-
-const sectionsOf = (
-  headings: readonly Heading[],
-  sentences: readonly Sentence[],
-  strongs: readonly Span[],
-  length: number,
-  tokensOf: (heading: string) => { headingTokens?: readonly Token[] },
-): Section[] => {
-  const bounds = headings.map((heading, index) => ({ heading, from: heading.end, to: headings[index + 1]?.start ?? length }));
-  const lead = { heading: { depth: 0, text: "", start: 0, end: 0 }, from: 0, to: headings[0]?.start ?? length };
-  return [lead, ...bounds]
-    .filter((bound) => bound.to > bound.from)
-    .map(({ heading, from, to }) => {
-      const inside = sentences.filter((sentence) => within(sentence.span, from, to));
-      return {
-        depth: heading.depth,
-        heading: heading.text,
-        ...tokensOf(heading.text),
-        span: { start: from, end: to },
-        sentences: inside,
-        strongCount: strongs.filter((span) => within(span, from, to)).length,
-        firstSentence: inside[0],
-      };
-    });
-};
-
 /** sentences（並び順）の中で、start が offset 以上の最初の添字。1 行 1 段落の何万行でも、段落ごとに全部をなめない。 */
 const firstStartingAt = (sentences: readonly Sentence[], offset: number): number => {
   const search = (low: number, high: number): number => {
@@ -247,24 +201,18 @@ const firstStartingAt = (sentences: readonly Sentence[], offset: number): number
 const sentencesWithin = (sentences: readonly Sentence[], span: Span): Sentence[] =>
   sentences.slice(firstStartingAt(sentences, span.start), firstStartingAt(sentences, span.end));
 
-const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] =>
-  spans
-    // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
-    // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
-    .filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end))
-    .flatMap((span) =>
-      lineParagraphs(
-        prose,
-        span,
-        sentencesWithin(sentences, span).map((sentence) => sentence.span),
-      ),
-    )
-    .map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
+const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] => {
+  // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
+  // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
+  const outsideLists = spans.filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end));
+  const withSentences = outsideLists.map((span) => ({ span, sentences: sentencesWithin(sentences, span).map((sentence) => sentence.span) }));
+  return documentLineParagraphs(prose, withSentences).map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
+};
 
 /** 箇条書きは list ノードの直下の項目を数える。入れ子の項目は内側の list のものとして数える。目次のような案内だけの箇条書きは数えない。 */
 const listsOf = (root: Node, source: string, anchors: InPageAnchors): BulletList[] => {
   const found: BulletList[] = [];
-  walk(root, (node) => {
+  eachPreOrder(root, (node) => {
     if (node.type !== "list" || isNavigationList(node, anchors)) return;
     const span = spanOf(node);
     if (span === undefined) return;
@@ -311,33 +259,35 @@ const proseOf = (source: string, masked: readonly Span[]): string => {
   return maskSpans(unlabelled, speakerLabels(unlabelled));
 };
 
-export const buildDocument = (
-  path: string,
-  source: string,
-  adapter: LanguageAdapter,
-  team: TeamRules = EMPTY_TEAM,
-  profile: DocumentProfile | undefined = undefined,
-): ProseDocument => {
-  const root = parse(source);
+const teamLexicons = (adapter: LanguageAdapter, team: TeamRules): LanguageAdapter["lexicons"] => ({
+  ...adapter.lexicons,
+  "internal-jargon": team.jargon.map((pattern) => ({ pattern })),
+  // 使わない書き方を pattern に、使う書き方を instead_of に置く。語彙表の「同じことの別の書き方」と同じ向き。
+  "preferred-term": Object.entries(team.prefer ?? {}).map(([pattern, use]) => ({ pattern, instead_of: use })),
+});
+
+const documentOf = (path: string, source: string, adapter: LanguageAdapter, team: TeamRules, profile: DocumentProfile | undefined): ProseDocument => {
+  const markdown = isMarkdownPath(path);
+  const { root, syntax } = markdown ? readMarkdown(source) : { root: parse(source), syntax: [] };
   const anchors = inPageAnchors(root);
+  const emailLayout = emailParts(source, emailVocabulary(adapter.lexicons));
   // 強調の記号は「本文でないもの」だが、太字の数を数えるときの「覆われた場所」ではない。
   // 同じ集合にすると、太字が自分の記号のせいで覆われた場所にあることになり、1 つも数えられなくなる。
-  // テキストの文書は、ページのヘッダーとフッターも本文ではない（Markdown には改ページが無い）。
-  const blocks = [...collectMasks(root, source, anchors), ...(isMarkdownPath(path) ? [] : pageFurniture(source))];
-  const masked = [...blocks, ...emphasisSpans(root, source)];
-  const prose = proseOf(source, masked);
-  // ページの案内は段落としても数えない。数えると、目次の行が「本題までの段落」に入る。
-  const paragraphSpans = spansOfType(root, "paragraph", (node) => !isInPageNavigation(node, anchors));
+  // ページのヘッダーとフッター（テキストの文書）と、線で描いた図も本文ではない。
+  const blocks = [...collectMasks(root, source, anchors, syntax), ...layoutMasks(root, source, markdown), ...emailLayout.furniture];
+  const prose = proseOf(source, [...blocks, ...emphasisSpans(root, source)]);
+  // ページの案内は段落としても数えない。数えると、目次の行が「本題までの段落」に入る。メールのヘッダーや署名の行は段落から切り取る。
+  const paragraphSpans = cutTextSpans(
+    spansOfType(root, "paragraph", (node) => !isInPageNavigation(node, anchors)),
+    emailLayout.furniture,
+    source,
+  );
+  const headings = headingsOf(root, source, maskSpans(source, syntax), emailLayout.replyQuotes);
   const listItems = spansOfType(root, "listItem");
   // Markdown は段落を流し込んで表示するので、段落の中の改行は読み手に見えない。テキストの文書は行をそのまま見せる（法令は 1 行 1 号）。
-  const softBreaks = isMarkdownPath(path) ? unmaskedSoftBreaks(source, prose) : [];
-  const sentences = sentencesOf(prose, paragraphSpans, adapter, softBreaks);
-  const lexicons = {
-    ...adapter.lexicons,
-    "internal-jargon": team.jargon.map((pattern) => ({ pattern })),
-    // 使わない書き方を pattern に、使う書き方を instead_of に置く。語彙表の「同じことの別の書き方」と同じ向き。
-    "preferred-term": Object.entries(team.prefer ?? {}).map(([pattern, use]) => ({ pattern, instead_of: use })),
-  };
+  const softBreaks = markdown ? unmaskedSoftBreaks(source, prose) : [];
+  const sentences = sentencesOf(prose, piecesOf(prose, paragraphSpans), adapter, softBreaks);
+  const lexicons = teamLexicons(adapter, team);
   const tagged = sentences.some((sentence) => sentence.tokens !== undefined);
   const patterns = adapter.structure;
   // null は「作ったが構造を読めない言語だった」、undefined は「まだ作っていない」。
@@ -348,7 +298,7 @@ export const buildDocument = (
     language: adapter.id,
     lengthUnit: adapter.capabilities.lengthUnit,
     capabilities: adapter.capabilities,
-    sections: sectionsOf(headingsOf(root, source), sentences, strongSpans(root, blocks), source.length, (heading) => headingTokensOf(adapter, tagged, heading)),
+    sections: sectionsOf(headings, sentences, strongSpans(root, blocks), source.length, headingReadersOf(adapter, tagged, lexicons)),
     sentences,
     listSpans: listItems,
     paragraphs: paragraphsOf(prose, paragraphSpans, sentences, listItems),
@@ -368,7 +318,7 @@ export const buildDocument = (
                 source,
                 language: adapter.id,
                 // テキストの文書は、ページの飾りを覆って読む。フッターの「Section 9」を木の節にしない。
-                outline: isMarkdownPath(path) ? outlineOf(root, source) : textOutline(source),
+                outline: markdown ? outlineOf(root, source, syntax, emailLayout.replyQuotes) : textOutline(source, emailLayout.replyQuotes),
                 markdown: isMarkdownPath(path),
                 profile,
               },
@@ -378,16 +328,34 @@ export const buildDocument = (
     },
     profile,
     prose,
+    replyQuotes: emailLayout.replyQuotes,
+    get markup(): Markup {
+      return documentMarkup(root, source, markdown, [...emailLayout.replyQuotes, ...syntax]);
+    },
   };
 };
 
-/** 番号を探してはいけない範囲。コードの中の「第3条」は条ではなく、参照でもない。 */
-const OPAQUE = ["code", "inlineCode", "html", "yaml", "toml"];
+/**
+ * 文書モデルを作る。text はファイルの中身のままでよい。先頭の BOM を外し、CRLF と CR を LF にそろえた doc.source を読む。
+ * 位置（span・offset）はすべて doc.source の上の位置で、渡した text の上の位置ではない。行と桁は同じ。
+ */
+export const buildDocument = (
+  path: string,
+  text: string,
+  adapter: LanguageAdapter,
+  team: TeamRules = EMPTY_TEAM,
+  profile: DocumentProfile | undefined = undefined,
+): ProseDocument => documentOf(path, plainSource(text), adapter, team, profile);
 
-const outlineOf = (root: Node, source: string): Outline => ({
-  headings: headingsOf(root, source),
-  opaque: OPAQUE.flatMap((type) => spansOfType(root, type)),
+/** 引用した返信も中を読まない。ほかの人の言葉の中の定義や番号は、この文書のものではない。 */
+const outlineOf = (root: Node, source: string, syntax: readonly Span[], replyQuotes: readonly Span[]): Outline => ({
+  headings: headingsOf(root, source, maskSpans(source, syntax), replyQuotes),
+  opaque: [...opaqueSpans(root), ...markdownFigures(root, source), ...syntax, ...replyQuotes],
+  tables: spansOfType(root, "table"),
 });
 
-/** 構造を読むための Markdown の手がかり。見出しと、中を読まない範囲。 */
-export const markdownOutline = (source: string): Outline => outlineOf(parse(source), source);
+/** 構造を読むための Markdown の手がかり。見出しと、中を読まない範囲。返信の引用は、言語パッケージの語彙表で見分ける。 */
+export const markdownOutline = (source: string, lexicons: LanguageAdapter["lexicons"] = {}): Outline => {
+  const { root, syntax } = readMarkdown(source);
+  return outlineOf(root, source, syntax, emailParts(source, emailVocabulary(lexicons)).replyQuotes);
+};

package/src/email-parts.ts

@@ -0,0 +1,269 @@
+import type { Lexicon, Span } from "./plugin.ts";
+import { linesOf, type Line } from "./structure/lines.ts";
+
+/**
+ * The parts of a plain-text or Markdown email that are not the writer's prose: the header, the separator lines between
+ * messages, the signature, and a quoted reply. Found by shape, in any genre; the field names and the attribution words
+ * come from the language package's lexicons.
+ */
+export type EmailVocabulary = {
+  /** Header field names, lower-cased. */
+  readonly headerFields: ReadonlySet<string>;
+  /** The fields whose value the writer types (Subject, To), lower-cased. Their value stays prose, on a line of its own. */
+  readonly writtenFields: ReadonlySet<string>;
+  /** Words an attribution line ends with, before its colon ("wrote", 「書きました」). */
+  readonly attributions: readonly string[];
+  /** How an archive's note in place of a removed attachment ends ("was scrubbed...", 「を保管しました...」). */
+  readonly attachmentNotes: readonly string[];
+};
+
+export type EmailParts = {
+  /**
+   * Header lines (all of them but a written field's value), separator lines, signatures and attribution lines: masked
+   * from prose, and paragraphs are cut around them.
+   */
+  readonly furniture: readonly Span[];
+  /** An attribution line and the quoted lines after it: someone else's words, so not this document's headings or structure. */
+  readonly replyQuotes: readonly Span[];
+};
+
+const lowerCased = (lexicon: Lexicon | undefined): ReadonlySet<string> => new Set((lexicon ?? []).map((entry) => entry.pattern.toLowerCase()));
+
+export const emailVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): EmailVocabulary => ({
+  headerFields: lowerCased(lexicons["email-header-field"]),
+  writtenFields: lowerCased(lexicons["email-written-field"]),
+  attributions: (lexicons["email-attribution"] ?? []).map((entry) => entry.pattern),
+  attachmentNotes: (lexicons["email-attachment-note"] ?? []).map((entry) => entry.pattern),
+});
+
+/** A signature has at most this many lines after its "-- " line (the convention is four). More is prose again. */
+const SIGNATURE_MAX_LINES = 4;
+/** An attribution paragraph longer than this is a paragraph of the writer's that happens to end with "wrote:". */
+const ATTRIBUTION_MAX_LINES = 3;
+/** Away from the top of a message, one known field ("Date: the first Monday") is a sentence; a header names several. */
+const HEADER_MIN_FIELDS = 2;
+/** An attachment stub is its note and a few fields: Mailman's longest is the note, Name, Type, Size, Desc and URL. */
+const ATTACHMENT_STUB_MAX_LINES = 6;
+
+/** "Field: value" or 「件名：値」. A colon followed by text, as in a URL, is not a field. */
+const FIELD = /^([^\s:：]{1,40})(?::(?:[ \t]+|$)|：[ \t]*)/u;
+const FOLDED = /^[ \t]+\S/u;
+/** An mbox archive's envelope line (RFC 4155): "From ", the sender, then the date. It has no colon after From. */
+const ENVELOPE = /^From [^\s:]+ /u;
+const SIGNATURE = /^-- ?$/u;
+/** A line of dashes, underscores or equals signs, with a few words between them or none: "-----Original Message-----". */
+const SEPARATOR = /^[-_=]{3,}(?:([^-_=]{1,40})[-_=]{2,})?$/u;
+/** Two rules with only spaces or strokes between them are a drawing (an RFC's diagram), not a separator. */
+const LETTER = /\p{L}/u;
+const QUOTED = /^ {0,3}>/u;
+const QUOTED_TEXT = /^ {0,3}>[\s>]*\S/u;
+/** Gmail's attribution ends with the sender's address and a colon: 「… 山田太郎 <taro@example.com>:」. An archive writes "taro at example.com". */
+const BRACKETED_THEN_COLON = /<([^<>]*)>[ \t]*[:：]$/u;
+const ADDRESS_MARK = /@| at /u;
+const LATIN_LETTER = /[A-Za-z]/u;
+/** The stub's last field: its value is a URL alone, bare or in angle brackets. */
+const URL_VALUE = /^<?[a-z][a-z0-9+.-]*:\/\/\S+?>?$/iu;
+
+const isBlank = (line: Line | undefined): boolean => line !== undefined && line.text.trim() === "";
+const isSeparator = (line: Line | undefined): boolean => {
+  const match = line === undefined ? null : SEPARATOR.exec(line.text.trim());
+  return match !== null && (match[1] === undefined || LETTER.test(match[1]));
+};
+const isQuoted = (line: Line | undefined): boolean => line !== undefined && QUOTED.test(line.text);
+const endOf = (line: Line): number => line.start + line.text.length;
+const spanOfLines = (first: Line, last: Line): Span => ({ start: first.start, end: endOf(last) });
+
+/** The first index at or after `from` whose line meets `test`, or -1. Scans forward only, so a long document stays linear. */
+const indexFrom = (lines: readonly Line[], from: number, test: (line: Line, index: number) => boolean): number => {
+  for (let index = Math.max(0, from); index < lines.length; index += 1) {
+    const line = lines[index];
+    if (line !== undefined && test(line, index)) return index;
+  }
+  return -1;
+};
+
+/** The last index at or before `from` whose line meets `test`, or -1. */
+const indexBackFrom = (lines: readonly Line[], from: number, test: (line: Line) => boolean): number => {
+  for (let index = Math.min(from, lines.length - 1); index >= 0; index -= 1) {
+    const line = lines[index];
+    if (line !== undefined && test(line)) return index;
+  }
+  return -1;
+};
+
+/** Runs of lines between blank lines and separator lines. */
+const blocksOf = (lines: readonly Line[]): Line[][] =>
+  lines.reduce<Line[][]>((blocks, line, index) => {
+    if (isBlank(line) || isSeparator(line) || SIGNATURE.test(line.text)) return blocks;
+    const previous = lines[index - 1];
+    if (previous !== undefined && !isBlank(previous) && !isSeparator(previous) && !SIGNATURE.test(previous.text)) blocks.at(-1)?.push(line);
+    else blocks.push([line]);
+    return blocks;
+  }, []);
+
+/** The block opens a message: nothing but blank lines before it, or before it and the nearest separator line. */
+const opensMessage = (lines: readonly Line[], first: Line): boolean => {
+  const last = indexBackFrom(lines, first.number - 2, (line) => !isBlank(line));
+  return last === -1 || isSeparator(lines[last]);
+};
+
+const knownField = (text: string, vocabulary: EmailVocabulary): boolean => {
+  const name = FIELD.exec(text)?.[1];
+  return name !== undefined && vocabulary.headerFields.has(name.toLowerCase());
+};
+
+const isHeaderLine = (line: Line, index: number): boolean =>
+  FIELD.test(line.text) || (index > 0 && FOLDED.test(line.text)) || (index === 0 && ENVELOPE.test(line.text));
+
+const isHeader = (block: readonly Line[], atTop: boolean, vocabulary: EmailVocabulary): boolean => {
+  if (!block.every(isHeaderLine)) return false;
+  const known = block.filter((line) => knownField(line.text, vocabulary)).length;
+  if (known >= HEADER_MIN_FIELDS) return true;
+  // Fewer known fields make a header only at the top of a message, and only if no other field sits beside them: "Summary:" is the writer's.
+  return atTop && known > 0 && block.every((line) => !FIELD.test(line.text) || knownField(line.text, vocabulary));
+};
+
+/** A header's fields: each line with the folded lines under it. */
+const fieldsOf = (block: readonly Line[]): Line[][] =>
+  block.reduce<Line[][]>((fields, line, index) => {
+    if (index > 0 && FOLDED.test(line.text)) fields.at(-1)?.push(line);
+    else fields.push([line]);
+    return fields;
+  }, []);
+
+/** A written field loses only its label and keeps its value; any other field, and the envelope line, go whole. */
+const fieldMask = (field: readonly Line[], vocabulary: EmailVocabulary): Span | undefined => {
+  const first = field[0];
+  const last = field.at(-1);
+  if (first === undefined || last === undefined) return undefined;
+  const label = FIELD.exec(first.text);
+  const name = label?.[1]?.toLowerCase();
+  if (label === null || name === undefined || !vocabulary.writtenFields.has(name)) return spanOfLines(first, last);
+  return { start: first.start, end: first.start + label[0].length };
+};
+
+const headerSpans = (lines: readonly Line[], vocabulary: EmailVocabulary): Span[] =>
+  blocksOf(lines).flatMap((block) => {
+    const first = block[0];
+    if (first === undefined || !isHeader(block, opensMessage(lines, first), vocabulary)) return [];
+    return fieldsOf(block).flatMap((field) => fieldMask(field, vocabulary) ?? []);
+  });
+
+const isUrlField = (line: Line | undefined): boolean => {
+  const label = line === undefined ? null : FIELD.exec(line.text);
+  return line !== undefined && label !== null && URL_VALUE.test(line.text.slice(label[0].length).trim());
+};
+
+/**
+ * An archive's note where it removed an attachment: right after a separator, a line ending as the lexicon says, then
+ * only field lines, the last one a URL. Mailman's "An HTML attachment was scrubbed..." and its "URL: <…>" line.
+ */
+const isAttachmentStub = (lines: readonly Line[], block: readonly Line[], vocabulary: EmailVocabulary): boolean => {
+  const [note, ...fields] = block;
+  if (note === undefined || block.length > ATTACHMENT_STUB_MAX_LINES) return false;
+  const noteText = note.text.trim();
+  if (!vocabulary.attachmentNotes.some((ending) => noteText.endsWith(ending))) return false;
+  return fields.every((line) => FIELD.test(line.text)) && isUrlField(fields.at(-1)) && isSeparator(lines[note.number - 2]);
+};
+
+const attachmentStubSpans = (lines: readonly Line[], vocabulary: EmailVocabulary): Span[] =>
+  blocksOf(lines).flatMap((block) => {
+    const first = block[0];
+    const last = block.at(-1);
+    return first !== undefined && last !== undefined && isAttachmentStub(lines, block, vocabulary) ? [spanOfLines(first, last)] : [];
+  });
+
+/** From the "-- " line: the blank lines right after it, then its lines up to a blank line, a separator or a quote. */
+const signatureAt = (lines: readonly Line[], delimiter: Line): Span => {
+  const firstText = indexFrom(lines, delimiter.number, (line) => !isBlank(line));
+  if (firstText === -1) return spanOfLines(delimiter, delimiter);
+  const stop = indexFrom(
+    lines,
+    firstText,
+    (line, index) => index - firstText >= SIGNATURE_MAX_LINES || isBlank(line) || isSeparator(line) || isQuoted(line) || SIGNATURE.test(line.text),
+  );
+  return spanOfLines(delimiter, lines[(stop === -1 ? lines.length : stop) - 1] ?? delimiter);
+};
+
+const signatureSpans = (lines: readonly Line[]): Span[] => lines.filter((line) => SIGNATURE.test(line.text)).map((line) => signatureAt(lines, line));
+
+const endsWithAttribution = (text: string, vocabulary: EmailVocabulary): boolean => {
+  const trimmed = text.trimEnd();
+  if (ADDRESS_MARK.test(BRACKETED_THEN_COLON.exec(trimmed)?.[1] ?? "")) return true;
+  if (!/[:：]$/u.test(trimmed)) return false;
+  const words = trimmed.slice(0, -1).trimEnd();
+  return vocabulary.attributions.some((word) => {
+    if (!words.endsWith(word)) return false;
+    const before = words.at(-word.length - 1) ?? "";
+    return !(LATIN_LETTER.test(word.charAt(0)) && LATIN_LETTER.test(before));
+  });
+};
+
+/** The paragraph that ends right before `index`, skipping blank lines: its lines, if it is short enough to be an attribution. */
+const attributionBefore = (lines: readonly Line[], index: number, vocabulary: EmailVocabulary): Line[] | undefined => {
+  const lastText = indexBackFrom(lines, index - 1, (line) => !isBlank(line));
+  if (lastText === -1) return undefined;
+  const floor = lastText - ATTRIBUTION_MAX_LINES;
+  const window = lines.slice(Math.max(0, floor), lastText + 1);
+  const opening = window.findLastIndex((line) => isBlank(line) || isSeparator(line) || isQuoted(line));
+  if (opening === -1 && floor > 0) return undefined;
+  const paragraph = window.slice(opening + 1);
+  if (paragraph.length === 0 || paragraph.length > ATTRIBUTION_MAX_LINES) return undefined;
+  return endsWithAttribution(paragraph.map((line) => line.text.trim()).join(" "), vocabulary) ? paragraph : undefined;
+};
+
+/** A quoted line, or a line with text run on from a quoted line with text (a lazy continuation, as Markdown reads it). */
+const continuesQuote = (lines: readonly Line[], index: number): boolean => {
+  const line = lines[index];
+  if (isQuoted(line)) return true;
+  const previous = lines[index - 1];
+  return line !== undefined && !isBlank(line) && !isSeparator(line) && previous !== undefined && QUOTED_TEXT.test(previous.text);
+};
+
+/** The last line of the quote that starts at `index`. Blank lines between quoted lines stay inside it. */
+const quoteEnd = (lines: readonly Line[], index: number): number => {
+  const chunk = { start: index };
+  for (;;) {
+    const next = indexFrom(lines, chunk.start + 1, (_, at) => !continuesQuote(lines, at));
+    if (next === -1) return lines.length - 1;
+    const resumes = indexFrom(lines, next, (line) => !isBlank(line));
+    if (resumes === -1 || !isQuoted(lines[resumes])) return next - 1;
+    chunk.start = resumes;
+  }
+};
+
+const quoteStarts = (lines: readonly Line[]): number[] => lines.flatMap((line, index) => (isQuoted(line) && !continuesQuote(lines, index - 1) ? [index] : []));
+
+type Reply = { readonly attribution: Span; readonly quote: Span };
+
+const repliesOf = (lines: readonly Line[], vocabulary: EmailVocabulary): Reply[] =>
+  quoteStarts(lines).reduce<Reply[]>((replies, start) => {
+    const covered = replies.at(-1);
+    const line = lines[start];
+    if (line === undefined || (covered !== undefined && line.start < covered.quote.end)) return replies;
+    const attribution = attributionBefore(lines, start, vocabulary);
+    const first = attribution?.[0];
+    const last = attribution?.at(-1);
+    if (first === undefined || last === undefined) return replies;
+    const end = lines[quoteEnd(lines, start)];
+    if (end === undefined) return replies;
+    return [...replies, { attribution: spanOfLines(first, last), quote: spanOfLines(first, end) }];
+  }, []);
+
+const bySpanStart = (left: Span, right: Span): number => left.start - right.start;
+
+export const emailParts = (source: string, vocabulary: EmailVocabulary): EmailParts => {
+  const lines = linesOf(source);
+  const replies = repliesOf(lines, vocabulary);
+  const separators = lines.filter((line) => isSeparator(line)).map((line) => spanOfLines(line, line));
+  return {
+    furniture: [
+      ...headerSpans(lines, vocabulary),
+      ...attachmentStubSpans(lines, vocabulary),
+      ...separators,
+      ...signatureSpans(lines),
+      ...replies.map((reply) => reply.attribution),
+    ].toSorted(bySpanStart),
+    replyQuotes: replies.map((reply) => reply.quote),
+  };
+};