chaffjs 0.35.0 → 0.36.0

package/src/detectors/titled-run.ts

@@ -6,8 +6,12 @@ export type TitleWords = { readonly endings: readonly string[] };
 
 /** 題を囲む括弧。鉤括弧と、見出しや分類の名前を囲む隅付き括弧。 */
 const TITLE_MARKS = { "「": "」", "『": "』", "【": "】" };
-/** 名前のすぐ後ろに付く、その名前の補い（就職氷河期世代）、略称（以下「奨励金」という。）、日付や番号。 */
-const QUALIFIER_OPENERS = new Set(["（", "("]);
+/**
+ * 名前のすぐ後ろ（同じ行の空白を挟んでもよい）に付く、その名前の補い（就職氷河期世代、＜創業型＞）、略称（以下「奨励金」という。）、
+ * 日付や番号、ファイルの種類と大きさ（[PDFファイル／333KB]）の括弧。角括弧は同じ行で閉じ、Markdown のリンク（[文字](…)）でないもの。
+ * 半角の < は HTML の札と見分けられないので読まない。
+ */
+const QUALIFIER = /^[ \t\u3000]*(?:[（(＜〈［]|\[[^\]\n]*\](?![([]))/u;
 /** 閉じる語の前に要る字数。1 字なら、題ではなく普通の語（新法、本事業）。 */
 const MIN_BODY_LENGTH = 2;
 const BLANK = /^\s*$/u;
@@ -27,11 +31,12 @@ export const endsInTitle = (run: readonly Token[], words: TitleWords): boolean =
 /**
  * 文（text）の start から end までの連なりが、題の名前の置き方をしているか。
  * 括弧の中身の終わりで閉じる（「令和8年度高度統計人材育成強化拠点形成事業」）、すぐ後ろに補いの括弧が付く
- * （国家公務員中途採用者選考試験（就職氷河期世代））、または文がその連なりだけ（要綱の題を並べた行）。
+ * （国家公務員中途採用者選考試験（就職氷河期世代）、持続化補助金＜創業型＞、登録申込書 [PDFファイル／333KB]）、
+ * または文がその連なりだけ（要綱の題を並べた行）。
  */
 export const isTitlePosition = (text: string, start: number, end: number): boolean => {
   const closesBracket = quotedSpans(text, TITLE_MARKS).some((span) => span.end === end && span.start <= start);
-  const qualified = QUALIFIER_OPENERS.has(text.charAt(end));
+  const qualified = QUALIFIER.test(text.slice(end));
   const alone = BLANK.test(text.slice(0, start)) && BLANK.test(text.slice(end));
   return closesBracket || qualified || alone;
 };

package/src/detectors/token-shape.ts

@@ -1,10 +1,12 @@
 import { hasParticle, isClosed, lastContent } from "../sentence-shape.ts";
 import { isWithinAny, quotedIn } from "../quoted-span.ts";
+import { chronologyRows } from "../chronology-rows.ts";
+import { externalLinkTitleSpans } from "../link-title.ts";
 import type { Detector, Finding, Span, Token } from "../plugin.ts";
 
 /**
  * 文末が名詞で終わる（体言止め）。箇条書きでは普通だが、本文では文が途中で切れて読める。
- * 実文書で測ったら、指摘の 9 割が箇条書きの項目だった。箇条書きは除く。
+ * 実文書で測ったら、指摘の 9 割が箇条書きの項目だった。箇条書きは除く。年表の行（1950年4月 設立。）も名詞で止める書き方なので除く。
  */
 const NOMINAL = new Set(["NOUN", "PROPN"]);
 
@@ -14,8 +16,9 @@ const isDependent = (token: Token): boolean => token.features?.["NounType"] ===
 const inside = (span: Span, spans: readonly Span[]): boolean => spans.some((other) => span.start >= other.start && span.start < other.end);
 
 export const nounEnding: Detector = (doc, options): Finding[] => {
+  const rows = chronologyRows(doc);
   const ended = doc.sentences.filter((sentence) => {
-    if (inside(sentence.span, doc.listSpans) || !isClosed(sentence) || !hasParticle(sentence)) return false;
+    if (inside(sentence.span, doc.listSpans) || rows.has(sentence) || !isClosed(sentence) || !hasParticle(sentence)) return false;
     const last = lastContent(sentence);
     return last !== undefined && NOMINAL.has(last.pos) && !isDependent(last);
   });
@@ -62,19 +65,25 @@ const BREAK: Run = { surface: "", tokens: [] };
  *
  * 鉤括弧で引いたもの（括弧ごと）は、名詞と同じく連なりの中身として飛ばす。引いた題名（「食品中のウイルスの制御のための…ガイドライン」）や
  * 発言は書き手が言い換えられないので、中の助詞は数えない。題名も名前なので、外の連なり（弊社の「新製品」の販売の計画）は続けて数える。
+ * 外のページの題名を引いたリンクの文字（- [サービスの料金の計算の方法](https://…)）も、引いた題名と同じに読む。
  */
-const runsOf = (tokens: readonly Token[], nesting: readonly string[], quoted: readonly Span[]): Run[] =>
+const runsOf = (tokens: readonly Token[], nesting: readonly string[], breaks: readonly string[], quoted: readonly Span[]): Run[] =>
   tokens.reduce<Run[]>((acc, token) => {
     if (isWithinAny(quoted, token.span)) return acc;
     if (nesting.includes(token.surface)) return extend(acc, token);
-    if (token.pos === "PUNCT" || PARTICLE.has(token.pos)) acc.push(BREAK);
+    if (token.pos === "PUNCT" || PARTICLE.has(token.pos) || breaks.includes(token.surface)) acc.push(BREAK);
     return acc;
   }, []);
 
+/** 句と句の間の記号（制限=タグの制限、A→B）。半角の = や : は名詞として読まれ、句読点のようには切れない。 */
+const RUN_BREAK_LEXICON = "particle-run-break";
+
 export const doubledParticle: Detector = (doc, options): Finding[] => {
   const nesting = (options.lexicon ?? []).map((entry) => entry.pattern);
+  const titles = externalLinkTitleSpans(doc.source, doc.links);
+  const breaks = (doc.lexicons[RUN_BREAK_LEXICON] ?? []).map((entry) => entry.pattern);
   return doc.sentences.flatMap((sentence) =>
-    runsOf(sentence.tokens ?? [], nesting, quotedIn(sentence))
+    runsOf(sentence.tokens ?? [], nesting, breaks, [...quotedIn(sentence), ...titles])
       .filter((run) => run.tokens.length > options.limit)
       .map((run) => ({
         rule: "no-doubled-joshi",

package/src/detectors/vague-density.ts

@@ -8,8 +8,11 @@ import { escapeRegExp } from "../orthography.ts";
 import type { Detector, Finding, Lexicon, LexiconEntry, Sentence } from "../plugin.ts";
 import { comparableText, comparableWords, entryRanges } from "./lexicon-match.ts";
 import { rateOver } from "./lexicon.ts";
+import { specifiedByBracket } from "./bracket-specified.ts";
 
 const TERMS_OF_ART = "vague-word-term-of-art";
+/** The words that start an aside giving a name (以下「HR」という, hereinafter): such an aside pins nothing down. */
+const NAME_MARKERS = "definition-marker";
 
 type Place = { readonly start: number; readonly end: number };
 
@@ -36,13 +39,25 @@ const covers = (outer: Place, inner: Place): boolean => outer.start <= inner.sta
 
 const isLonger = (outer: Match, inner: Match): boolean => outer.place.end - outer.place.start > inner.place.end - inner.place.start;
 
-/** The vague words of one sentence, in the order written, each once: none inside a longer vague word or a term of art. */
-export const vagueWordsIn = (sentence: Sentence, vague: Lexicon, termsOfArt: Lexicon): string[] => {
+/** Whether a bracket right after the match pins the word down; token places are in the document, text places in the comparable text. */
+const pinnedDown = (sentence: Sentence, match: Match, byTokens: boolean, nameMarkers: readonly string[]): boolean =>
+  byTokens
+    ? specifiedByBracket(sentence.text, match.place.end - sentence.span.start, nameMarkers)
+    : specifiedByBracket(comparableText(sentence), match.place.end, nameMarkers);
+
+/**
+ * The vague words of one sentence, in the order written, each once: none inside a longer vague word or a term of art,
+ * and none pinned down by the bracket right after it (一定の事項（…）). nameMarkers: the words that start a name in an aside.
+ */
+export const vagueWordsIn = (sentence: Sentence, vague: Lexicon, termsOfArt: Lexicon, nameMarkers: readonly string[] = []): string[] => {
   const byTokens = readsTokens(sentence, [...vague, ...termsOfArt]);
   const matches = matchesOf(sentence, vague, byTokens);
   const terms = matchesOf(sentence, termsOfArt, byTokens);
   const kept = matches.filter(
-    (match) => !matches.some((other) => isLonger(other, match) && covers(other.place, match.place)) && !terms.some((term) => covers(term.place, match.place)),
+    (match) =>
+      !matches.some((other) => isLonger(other, match) && covers(other.place, match.place)) &&
+      !terms.some((term) => covers(term.place, match.place)) &&
+      !pinnedDown(sentence, match, byTokens, nameMarkers),
   );
   return [...new Set(kept.toSorted((left, right) => left.place.start - right.place.start).map((match) => match.pattern))];
 };
@@ -51,8 +66,9 @@ type SentenceHits = { readonly sentence: Sentence; readonly words: readonly stri
 
 export const vagueDensity: Detector = (doc, options): Finding[] => {
   const termsOfArt = entriesForGenre(doc.lexicons[TERMS_OF_ART] ?? [], options.genre);
+  const nameMarkers = (doc.lexicons[NAME_MARKERS] ?? []).map((entry) => entry.pattern);
   const hits: SentenceHits[] = doc.sentences
-    .map((sentence) => ({ sentence, words: vagueWordsIn(sentence, options.lexicon ?? [], termsOfArt) }))
+    .map((sentence) => ({ sentence, words: vagueWordsIn(sentence, options.lexicon ?? [], termsOfArt, nameMarkers) }))
     .filter((hit) => hit.words.length > 0);
   const count = hits.reduce((sum, hit) => sum + hit.words.length, 0);
   const density = rateOver(doc, count, options.limit);

package/src/document-paragraphs.ts

@@ -0,0 +1,27 @@
+import { documentLineParagraphs } from "./line-paragraphs.ts";
+import { listItemParagraphs } from "./list-item-paragraphs.ts";
+import type { Paragraph, Sentence, Span } from "./plugin.ts";
+
+/** sentences（並び順）の中で、start が offset 以上の最初の添字。1 行 1 段落の何万行でも、段落ごとに全部をなめない。 */
+const firstStartingAt = (sentences: readonly Sentence[], offset: number): number => {
+  const search = (low: number, high: number): number => {
+    if (low >= high) return low;
+    const middle = (low + high) >> 1;
+    return (sentences[middle]?.span.start ?? Number.POSITIVE_INFINITY) >= offset ? search(low, middle) : search(middle + 1, high);
+  };
+  return search(0, sentences.length);
+};
+
+const sentencesWithin = (sentences: readonly Sentence[], span: Span): Sentence[] =>
+  sentences.slice(firstStartingAt(sentences, span.start), firstStartingAt(sentences, span.end));
+
+/** 文書の段落と、その中の文。spans は Markdown の段落、listSpans は箇条書きの項目と同じに読む範囲。 */
+export const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] => {
+  // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
+  // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
+  const outsideLists = spans.filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end));
+  const withSentences = outsideLists.map((span) => ({ span, sentences: sentencesWithin(sentences, span).map((sentence) => sentence.span) }));
+  // 1 行 1 段落の書き方かは、行頭の印の項目で割る前の段落で決める。割った後の段落は、その手がかりにならない。
+  const items = documentLineParagraphs(prose, withSentences).flatMap((span) => listItemParagraphs(span, sentencesWithin(sentences, span)));
+  return items.map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
+};

package/src/document.ts

@@ -4,7 +4,7 @@ import { hasTitle } from "./heading-title.ts";
 import { maskSpans } from "./mask.ts";
 import { spansWithin, unmaskedSoftBreaks } from "./soft-break.ts";
 import { segmentJoined } from "./joined-view.ts";
-import { documentLineParagraphs } from "./line-paragraphs.ts";
+import { paragraphsOf } from "./document-paragraphs.ts";
 import { closingRun, standaloneLines, subheadingPieces, type StandsAlone } from "./subheading-line.ts";
 import { isLinkLine } from "./link-line.ts";
 import { speakerLabels } from "./speaker-labels.ts";
@@ -26,7 +26,7 @@ import { markdownFigures } from "./text-figures.ts";
 import { documentMarkup } from "./markup.ts";
 import { BARE_URL } from "./bare-url.ts";
 import { lazyCells, tableCellsOf } from "./table-cells.ts";
-import type { BulletList, LanguageAdapter, Markup, Paragraph, ProseDocument, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
+import type { BulletList, LanguageAdapter, Markup, ProseDocument, Sentence, Span, StructureNode, DocumentProfile } from "./plugin.ts";
 
 /**
  * 本文として数えないもの。
@@ -234,30 +234,10 @@ const sentencesOf = (prose: string, paragraphs: readonly Span[], adapter: Langua
         ...(sentence.tokens === undefined ? {} : { tokens: sentence.tokens.map((token) => ({ ...token, span: shift(token.span, paragraph.start) })) }),
         ...(sentence.wrapBreaks === undefined ? {} : { wrapBreaks: sentence.wrapBreaks.map((span) => shift(span, paragraph.start)) }),
         ...(sentence.embeddedLanguage === undefined ? {} : { embeddedLanguage: sentence.embeddedLanguage }),
+        ...(sentence.listItem === undefined ? {} : { listItem: sentence.listItem }),
       })),
   );
 
-/** sentences（並び順）の中で、start が offset 以上の最初の添字。1 行 1 段落の何万行でも、段落ごとに全部をなめない。 */
-const firstStartingAt = (sentences: readonly Sentence[], offset: number): number => {
-  const search = (low: number, high: number): number => {
-    if (low >= high) return low;
-    const middle = (low + high) >> 1;
-    return (sentences[middle]?.span.start ?? Number.POSITIVE_INFINITY) >= offset ? search(low, middle) : search(middle + 1, high);
-  };
-  return search(0, sentences.length);
-};
-
-const sentencesWithin = (sentences: readonly Sentence[], span: Span): Sentence[] =>
-  sentences.slice(firstStartingAt(sentences, span.start), firstStartingAt(sentences, span.end));
-
-const paragraphsOf = (prose: string, spans: readonly Span[], sentences: readonly Sentence[], listSpans: readonly Span[]): Paragraph[] => {
-  // 箇条書きの中の段落は「段落」として数えない。項目 1 つを 1 段落と読むと、
-  // 段落あたりの文数も長さのばらつきも、箇条書きの多い文書で壊れる。
-  const outsideLists = spans.filter((span) => !listSpans.some((list) => span.start >= list.start && span.start < list.end));
-  const withSentences = outsideLists.map((span) => ({ span, sentences: sentencesWithin(sentences, span).map((sentence) => sentence.span) }));
-  return documentLineParagraphs(prose, withSentences).map((span) => ({ span, sentences: sentencesWithin(sentences, span) }));
-};
-
 /** 箇条書きは list ノードの直下の項目を数える。入れ子の項目は内側の list のものとして数える。目次のような案内だけの箇条書きは数えない。 */
 const listsOf = (root: Node, source: string, anchors: InPageAnchors): BulletList[] => {
   const found: BulletList[] = [];

package/src/figure-references.ts

@@ -1,6 +1,7 @@
 import { escapeRegExp } from "./orthography.ts";
 import { lowerBound } from "./detectors/token-column.ts";
 import type { DocumentNamer, Span } from "./plugin.ts";
+import { isTitleLine, type TitleNotion } from "./title-line.ts";
 
 /**
  * 本文で番号を指した図・表・付録（図3、Table 2、Appendix B）と、それを行の頭に書いた所（キャプション、見出し）。
@@ -79,16 +80,20 @@ const HEADING = /^[ \t]{0,3}#/u;
  */
 const CAPTION_AFTER = /^(?:[ \t]*$|[:：.．—–\-|()（）【】\]*_]|[ \t\u3000]+[^\p{Script=Hiragana}\p{Ll}])/u;
 
+/** A caption's title (Exhibit A to Merger Agreement): every long word capitalised, since Figure 3 shows the trend is a sentence. */
+export const CAPTION_TITLE: TitleNotion = { sentenceEnd: /[.!?。．？！]$/u, capitalised: true, edge: "any-whitespace" };
+
 /**
  * 行の頭に書いた番号。前にあるのが飾り（#、**、![、|、【、箇条書きの印）だけで、見出しの中か、後ろが題の区切りならキャプション。
- * 行の頭から始まる本文の文（図３の例では、…）は、図の置き場所ではない。
+ * 行の頭から始まる本文の文（図３の例では、…）は、図の置き場所ではない。題だけの行（Exhibit A to Merger Agreement）も見出し。
  */
 const isLabelled = (source: string, mention: Mention): boolean => {
   const lineStart = source.lastIndexOf("\n", mention.start - 1) + 1;
   const lineEnd = source.indexOf("\n", mention.end);
   const before = source.slice(lineStart, mention.start);
   if (LETTER.test(outsideMarkup(before))) return false;
-  return HEADING.test(before) || CAPTION_AFTER.test(source.slice(mention.end, lineEnd === -1 ? source.length : lineEnd));
+  const rest = source.slice(mention.end, lineEnd === -1 ? source.length : lineEnd);
+  return HEADING.test(before) || CAPTION_AFTER.test(rest) || isTitleLine(source, lineStart, lineEnd, rest, CAPTION_TITLE);
 };
 
 const LATIN_WORD = /^[A-Za-z]/u;

package/src/format-template.ts

@@ -0,0 +1,15 @@
+// A code block that holds only a format template: named placeholders joined by separators (<MAJOR>.<MINOR>.<PATCH>,
+// {year}-{month}). It shows the shape of a value, not code in any language, so it has no language to name. Pure.
+
+const PLACEHOLDER = /<[A-Za-z][\w-]*>|\{[A-Za-z][\w-]*\}/gu;
+const SEPARATORS = /^\s*[.,:;/_+\-|][\s.,:;/_+\-|]*$/u;
+/** A lone tag (`<div>`) or two tags side by side are markup; a template joins two or more parts with a separator. */
+const MIN_PARTS = 2;
+
+const isTemplateLine = (line: string): boolean => (line.match(PLACEHOLDER) ?? []).length >= MIN_PARTS && SEPARATORS.test(line.replace(PLACEHOLDER, ""));
+
+/** Whether every non-blank line is two or more placeholders joined by separators, with at least one such line. */
+export const isFormatTemplate = (lines: readonly string[]): boolean => {
+  const filled = lines.filter((line) => line.trim() !== "");
+  return filled.length > 0 && filled.every(isTemplateLine);
+};

package/src/formula-brackets.ts

@@ -0,0 +1,36 @@
+import type { Span } from "./plugin.ts";
+
+// Round brackets that group an arithmetic formula (賃金 × (1 - 時間 ÷ 月平均)) rather than set off an aside in prose.
+
+const OPENING = new Set(["(", "（"]);
+const CLOSING = new Set([")", "）"]);
+/** Inside the brackets: a multiplication or division sign, or a plus, minus or equals sign with a space on each side. Unspaced, - and = are hyphens and labels (A-1, 2026-04). */
+const OPERATOR_INSIDE = /[×÷]|[ \u3000][+＋\-−－=＝][ \u3000]/u;
+/** Just before the opening bracket, past spaces: a multiplication or division sign, or a spaced sign. */
+const OPERATOR_BEFORE = /(?:[×÷]|[ \u3000][+＋\-−－=＝])[ \u3000]*$/u;
+/** Just after the closing bracket, past spaces. An asterisk is not a sign here: in prose it marks a note (*（注）). */
+const OPERATOR_AFTER = /^[ \u3000]*(?:[×÷]|[+＋\-−－=＝][ \u3000])/u;
+
+type Scan = { readonly open: number[]; readonly pairs: Span[] };
+
+const step = (scan: Scan, char: string, at: number): Scan => {
+  if (OPENING.has(char)) scan.open.push(at);
+  const start = CLOSING.has(char) ? scan.open.pop() : undefined;
+  if (start !== undefined) scan.pairs.push({ start, end: at + 1 });
+  return scan;
+};
+
+const isFormula = (text: string, pair: Span): boolean =>
+  OPERATOR_INSIDE.test(text.slice(pair.start + 1, pair.end - 1)) ||
+  OPERATOR_BEFORE.test(text.slice(0, pair.start)) ||
+  OPERATOR_AFTER.test(text.slice(pair.end));
+
+/**
+ * The bracket pairs in text that belong to a formula, each span covering both brackets. Full-width and half-width brackets
+ * pair with each other. A bracket left unclosed or closing nothing is not part of any pair. Positions count from 0 in text.
+ */
+export const formulaBracketSpans = (text: string): Span[] =>
+  text
+    .split("")
+    .reduce<Scan>(step, { open: [], pairs: [] })
+    .pairs.filter((pair) => isFormula(text, pair));

package/src/html/html-markdown.ts

@@ -37,6 +37,7 @@ import { withoutFootnoteMarks } from "./html-footnote-marks.ts";
 import { withoutHeadingSelfLinks } from "./html-heading-links.ts";
 import { withLostParagraphTagsRestored } from "./html-lost-tags.ts";
 import { withoutReaderOnlyText } from "./html-reader-only.ts";
+import { withoutSiteChrome } from "./html-site-chrome.ts";
 import { withRunInHeadingsRead } from "./html-run-in-headings.ts";
 import { withoutWidgets } from "./html-widgets.ts";
 import { withPreformattedRestored, withPreformattedStashed } from "./html-preformatted.ts";
@@ -465,7 +466,7 @@ export const htmlToMarkdown = (html: string): string => {
   const uncommented = withLostParagraphTagsRestored(withAttributeMarkupEscaped(html).replace(/<!--[\s\S]*?-->|<\?[\s\S]*?\?>/gu, ""));
   const preformatted = withPreformattedStashed(withTablesRead(DROPPED.reduce(withoutElement, withoutRubyText(uncommented))));
   const kept = withoutFootnoteMarks(
-    withoutHeadingSelfLinks(withoutWidgets(withoutButtons(withoutReaderOnlyText(withoutHiddenElements(mainContent(preformatted.html)))))),
+    withoutHeadingSelfLinks(withoutWidgets(withoutButtons(withoutReaderOnlyText(withoutHiddenElements(mainContent(withoutSiteChrome(preformatted.html))))))),
   ).replace(MARKUP_SPACE, " ");
   const content = withoutClosingLinks(
     withoutLinkGroups(withoutNavigation(withoutIconMenus(withoutSiteHeader(withoutNavigationLandmarks(withoutClosingAddress(kept)))))),

package/src/html/html-site-chrome.ts

@@ -0,0 +1,47 @@
+// The site's own chrome around a page, read from what the markup says a block is rather than from
+// any site's wording. A landmark role (banner, search, contentinfo), a <search> element, a <header> that belongs to the
+// page rather than to a section, and a block named by a convention CMSs share: a breadcrumb trail (breadcrumb, pankuzu,
+// topicpath), a search box, the header or footer block (id or class exactly "header" or "footer"), and the U.S.
+// government's ".gov" banner (usa-banner). A block holding the page's <h1> is never chrome. Pure; a regular-expression
+// reading, not a parser.
+import { ATTRIBUTES, elementRanges, isInside, withoutElementsOpening, withoutRanges, type ElementRange } from "./html-elements.ts";
+
+const CHROME_ROLES = new Set(["banner", "search", "contentinfo"]);
+
+// A name segment, so that "pankuzu_wrap", "foot_pankuzu" and "cse-search-box" match and "research" does not.
+const CHROME_NAME = /(?:^|[-_])(?:breadcrumbs?|pankuzu|topic-?path|search)(?:[-_\d]|$)/iu;
+
+const REGION_NAME = /^(?:header|footer|usa-banner)$/iu;
+
+const NAMED_OPENING = /<([a-z][a-z0-9-]*)\b[^>]*\s(?:class|id|role)\s*=/giu;
+
+const attributeValue = (openTag: string, name: string): string => {
+  const value = new RegExp(String.raw`^<[a-z][a-z0-9-]*${ATTRIBUTES}\s+${name}\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))`, "iu").exec(openTag);
+  return value?.[1] ?? value?.[2] ?? value?.[3] ?? "";
+};
+
+const tokens = (value: string): string[] => value.split(/\s+/u).filter((token) => token !== "");
+
+const holdsTitle = (range: ElementRange): boolean => /<h1\b/iu.test(range.inner);
+
+/** A role is a list of tokens and the first one counts, as for role="main". */
+const hasChromeRole = (openTag: string): boolean => CHROME_ROLES.has((tokens(attributeValue(openTag, "role"))[0] ?? "").toLowerCase());
+
+const hasChromeName = (openTag: string): boolean =>
+  [...tokens(attributeValue(openTag, "class")), ...tokens(attributeValue(openTag, "id"))].some((name) => CHROME_NAME.test(name) || REGION_NAME.test(name));
+
+/** A block the markup names or marks as the site's chrome, unless it holds the page's title. */
+export const isSiteChrome = (range: ElementRange): boolean => !holdsTitle(range) && (hasChromeRole(range.openTag) || hasChromeName(range.openTag));
+
+const SECTIONING = ["article", "section", "main", "aside", "nav"];
+
+/** A <header> inside an article or a section heads that part; one outside them is the page's banner. */
+const withoutPageHeaders = (html: string): string => {
+  const sections = SECTIONING.flatMap((tag) => elementRanges(html, tag));
+  const banners = elementRanges(html, "header").filter((header) => !holdsTitle(header) && !sections.some((section) => isInside(section, header)));
+  return withoutRanges(html, banners);
+};
+
+const withoutSearchElements = (html: string): string => withoutRanges(html, elementRanges(html, "search"));
+
+export const withoutSiteChrome = (html: string): string => withoutPageHeaders(withoutSearchElements(withoutElementsOpening(html, NAMED_OPENING, isSiteChrome)));

package/src/joined-view.ts

@@ -62,6 +62,7 @@ const sentenceInSource =
       text: text.slice(span.start, span.end),
       ...(sentence.tokens === undefined ? {} : { tokens: sentence.tokens.map(tokenInSource(view)) }),
       ...(sentence.embeddedLanguage === undefined ? {} : { embeddedLanguage: sentence.embeddedLanguage }),
+      ...(sentence.listItem === undefined ? {} : { listItem: sentence.listItem }),
     };
   };
 

package/src/kanji-compound-slip.ts

@@ -0,0 +1,98 @@
+// A long kanji compound the document writes several times, and once or twice with one kanji changed (就職氷河期世代 ×5,
+// 就職永河期世代 ×2). Pure: the compounds come in with the words the analyser cut them into; which kanji may differ by
+// design (一, 第, 条) comes from a word list.
+
+/** One word of the analyser inside a compound, by kanji index: [start, end). `known` is false for a word not in its dictionary. */
+export type CompoundWord = {
+  readonly start: number;
+  readonly end: number;
+  readonly known: boolean;
+  /** The word reaches past the run (並びに after 受遺者); only its part inside is given. */
+  readonly crossesRun?: boolean;
+};
+
+export type CompoundUse = {
+  readonly compound: string;
+  readonly offset: number;
+  /** Absent where nothing was analysed. */
+  readonly words?: readonly CompoundWord[] | undefined;
+};
+
+type CompoundSlip = {
+  readonly written: string;
+  readonly usual: string;
+  readonly usualCount: number;
+  readonly offset: number;
+  readonly index: number;
+};
+
+const MIN_COMPOUND_LENGTH = 4;
+const MIN_USUAL_COUNT = 3;
+/** A slip copied once (a link and its twin) is still a slip; the usual form must then be written more than twice as often. */
+const MAX_SLIP_COUNT = 2;
+
+/** The one position where two compounds of the same length differ, or undefined when they differ in none or several. */
+export const singleDifference = (left: readonly string[], right: readonly string[]): number | undefined => {
+  if (left.length !== right.length) return undefined;
+  const differing = left.flatMap((char, index) => (char === right[index] ? [] : [index]));
+  return differing.length === 1 ? differing[0] : undefined;
+};
+
+/** The first and last kanji of a run are where a suffix (等, 者) or a conjunction written in kanji (及, 又) joins, not where a word is mistyped. */
+const isInside = (index: number, length: number): boolean => index > 0 && index < length - 1;
+
+const wordAt = (use: CompoundUse, index: number): CompoundWord | undefined => use.words?.find((word) => word.start <= index && index < word.end);
+
+const isLooseWithin = (word: CompoundWord | undefined, outer: CompoundWord): boolean =>
+  word !== undefined && word.crossesRun !== true && (!word.known || word.end - word.start === 1) && word.start >= outer.start && word.end <= outer.end;
+
+/**
+ * The analyser cannot read the changed word: where the usual form holds the kanji inside one word (氷河期), the slip
+ * falls apart there into single kanji or words it does not know (永 / 河 / 期). Returns the slip's kanji over that word.
+ */
+export const fallenWord = (slip: CompoundUse, usualUses: readonly CompoundUse[], index: number): string | undefined => {
+  const usualWord = usualUses.map((use) => wordAt(use, index)).find((word) => word !== undefined && word.end - word.start > 1);
+  if (usualWord === undefined) return undefined;
+  const positions = Array.from({ length: usualWord.end - usualWord.start }, (_, offset) => usualWord.start + offset);
+  const apart = positions.every((position) => isLooseWithin(wordAt(slip, position), usualWord));
+  return apart ? Array.from(slip.compound).slice(usualWord.start, usualWord.end).join("") : undefined;
+};
+
+/** The changed word written in another kanji run (質権 in 質権者は beside 当該質権者): a word the document knows. */
+const isWrittenElsewhere = (word: string, slip: string, uses: readonly CompoundUse[]): boolean =>
+  uses.some((use) => use.compound !== slip && use.compound.includes(word));
+
+type SlipContext = { readonly ignored: ReadonlySet<string>; readonly uses: readonly CompoundUse[] };
+
+type Usual = { readonly compound: string; readonly count: number; readonly uses: readonly CompoundUse[] };
+
+const countsOf = (uses: readonly CompoundUse[]): ReadonlyMap<string, number> =>
+  uses.reduce((counts, use) => counts.set(use.compound, (counts.get(use.compound) ?? 0) + 1), new Map<string, number>());
+
+const usualsOf = (uses: readonly CompoundUse[], counts: ReadonlyMap<string, number>): Usual[] =>
+  [...counts]
+    .filter(([compound, count]) => count >= MIN_USUAL_COUNT && Array.from(compound).length >= MIN_COMPOUND_LENGTH)
+    .map(([compound, count]) => ({ compound, count, uses: uses.filter((use) => use.compound === compound) }));
+
+const slipAgainst = (use: CompoundUse, count: number, usual: Usual, context: SlipContext): CompoundSlip | undefined => {
+  const [written, other] = [Array.from(use.compound), Array.from(usual.compound)];
+  const index = usual.count > count * 2 ? singleDifference(written, other) : undefined;
+  if (index === undefined || !isInside(index, written.length)) return undefined;
+  if (context.ignored.has(written[index] ?? "") || context.ignored.has(other[index] ?? "")) return undefined;
+  const fallen = fallenWord(use, usual.uses, index);
+  if (fallen === undefined || isWrittenElsewhere(fallen, use.compound, context.uses)) return undefined;
+  return { written: use.compound, usual: usual.compound, usualCount: usual.count, offset: use.offset, index };
+};
+
+/** Each compound written once or twice that differs in one kanji from a compound of the same length written at least three times. */
+export const compoundSlipsIn = (uses: readonly CompoundUse[], ignored: ReadonlySet<string>): CompoundSlip[] => {
+  const counts = countsOf(uses);
+  const usuals = usualsOf(uses, counts);
+  if (usuals.length === 0) return [];
+  return uses.flatMap((use) => {
+    const count = counts.get(use.compound) ?? 0;
+    if (count > MAX_SLIP_COUNT || Array.from(use.compound).length < MIN_COMPOUND_LENGTH) return [];
+    const slips = usuals.flatMap((usual) => slipAgainst(use, count, usual, { ignored, uses }) ?? []);
+    return slips.length === 1 ? slips : [];
+  });
+};

package/src/latin-company-form.ts

@@ -0,0 +1,38 @@
+import type { Span } from "./plugin.ts";
+
+const FULLWIDTH_OFFSET = 0xfee0;
+const LATIN_LETTER = /[A-Za-z]/u;
+const SPACE = "[\\s\\u3000]";
+
+/**
+ * 形は名前の後ろに立つ。前は大文字か数字で始まる名前の語か括弧で、あいだは空白か「,」だけ（AOBA INC.、AOBA,INC.、
+ * (THAILAND) CO.,LTD.）。語の中（Zinc の inc）、略語の中（U.S.A. の S.A.）、名前の無い所（the LLC）の同じ字は形ではない。
+ */
+const NAME_WORD = "(?<![A-Za-zＡ-Ｚａ-ｚ])[A-ZＡ-Ｚ0-9０-９][A-Za-zＡ-Ｚａ-ｚ0-9０-９&＆]*";
+const BEFORE = `(?<=${NAME_WORD}(?:${SPACE}|[,，]${SPACE}?)|[)）]${SPACE}?)`;
+const AFTER = "(?![A-Za-zＡ-Ｚａ-ｚ0-9０-９&＆])";
+
+const escaped = (char: string): string => char.replace(/[.*+?^${}()|[\]\\]/u, "\\$&");
+
+const widths = (char: string): string[] => [char, String.fromCodePoint((char.codePointAt(0) ?? 0) + FULLWIDTH_OFFSET)];
+
+/** 一字の形。英字は大文字・小文字と全角・半角のどれでも、点と「,」は全角・半角のどちらでも合う。空白は無くても合う。 */
+const charPattern = (char: string): string => {
+  if (char === " ") return `${SPACE}?`;
+  if (LATIN_LETTER.test(char)) return `[${[char.toUpperCase(), char.toLowerCase()].flatMap(widths).join("")}]`;
+  return /[.,]/u.test(char) ? `[${widths(char).map(escaped).join("")}]` : escaped(char);
+};
+
+/** 形は点まで書いたとおりに照らす。点の無い LTD（the LTD fund）は形と読まない。英字で終わる形（LLC）は語の終わりで閉じる。 */
+const formPattern = (form: string): string => [...form].map(charPattern).join("") + (LATIN_LETTER.test(form.slice(-1)) ? AFTER : "");
+
+/** 形の語彙表を、一つの正規表現にする。長い形を先に試す。名前の語は大文字で始まるので、大文字・小文字は形の側だけで問わない。 */
+export const latinCompanyFormRegExp = (forms: readonly string[]): RegExp | undefined => {
+  if (forms.length === 0) return undefined;
+  const longestFirst = [...forms].sort((a, b) => b.length - a.length);
+  return new RegExp(`${BEFORE}(?:${longestFirst.map(formPattern).join("|")})`, "gu");
+};
+
+/** text の中の、英字の会社の名前の後ろに立つ形（CO.，LTD.、Inc.）の位置。大文字・小文字と全角・半角を問わない。 */
+export const latinCompanyFormSpans = (text: string, pattern: RegExp | undefined): Span[] =>
+  pattern === undefined ? [] : [...text.matchAll(pattern)].map((match) => ({ start: match.index, end: match.index + match[0].length }));

package/src/latin-word-boundary.ts

@@ -0,0 +1,46 @@
+import type { Boundary } from "./orthography.ts";
+import type { Span, Token } from "./plugin.ts";
+
+// 英字の語の側で決まる、空け方の好みでない境目。pure: 語は語彙表から受け取る。
+
+/** 英字の語を作る字。vs. の点も語の一部。 */
+const LATIN_WORD_CHAR = /[A-Za-z.]/u;
+const LATIN_WORD_AFTER = /^[A-Za-z.]+/u;
+const JAPANESE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}ー]/u;
+
+/** 境目の英字の側の語の範囲（「名前 or 任意」の or）。 */
+const latinWordAt = (text: string, boundary: Boundary): Span => {
+  if (LATIN_WORD_CHAR.test(text[boundary.offset - 1] ?? "")) {
+    const reversed = Array.from(text.slice(0, boundary.offset)).toReversed().join("");
+    return { start: boundary.offset - (LATIN_WORD_AFTER.exec(reversed)?.[0].length ?? 0), end: boundary.offset };
+  }
+  const start = boundary.offset + (boundary.spaced ? 1 : 0);
+  return { start, end: start + (LATIN_WORD_AFTER.exec(text.slice(start))?.[0].length ?? 0) };
+};
+
+/** 語の前後が空白一つで、その外がどちらも日本語の字。 */
+const standsBetweenJapanese = (text: string, word: Span): boolean =>
+  text[word.start - 1] === " " && text[word.end] === " " && JAPANESE.test(text[word.start - 2] ?? "") && JAPANESE.test(text[word.end + 1] ?? "");
+
+/**
+ * 日本語の句と句をつなぐ英語の語（「名前 or メッセージ」「甲案 vs. 乙案」の or、vs.）の前後の空白。or を詰めて書くことは無く、
+ * 語を区切る空白で、日本語と英字のあいだの空け方の票ではない。片側が英字の語（A or 任意、名前 or API）なら数える。
+ * joiners は語彙表 latin-joiner の語（小文字）。
+ */
+export const isJoinerSpace = (text: string, boundary: Boundary, joiners: ReadonlySet<string>): boolean => {
+  if (boundary.kind !== "letter" || !boundary.spaced) return false;
+  const word = latinWordAt(text, boundary);
+  return joiners.has(text.slice(word.start, word.end).toLowerCase()) && standsBetweenJapanese(text, word);
+};
+
+/**
+ * 英字の語に付けて一つの語にする日本語の接尾の語（PDF版、IT化、Apple製）の前の境目。空けて書くことはほとんど無く、
+ * 詰めたのは空け方の好みではない。語の頭の字が同じでも別の語（製品、型番）なら数える。suffixes は語彙表 latin-bound-suffix の語。
+ * tokens は文書全体の座標で、base は text の先頭の位置。
+ */
+export const isBoundSuffix = (text: string, boundary: Boundary, tokens: readonly Token[] | undefined, base: number, suffixes: ReadonlySet<string>): boolean => {
+  if (tokens === undefined || boundary.kind !== "letter" || !LATIN_WORD_CHAR.test(text[boundary.offset - 1] ?? "")) return false;
+  const start = base + boundary.offset + (boundary.spaced ? 1 : 0);
+  const suffix = tokens.find((token) => token.span.start === start);
+  return suffix !== undefined && suffixes.has(suffix.surface);
+};

package/src/lexicon-genres.ts

@@ -1,7 +1,7 @@
 import type { Lexicon } from "./plugin.ts";
 
 /** A genre is in a target when it is the target or one of its genres (legal/contract is in legal). */
-const isIn = (genre: string, target: string): boolean => genre === target || genre.startsWith(`${target}/`);
+export const isIn = (genre: string, target: string): boolean => genre === target || genre.startsWith(`${target}/`);
 
 /**
  * The entries of a word list that hold for the document's genre: an entry without genres holds everywhere, one with genres only

package/src/link-shape.ts

@@ -60,3 +60,22 @@ export const isFileNameAlt = (alt: string, prefixes: readonly string[]): boolean
   const lowered = written.toLowerCase();
   return prefixes.some((prefix) => lowered.startsWith(prefix.toLowerCase()) && COUNTER.test(written.slice(prefix.length)));
 };
+
+/** A list item's marker before a line's text (`- `, `* `, `1. `). */
+const LIST_MARKER = /^\s*(?:[-*+]|\d{1,3}[.)])\s+/u;
+/**
+ * A field's label: a few words in bold (`**Licence text**`, a colon after it or not), or ending in a colon (`Website:`).
+ * Sentence punctuation or a comma in it makes it prose (`For example, visit:`).
+ */
+const BOLD_LABEL = /^(?:\*\*[^*\n.!?。！？,，、:：]{1,40}\*\*|__[^_\n.!?。！？,，、:：]{1,40}__)\s*[:：]?$/u;
+const COLON_LABEL = /^[^.!?。！？,，、:：]{1,40}[:：]$/u;
+
+/**
+ * Whether a URL is the value of a field (`**Licence text** https://…`, `Website: https://…`): on its line only a label comes
+ * before it and nothing after. There the address is the content, as in a table cell, so writing it bare is right.
+ */
+export const isFieldValueUrl = (lineBefore: string, lineAfter: string): boolean => {
+  if (lineAfter.trim() !== "") return false;
+  const label = lineBefore.replace(LIST_MARKER, "").trim();
+  return BOLD_LABEL.test(label) || COLON_LABEL.test(label);
+};