chaffjs 0.17.0 → 0.18.0

package/src/detectors/list-item.ts

@@ -0,0 +1,122 @@
+import type { Token } from "../plugin.ts";
+import { APPOSITIVE_ANCHOR, NOMINAL, verbFormOf, type Columns, type ItemScope } from "./list-sentence.ts";
+import { countIn, firstIn, lastIn, lowerBound, type TokenRange } from "./token-column.ts";
+
+// 並びの項目への問い。項目は文の中の一続きの範囲で、and / or と同じ深さの読点を除いたもの。どの問いも範囲を読み直さず、
+// 文を一度読んで作った列（list-sentence.ts）から答える。
+
+type ColumnName = keyof Columns;
+
+const count = (scope: ItemScope, name: ColumnName, item: TokenRange): number => countIn(scope.sentence.column[name], item, scope.level);
+
+const tokenAt = (scope: ItemScope, index: number): Token | undefined => (index === -1 ? undefined : scope.sentence.tokens[index]);
+
+export const firstAt = (scope: ItemScope, name: ColumnName, item: TokenRange): number => firstIn(scope.sentence.column[name], item, scope.level);
+
+const first = (scope: ItemScope, name: ColumnName, item: TokenRange): Token | undefined => tokenAt(scope, firstAt(scope, name, item));
+
+const last = (scope: ItemScope, name: ColumnName, item: TokenRange): Token | undefined =>
+  tokenAt(scope, lastIn(scope.sentence.column[name], item, scope.level));
+
+export const hasContent = (scope: ItemScope, item: TokenRange): boolean => count(scope, "content", item) > 0;
+
+export const contentCount = (scope: ItemScope, item: TokenRange): number => count(scope, "content", item);
+
+export const firstContent = (scope: ItemScope, item: TokenRange): Token | undefined => first(scope, "content", item);
+
+export const lastContent = (scope: ItemScope, item: TokenRange): Token | undefined => last(scope, "content", item);
+
+const shapeOfHead = (head: Token | undefined): string | undefined => {
+  if (head === undefined) return undefined;
+  return NOMINAL.has(head.pos) ? "NOMINAL" : head.pos;
+};
+
+/** 冠詞や引用符を飛ばした、項目の頭の品詞。the parser と an exporter と samples を同じ形と見る。 */
+export const shapeOf = (scope: ItemScope, item: TokenRange): string | undefined => shapeOfHead(firstContent(scope, item));
+
+/** 副詞を飛ばした頭の形。explain and justify と ultimately ensure は同じ動詞の項目。 */
+export const leadShape = (scope: ItemScope, item: TokenRange): string | undefined => shapeOfHead(first(scope, "contentNotAdverb", item));
+
+export const openingOf = (scope: ItemScope, item: TokenRange): Token | undefined => first(scope, "open", item);
+
+export const isParticiple = (token: Token): boolean => token.features?.["VerbForm"] === "Part";
+
+export const openingKind = (scope: ItemScope, item: TokenRange): string | undefined => {
+  const opening = openingOf(scope, item);
+  return opening !== undefined && isParticiple(opening) ? "PARTICIPLE" : shapeOf(scope, item);
+};
+
+export const hasVerb = (scope: ItemScope, item: TokenRange): boolean => count(scope, "verbal", item) > 0;
+
+export const hasListConjunction = (scope: ItemScope, item: TokenRange): boolean => count(scope, "conjunction", item) > 0;
+
+export const hasAdposition = (scope: ItemScope, item: TokenRange): boolean => count(scope, "adposition", item) > 0;
+
+export const hasDeterminer = (scope: ItemScope, item: TokenRange): boolean => count(scope, "determiner", item) > 0;
+
+/** 主語と述語のある項目。the team fixed the bug / these are crucial。gets us more は述語だけ。頭は冠詞か名詞なので、動詞はその後ろ。 */
+export const isClause = (scope: ItemScope, item: TokenRange): boolean => {
+  const opening = openingOf(scope, item);
+  if (opening === undefined || !(opening.pos === "DET" || NOMINAL.has(opening.pos))) return false;
+  return hasVerb(scope, item);
+};
+
+/** 頭の語のあとに前置詞の句を連れた項目（updates on the Google Doc）。 */
+export const hasPrepositionalTail = (scope: ItemScope, item: TokenRange): boolean => {
+  const head = firstAt(scope, "content", item);
+  return hasAdposition(scope, { start: head === -1 ? item.start : head + 1, end: item.end });
+};
+
+/** 項目の頭の分詞とその形。副詞は飛ばす（originally written）。引用符の中の語（‘modelling’）は語の例なので分詞と読まない。 */
+export const participleOpening = (scope: ItemScope, item: TokenRange): { readonly at: number; readonly form: string } | undefined => {
+  const at = firstAt(scope, "notPunctuationOrAdverb", item);
+  const opening = tokenAt(scope, at);
+  if (opening === undefined) return undefined;
+  const form = scope.sentence.words.participle.has(opening.surface.toLowerCase()) ? "Ger" : verbFormOf(opening);
+  return form === undefined ? undefined : { at, form };
+};
+
+export const hasParticiple = (scope: ItemScope, item: TokenRange, form: string): boolean => count(scope, form === "Ger" ? "gerund" : "participle", item) > 0;
+
+/** 名詞にかかる語。解析器は分詞を過去形とも読む（registered or certified mail の certified）ので、動詞も数える。 */
+const MODIFIER = new Set(["ADJ", "VERB"]);
+
+export const isModifier = (token: Token | undefined): boolean => token?.pos === "ADJ" || (token !== undefined && isParticiple(token));
+
+/** 修飾語と名詞だけの句（certified mail / artificial flavor）。冠詞や目的語を連れた動詞（adopt their resources）は違う。 */
+export const isModifiedNoun = (scope: ItemScope, item: TokenRange): boolean => {
+  const headAt = firstAt(scope, "open", item);
+  const tail = { start: headAt === -1 ? item.end : headAt + 1, end: item.end };
+  const tailLength = count(scope, "open", tail);
+  const nounPhrase = tailLength > 0 && tailLength === count(scope, "nounPhraseTail", tail);
+  return MODIFIER.has(tokenAt(scope, headAt)?.pos ?? "") && nounPhrase && APPOSITIVE_ANCHOR.has(last(scope, "open", tail)?.pos ?? "");
+};
+
+export const isPluralNounPhrase = (scope: ItemScope, item: TokenRange): boolean =>
+  count(scope, "content", item) === count(scope, "nounPhraseTail", item) && lastContent(scope, item)?.features?.["Number"] === "Plur";
+
+export const firstNoun = (scope: ItemScope, item: TokenRange): Token | undefined => first(scope, "noun", item);
+
+export const lastNoun = (scope: ItemScope, item: TokenRange): Token | undefined => last(scope, "noun", item);
+
+/** and / or の後ろの項目。次の読点か節の切れ目まで。 */
+export const itemAfter = (scope: ItemScope, at: number): TokenRange => {
+  const rest = { start: at + 1, end: scope.sentence.tokens.length };
+  const end = firstIn(scope.sentence.column.itemEnd, rest);
+  return { start: rest.start, end: end === -1 ? rest.end : end };
+};
+
+/**
+ * 項目の中で、and / or と同じ深さにある最後の例の句（such as / e.g.）の直後の位置。無ければ -1。句は項目の中で終わるものだけ。
+ * 項目の終わりをまたぐ句は、項目の最後の数語から始まるものだけなので、後ろから見て数個で決まる。
+ */
+export const exampleEnd = (scope: ItemScope, item: TokenRange): number => {
+  const starts = scope.sentence.exampleStarts.get(scope.level) ?? [];
+  const within = (from: number): number => {
+    const start = starts[from];
+    if (start === undefined || start < item.start) return -1;
+    const end = (scope.sentence.exampleEnds[start] ?? []).find((phraseLast) => phraseLast < item.end);
+    return end === undefined ? within(from - 1) : end + 1;
+  };
+  return within(lowerBound(starts, item.end) - 1);
+};

package/src/detectors/list-items.ts

@@ -0,0 +1,163 @@
+import {
+  exampleEnd,
+  firstContent,
+  hasContent,
+  hasPrepositionalTail,
+  hasVerb,
+  isClause,
+  lastContent,
+  openingOf,
+  participleOpening,
+  shapeOf,
+} from "./list-item.ts";
+import { isListConjunction, scopeOf, VERBAL, type ItemScope, type ListSentence } from "./list-sentence.ts";
+import { lowerBound, type TokenRange } from "./token-column.ts";
+
+// and / or の前の項目を、節の頭から読点ごとに切った並び。and / or ごとに節の頭から切り直すと、and の多い文で語数の二乗になる。
+// ここでは深さごとに一度だけ左から読み、読点で閉じた項目は固めて、並び全体への問い（どれも節か、最後に例の句がある項目はどれか）
+// に答えるための表を足していく。and / or のたびに作るのは、まだ閉じていない最後の 1、2 項目だけ。
+
+/** 項目ごとに一度だけ答えを覚えておく問い。 */
+export type Flag = "closed" | "example" | "participle" | "verbalShape" | "hasVerb" | "isClause" | "prepositionalTail" | "notNounHead" | "notProperNounHead";
+
+type FlagTest = (scope: ItemScope, item: TokenRange) => boolean;
+
+export const FLAG_TESTS: Readonly<Record<Flag, FlagTest>> = {
+  closed: (scope, item) => isListConjunction(openingOf(scope, item)),
+  example: (scope, item) => exampleEnd(scope, item) !== -1,
+  participle: (scope, item) => participleOpening(scope, item) !== undefined,
+  verbalShape: (scope, item) => VERBAL.has(shapeOf(scope, item) ?? ""),
+  hasVerb,
+  isClause,
+  prepositionalTail: hasPrepositionalTail,
+  notNounHead: (scope, item) => firstContent(scope, item)?.pos !== "NOUN",
+  notProperNounHead: (scope, item) => firstContent(scope, item)?.pos !== "PROPN",
+};
+
+const FLAGS = Object.keys(FLAG_TESTS).filter((name): name is Flag => Object.hasOwn(FLAG_TESTS, name));
+
+type PerFlag<T> = Record<Flag, T>;
+
+const perFlag = <T>(make: () => T): PerFlag<T> => ({
+  closed: make(),
+  example: make(),
+  participle: make(),
+  verbalShape: make(),
+  hasVerb: make(),
+  isClause: make(),
+  prepositionalTail: make(),
+  notNounHead: make(),
+  notProperNounHead: make(),
+});
+
+/** 閉じた項目と、その問いの答え。trueAt は答えが真の項目の位置、lastFalse は最後に偽だった位置（WithContent は中身のある項目だけ）。 */
+type Frozen = {
+  readonly items: TokenRange[];
+  readonly content: boolean[];
+  readonly contentAt: number[];
+  readonly shapes: (string | undefined)[];
+  /** 中身のある項目のうち、次の中身のある項目と頭の形が違う最後のもの。無ければ -1。 */
+  lastShapeChange: number;
+  readonly flags: PerFlag<boolean[]>;
+  readonly trueAt: PerFlag<number[]>;
+  readonly trueWithContentAt: PerFlag<number[]>;
+  readonly lastFalse: PerFlag<number>;
+  readonly lastFalseWithContent: PerFlag<number>;
+};
+
+const emptyFrozen = (): Frozen => ({
+  items: [],
+  content: [],
+  contentAt: [],
+  shapes: [],
+  lastShapeChange: -1,
+  flags: perFlag((): boolean[] => []),
+  trueAt: perFlag((): number[] => []),
+  trueWithContentAt: perFlag((): number[] => []),
+  lastFalse: perFlag(() => -1),
+  lastFalseWithContent: perFlag(() => -1),
+});
+
+const recordFlags = (frozen: Frozen, scope: ItemScope, item: TokenRange, content: boolean): void => {
+  const index = frozen.items.length - 1;
+  FLAGS.forEach((flag) => {
+    const holds = FLAG_TESTS[flag](scope, item);
+    frozen.flags[flag].push(holds);
+    if (holds) frozen.trueAt[flag].push(index);
+    else frozen.lastFalse[flag] = index;
+    if (holds && content) frozen.trueWithContentAt[flag].push(index);
+    if (!holds && content) frozen.lastFalseWithContent[flag] = index;
+  });
+};
+
+const freeze = (frozen: Frozen, scope: ItemScope, item: TokenRange): void => {
+  const content = hasContent(scope, item);
+  const shape = shapeOf(scope, item);
+  const previous = frozen.contentAt.at(-1);
+  frozen.items.push(item);
+  frozen.content.push(content);
+  frozen.shapes.push(shape);
+  if (content && previous !== undefined && frozen.shapes[previous] !== shape) frozen.lastShapeChange = previous;
+  if (content) frozen.contentAt.push(frozen.items.length - 1);
+  recordFlags(frozen, scope, item, content);
+};
+
+/** 形容詞のあとの読点は、名詞の前で形容詞を重ねているだけのことが多い（the long, winding bridge）。次の項目が名詞で終わるならつなげ直す。 */
+const stacks = (scope: ItemScope, previous: TokenRange, item: TokenRange): boolean =>
+  lastContent(scope, previous)?.pos === "ADJ" && lastContent(scope, item)?.pos !== "ADJ";
+
+/** 1 つの深さの並び。start は節の頭、rawStart は読点でまだ閉じていない項目の頭、pending は次の項目とつながりうる最後の項目。 */
+type LevelItems = {
+  readonly start: number;
+  readonly scope: ItemScope;
+  readonly commas: readonly number[];
+  nextComma: number;
+  rawStart: number;
+  pending: TokenRange | undefined;
+  readonly frozen: Frozen;
+};
+
+/** 節の頭から and / or までの項目。閉じた項目の表と、まだ閉じていない最後の 1、2 項目。 */
+export type ItemList = { readonly scope: ItemScope; readonly frozen: Frozen; readonly tail: readonly TokenRange[] };
+
+const addItem = (items: LevelItems, item: TokenRange): void => {
+  if (item.end <= item.start) return;
+  if (items.pending !== undefined && stacks(items.scope, items.pending, item)) {
+    items.pending = { start: items.pending.start, end: item.end };
+    return;
+  }
+  if (items.pending !== undefined) freeze(items.frozen, items.scope, items.pending);
+  items.pending = item;
+};
+
+const tailOf = (scope: ItemScope, pending: TokenRange | undefined, live: TokenRange | undefined): TokenRange[] => {
+  if (live === undefined) return pending === undefined ? [] : [pending];
+  if (pending === undefined) return [live];
+  return stacks(scope, pending, live) ? [{ start: pending.start, end: live.end }] : [pending, live];
+};
+
+/** 深さごとの並びを、and / or の位置 at まで読み進める。at は呼ぶたびに大きくなる。返した並びは、次に読み進めるまでのもの。 */
+export const listReader = (sentence: ListSentence): ((at: number) => ItemList) => {
+  const byLevel = new Map<number, LevelItems>();
+  const opened = (level: number, start: number): LevelItems => {
+    const known = byLevel.get(level);
+    if (known !== undefined && known.start === start) return known;
+    const commas = sentence.commas.get(level) ?? [];
+    const scope = scopeOf(sentence, level);
+    const fresh = { start, scope, commas, nextComma: lowerBound(commas, start), rawStart: start, pending: undefined, frozen: emptyFrozen() };
+    byLevel.set(level, fresh);
+    return fresh;
+  };
+  return (at) => {
+    const level = sentence.depths[at] ?? 0;
+    const items = opened(level, (sentence.column.clauseBreak.previous[at] ?? -1) + 1);
+    const reached = lowerBound(items.commas, at);
+    items.commas.slice(items.nextComma, reached).forEach((comma) => {
+      addItem(items, { start: items.rawStart, end: comma });
+      items.rawStart = comma + 1;
+    });
+    items.nextComma = reached;
+    const live = items.rawStart < at ? { start: items.rawStart, end: at } : undefined;
+    return { scope: items.scope, frozen: items.frozen, tail: tailOf(items.scope, items.pending, live) };
+  };
+};

package/src/detectors/list-lead-in.ts

@@ -0,0 +1,33 @@
+import type { BulletList, Sentence, Span } from "../plugin.ts";
+import { endsWithColon } from "./lead-in.ts";
+
+/** Between a lead-in and its list: only blank space. A thematic break (`***`) separates them; bold marks sit inside the sentence. */
+const ONLY_BLANKS = /^\s*$/u;
+
+/** How many sentences end at or before an offset. Sentences are in document order, so a binary search. */
+const countEndingBy = (sentences: readonly Sentence[], offset: number): number => {
+  const search = (low: number, high: number): number => {
+    if (low >= high) return low;
+    const middle = Math.floor((low + high) / 2);
+    return (sentences[middle]?.span.end ?? Infinity) <= offset ? search(middle + 1, high) : search(low, middle);
+  };
+  return search(0, sentences.length);
+};
+
+/** The sentence that ends last before the list starts, if nothing but blanks lies between them. */
+const sentenceRightBefore = (sentences: readonly Sentence[], list: BulletList, source: string): Sentence | undefined => {
+  const before = sentences[countEndingBy(sentences, list.span.start) - 1];
+  return before !== undefined && ONLY_BLANKS.test(source.slice(before.span.end, list.span.start)) ? before : undefined;
+};
+
+const insideAny = (spans: readonly Span[], offset: number): boolean => spans.some((span) => span.start <= offset && offset < span.end);
+
+/**
+ * Prose sentences ending in a colon that hand the reader straight to a list (「以下の通りです：」, "Here's what you need:").
+ * An item of a list is not prose: 「例: `code`」 ends in a colon once its code is masked, and it leads nowhere.
+ */
+export const colonLeadIns = (sentences: readonly Sentence[], lists: readonly BulletList[], listSpans: readonly Span[], source: string): Sentence[] =>
+  lists.flatMap((list) => {
+    const before = sentenceRightBefore(sentences, list, source);
+    return before !== undefined && endsWithColon(before.text) && !insideAny(listSpans, before.span.start) ? [before] : [];
+  });

package/src/detectors/list-sentence.ts

@@ -0,0 +1,157 @@
+import type { Lexicon, Token } from "../plugin.ts";
+import { citedTitles } from "./cited-title.ts";
+import { columnOf, type Column } from "./token-column.ts";
+
+/**
+ * 並びを読むための語彙表。participle は解析器が分詞と読まないが読点のあとで分詞の句を始める語（meaning）、
+ * example は例を挙げる句（such as, e.g.）、pair は 2 つだけを結ぶ語とその接続詞（between and / either or）。
+ */
+export type ListWords = { readonly participle: ReadonlySet<string>; readonly example: Lexicon; readonly pair: ReadonlySet<string> };
+
+const LIST_CONJUNCTION: ReadonlySet<string> = new Set(["and", "or"]);
+
+export const isListConjunction = (token: Token | undefined): boolean => LIST_CONJUNCTION.has(token?.surface.toLowerCase() ?? "");
+
+/** 並列はこれをまたがない。セミコロンの前後は別の節。 */
+const CLAUSE_BREAK = new Set([";", ":", "—"]);
+
+/** 各 token の前で閉じていない括弧の数。括弧の中の読点（external users (e.g., guests), and ...）は外の並列を切らない。 */
+const PAREN_STEP: ReadonlyMap<string, number> = new Map([
+  ["(", 1],
+  [")", -1],
+]);
+
+export const depthsOf = (tokens: readonly Token[]): number[] => {
+  let open = 0;
+  return tokens.map((token) => {
+    const before = open;
+    open = Math.max(0, open + (PAREN_STEP.get(token.surface) ?? 0));
+    return before;
+  });
+};
+
+/** 冠詞や引用符を飛ばした、項目の頭の品詞。the parser と an exporter と samples を同じ形と見る。 */
+export const NOMINAL = new Set(["NOUN", "PRON", "PROPN", "NUM", "ADJ"]);
+export const VERBAL = new Set(["VERB", "AUX"]);
+const NOUN_PHRASE_TAIL = new Set(["ADJ", "NOUN", "PROPN"]);
+export const APPOSITIVE_ANCHOR = new Set(["NOUN", "PROPN"]);
+
+export const isContent = (token: Token): boolean => token.pos !== "DET" && token.pos !== "PUNCT" && token.pos !== "X";
+
+const isOpen = (token: Token): boolean => token.pos !== "PUNCT" && token.pos !== "X";
+
+/** -ing 形（VerbForm=Ger）と過去分詞（Part）。filling the gaps と sentenced to prison は並べても並列にならない。 */
+const VERB_FORMS = new Set(["Ger", "Part"]);
+
+export const verbFormOf = (token: Token): string | undefined => {
+  const form = token.features?.["VerbForm"];
+  return form !== undefined && VERB_FORMS.has(form) ? form : undefined;
+};
+
+/** 同じ深さにある語の位置を、深さごとに左から。 */
+const byDepth = (depths: readonly number[], test: (index: number) => boolean): ReadonlyMap<number, readonly number[]> =>
+  depths.reduce((found, depth, index) => {
+    if (!test(index)) return found;
+    const list = found.get(depth);
+    if (list === undefined) found.set(depth, [index]);
+    else list.push(index);
+    return found;
+  }, new Map<number, number[]>());
+
+const isComma = (token: Token): boolean => token.surface === ",";
+
+/** 項目への問いの列。読点は項目に入ったり入らなかったりするので、脇に置いて問いごとに数える（token-column.ts）。 */
+const columnsOf = (tokens: readonly Token[], depths: readonly number[]) => {
+  const commaDepth = (token: Token, index: number): number | undefined => (isComma(token) ? depths[index] : undefined);
+  const of = (test: (token: Token) => boolean): Column => columnOf(tokens, test, commaDepth);
+  return {
+    content: of(isContent),
+    open: of(isOpen),
+    verbal: of((token) => VERBAL.has(token.pos)),
+    adposition: of((token) => token.pos === "ADP"),
+    determiner: of((token) => token.pos === "DET"),
+    conjunction: of((token) => LIST_CONJUNCTION.has(token.surface.toLowerCase())),
+    gerund: of((token) => verbFormOf(token) === "Ger"),
+    participle: of((token) => verbFormOf(token) === "Part"),
+    nounPhraseTail: of((token) => NOUN_PHRASE_TAIL.has(token.pos)),
+    noun: of((token) => APPOSITIVE_ANCHOR.has(token.pos)),
+    contentNotAdverb: of((token) => isContent(token) && token.pos !== "ADV"),
+    notPunctuationOrAdverb: of((token) => token.pos !== "PUNCT" && token.pos !== "ADV"),
+    notComma: columnOf(tokens, (token) => !isComma(token)),
+    itemEnd: columnOf(tokens, (token) => isComma(token) || CLAUSE_BREAK.has(token.surface)),
+    clauseBreak: columnOf(tokens, (token) => CLAUSE_BREAK.has(token.surface)),
+  };
+};
+
+export type Columns = ReturnType<typeof columnsOf>;
+
+/** 文を一度だけ読んで作る、項目の問いに答えるための表。 */
+export type ListSentence = {
+  readonly tokens: readonly Token[];
+  readonly depths: readonly number[];
+  readonly words: ListWords;
+  readonly column: Columns;
+  /** 各位置から始まる例の句の、最後の語の位置。語彙表の順に。 */
+  readonly exampleEnds: readonly (readonly number[])[];
+  /** 例の句の始まる位置。深さごとに。 */
+  readonly exampleStarts: ReadonlyMap<number, readonly number[]>;
+  /** 2 つだけを結ぶ語の位置。組む接続詞ごと、深さごとに。 */
+  readonly pairOpeners: ReadonlyMap<string, ReadonlyMap<number, readonly number[]>>;
+  /** 読点の位置。深さごとに。 */
+  readonly commas: ReadonlyMap<number, readonly number[]>;
+  readonly inCitedTitle: (at: number) => boolean;
+};
+
+/**
+ * start から例の句の語を読んだときの、最後の語の位置。無ければ -1。項目は同じ深さの読点を持たないので、start と同じ深さの読点は
+ * 飛ばす。続いた読点は同じ深さなので、まとめて飛ぶ。
+ */
+const phraseEnd = (tokens: readonly Token[], depths: readonly number[], notComma: Column, start: number, words: readonly Token[]): number =>
+  words.reduce((at, word, offset) => {
+    if (at === -1) return -1;
+    const next = offset === 0 ? start : at + 1;
+    const kept = tokens[next]?.surface === "," && depths[next] === depths[start] ? (notComma.next[next] ?? tokens.length) : next;
+    return tokens[kept]?.surface === word.surface ? kept : -1;
+  }, start);
+
+const exampleEndsOf = (tokens: readonly Token[], depths: readonly number[], column: Columns, example: Lexicon): number[][] =>
+  tokens.map((token, start) =>
+    token.surface === ","
+      ? []
+      : example.flatMap(({ tokens: words = [] }) => {
+          const end = words.length > 0 ? phraseEnd(tokens, depths, column.notComma, start, words) : -1;
+          return end === -1 ? [] : [end];
+        }),
+  );
+
+const pairOpenersOf = (tokens: readonly Token[], depths: readonly number[], pair: ReadonlySet<string>): Map<string, ReadonlyMap<number, readonly number[]>> =>
+  new Map(
+    [...LIST_CONJUNCTION].map((conjunction) => [
+      conjunction,
+      byDepth(depths, (index) => tokens[index]?.surface !== "," && pair.has(`${tokens[index]?.surface.toLowerCase() ?? ""} ${conjunction}`)),
+    ]),
+  );
+
+/**
+ * 1 つの and / or の深さから見た文。その深さの読点は項目の区切りで、項目の中には無い。ほかの深さの読点（括弧の中）は項目の語。
+ */
+export type ItemScope = { readonly sentence: ListSentence; readonly level: number };
+
+export const scopeOf = (sentence: ListSentence, level: number): ItemScope => ({ sentence, level });
+
+export const listSentenceOf = (tokens: readonly Token[], words: ListWords, source: string): ListSentence => {
+  const depths = depthsOf(tokens);
+  const column = columnsOf(tokens, depths);
+  const exampleEnds = exampleEndsOf(tokens, depths, column, words.example);
+  return {
+    tokens,
+    depths,
+    words,
+    column,
+    exampleEnds,
+    exampleStarts: byDepth(depths, (index) => (exampleEnds[index]?.length ?? 0) > 0),
+    pairOpeners: pairOpenersOf(tokens, depths, words.pair),
+    commas: byDepth(depths, (index) => tokens[index]?.surface === ","),
+    inCitedTitle: citedTitles(tokens, source),
+  };
+};

package/src/detectors/list-view.ts

@@ -0,0 +1,157 @@
+import { hasContent, shapeOf } from "./list-item.ts";
+import { FLAG_TESTS, type Flag, type ItemList } from "./list-items.ts";
+import { lowerBound, type TokenRange } from "./token-column.ts";
+
+// 並びを切り直したもの。並びの頭を外す、途中の項目のあとから読む、中身の無い項目を除く、の 3 つだけで作れるので、
+// 「先頭に置いた項目（pre）+ 並びの from 番目から後ろ（contentOnly なら中身のある項目だけ）」で表す。項目の配列は作らない。
+
+export type View = { readonly list: ItemList; readonly pre: TokenRange | undefined; readonly from: number; readonly contentOnly: boolean };
+
+/** 並びの中の項目。j は並びの中の位置で、pre は -1。 */
+export type Member = { readonly item: TokenRange; readonly j: number };
+
+export const wholeList = (list: ItemList): View => ({ list, pre: undefined, from: 0, contentOnly: false });
+
+const frozenCount = (list: ItemList): number => list.frozen.items.length;
+
+const listCount = (list: ItemList): number => frozenCount(list) + list.tail.length;
+
+const itemAt = (list: ItemList, j: number): TokenRange | undefined => (j < frozenCount(list) ? list.frozen.items[j] : list.tail[j - frozenCount(list)]);
+
+const contentAt = (list: ItemList, j: number): boolean => {
+  const item = itemAt(list, j);
+  if (j < frozenCount(list)) return list.frozen.content[j] ?? false;
+  return item !== undefined && hasContent(list.scope, item);
+};
+
+const kept = (view: View, j: number): boolean => !view.contentOnly || contentAt(view.list, j);
+
+const flagAt = (list: ItemList, j: number, flag: Flag): boolean => {
+  const item = itemAt(list, j);
+  if (j < frozenCount(list)) return list.frozen.flags[flag][j] ?? false;
+  return item !== undefined && FLAG_TESTS[flag](list.scope, item);
+};
+
+const tailPositions = (list: ItemList, from: number): number[] => list.tail.map((__item, offset) => frozenCount(list) + offset).filter((j) => j >= from);
+
+const member = (list: ItemList, j: number): Member | undefined => {
+  const item = j === -1 ? undefined : itemAt(list, j);
+  return item === undefined ? undefined : { item, j };
+};
+
+/** from 以後の、閉じた項目の最初の位置。無ければ -1。 */
+const firstFrozenFrom = (view: View, from: number): number => {
+  const { frozen } = view.list;
+  if (view.contentOnly) return frozen.contentAt[lowerBound(frozen.contentAt, from)] ?? -1;
+  return from < frozen.items.length ? from : -1;
+};
+
+/** from 以後の最初の項目の位置。無ければ -1。 */
+const firstFrom = (view: View, from: number): number => {
+  const frozen = firstFrozenFrom(view, from);
+  if (frozen !== -1) return frozen;
+  return tailPositions(view.list, from).find((j) => kept(view, j)) ?? -1;
+};
+
+/** before より前の、view.from 以後の最後の項目の位置。無ければ -1。 */
+const lastBefore = (view: View, before: number): number => {
+  const { list } = view;
+  const fromTail = tailPositions(list, view.from)
+    .filter((j) => j < before && kept(view, j))
+    .at(-1);
+  if (fromTail !== undefined) return fromTail;
+  const end = Math.min(before, frozenCount(list));
+  const frozen = view.contentOnly ? (list.frozen.contentAt[lowerBound(list.frozen.contentAt, end) - 1] ?? -1) : end - 1;
+  return frozen >= view.from ? frozen : -1;
+};
+
+const preMember = (view: View): Member | undefined => (view.pre === undefined ? undefined : { item: view.pre, j: -1 });
+
+export const firstOf = (view: View): Member | undefined => preMember(view) ?? member(view.list, firstFrom(view, view.from));
+
+export const secondOf = (view: View): Member | undefined => {
+  const first = view.pre === undefined ? firstFrom(view, view.from) : view.from - 1;
+  return first === -1 ? undefined : member(view.list, firstFrom(view, first + 1));
+};
+
+export const lastOf = (view: View): Member | undefined => member(view.list, lastBefore(view, listCount(view.list))) ?? preMember(view);
+
+export const secondLastOf = (view: View): Member | undefined => {
+  const last = lastBefore(view, listCount(view.list));
+  if (last === -1) return undefined;
+  return member(view.list, lastBefore(view, last)) ?? preMember(view);
+};
+
+export const sizeOf = (view: View): number => {
+  const { list } = view;
+  const frozen = view.contentOnly ? list.frozen.contentAt.length - lowerBound(list.frozen.contentAt, view.from) : Math.max(0, frozenCount(list) - view.from);
+  const tail = tailPositions(list, view.from).filter((j) => kept(view, j)).length;
+  return (view.pre === undefined ? 0 : 1) + frozen + tail;
+};
+
+/** 先頭の項目を外す。 */
+export const withoutFirst = (view: View): View => {
+  if (view.pre !== undefined) return { ...view, pre: undefined };
+  const first = firstFrom(view, view.from);
+  return { ...view, from: first === -1 ? listCount(view.list) : first + 1 };
+};
+
+/** その項目より後ろ。 */
+export const after = (view: View, at: Member): View => ({ ...view, pre: undefined, from: at.j === -1 ? view.from : at.j + 1 });
+
+/** 先頭に項目を置いた、中身のある項目だけの並び。 */
+export const contentOnlyWith = (view: View, pre: TokenRange | undefined): View => ({
+  ...view,
+  pre: pre !== undefined && hasContent(view.list.scope, pre) ? pre : undefined,
+  contentOnly: true,
+});
+
+const holdsFor = (view: View, flag: Flag, item: TokenRange): boolean => FLAG_TESTS[flag](view.list.scope, item);
+
+/** 問いが真になる最後の項目。 */
+export const findLast = (view: View, flag: Flag): Member | undefined => {
+  const { list } = view;
+  const fromTail = tailPositions(list, view.from)
+    .filter((j) => kept(view, j) && flagAt(list, j, flag))
+    .at(-1);
+  if (fromTail !== undefined) return member(list, fromTail);
+  const frozen = (view.contentOnly ? list.frozen.trueWithContentAt : list.frozen.trueAt)[flag].at(-1) ?? -1;
+  if (frozen >= view.from) return member(list, frozen);
+  return view.pre !== undefined && holdsFor(view, flag, view.pre) ? preMember(view) : undefined;
+};
+
+/** 2 番目以後で、問いが真になる最初の項目。 */
+export const findAfterFirst = (view: View, flag: Flag): Member | undefined => {
+  const { list } = view;
+  const first = view.pre === undefined ? firstFrom(view, view.from) : view.from - 1;
+  if (first === -1) return undefined;
+  const found = view.contentOnly ? list.frozen.trueWithContentAt[flag] : list.frozen.trueAt[flag];
+  const frozen = found[lowerBound(found, first + 1)];
+  if (frozen !== undefined) return member(list, frozen);
+  return member(list, tailPositions(list, first + 1).find((j) => kept(view, j) && flagAt(list, j, flag)) ?? -1);
+};
+
+export const some = (view: View, flag: Flag): boolean => findLast(view, flag) !== undefined;
+
+export const every = (view: View, flag: Flag): boolean => {
+  const { list } = view;
+  if (view.pre !== undefined && !holdsFor(view, flag, view.pre)) return false;
+  if ((view.contentOnly ? list.frozen.lastFalseWithContent : list.frozen.lastFalse)[flag] >= view.from) return false;
+  return tailPositions(list, view.from).every((j) => !kept(view, j) || flagAt(list, j, flag));
+};
+
+/** 中身のある項目の頭の形が、1 種類以下か。 */
+export const sameShapes = (view: View): boolean => {
+  const { list } = view;
+  const { frozen } = list;
+  const lastFrozen = frozen.contentAt.at(-1) ?? -1;
+  if (lastFrozen >= view.from && frozen.lastShapeChange >= view.from) return false;
+  const shapes = new Set([
+    ...(view.pre === undefined || !hasContent(list.scope, view.pre) ? [] : [shapeOf(list.scope, view.pre)]),
+    ...(lastFrozen >= view.from ? [frozen.shapes[lastFrozen]] : []),
+    ...tailPositions(list, view.from)
+      .filter((j) => contentAt(list, j))
+      .map((j) => shapeOf(list.scope, itemAt(list, j) ?? { start: 0, end: 0 })),
+  ]);
+  return shapes.size <= 1;
+};