chaffjs 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/detectors/acronym-context.d.ts +17 -3
  2. package/dist/detectors/acronym-context.d.ts.map +1 -1
  3. package/dist/detectors/acronym-context.js +25 -32
  4. package/dist/detectors/acronym-context.js.map +1 -1
  5. package/dist/detectors/acronym-expansion.d.ts.map +1 -1
  6. package/dist/detectors/acronym-expansion.js +33 -6
  7. package/dist/detectors/acronym-expansion.js.map +1 -1
  8. package/dist/detectors/char-shape.d.ts.map +1 -1
  9. package/dist/detectors/char-shape.js +22 -36
  10. package/dist/detectors/char-shape.js.map +1 -1
  11. package/dist/detectors/conjugated-form.d.ts +3 -0
  12. package/dist/detectors/conjugated-form.d.ts.map +1 -0
  13. package/dist/detectors/conjugated-form.js +29 -0
  14. package/dist/detectors/conjugated-form.js.map +1 -0
  15. package/dist/detectors/enumerated-runs.d.ts +22 -0
  16. package/dist/detectors/enumerated-runs.d.ts.map +1 -0
  17. package/dist/detectors/enumerated-runs.js +53 -0
  18. package/dist/detectors/enumerated-runs.js.map +1 -0
  19. package/dist/detectors/middle-dot.d.ts +6 -0
  20. package/dist/detectors/middle-dot.d.ts.map +1 -0
  21. package/dist/detectors/middle-dot.js +9 -0
  22. package/dist/detectors/middle-dot.js.map +1 -0
  23. package/dist/detectors/orthography.d.ts.map +1 -1
  24. package/dist/detectors/orthography.js +19 -1
  25. package/dist/detectors/orthography.js.map +1 -1
  26. package/dist/detectors/place-run.d.ts +8 -0
  27. package/dist/detectors/place-run.d.ts.map +1 -0
  28. package/dist/detectors/place-run.js +54 -0
  29. package/dist/detectors/place-run.js.map +1 -0
  30. package/dist/detectors/polite-word.d.ts +4 -0
  31. package/dist/detectors/polite-word.d.ts.map +1 -0
  32. package/dist/detectors/polite-word.js +10 -0
  33. package/dist/detectors/polite-word.js.map +1 -0
  34. package/dist/detectors/register.d.ts +34 -0
  35. package/dist/detectors/register.d.ts.map +1 -0
  36. package/dist/detectors/register.js +64 -0
  37. package/dist/detectors/register.js.map +1 -0
  38. package/dist/detectors/roman-numeral.d.ts +6 -0
  39. package/dist/detectors/roman-numeral.d.ts.map +1 -0
  40. package/dist/detectors/roman-numeral.js +6 -0
  41. package/dist/detectors/roman-numeral.js.map +1 -0
  42. package/dist/detectors/sentence-ending.d.ts.map +1 -1
  43. package/dist/detectors/sentence-ending.js +28 -29
  44. package/dist/detectors/sentence-ending.js.map +1 -1
  45. package/dist/detectors/signals.d.ts.map +1 -1
  46. package/dist/detectors/signals.js +21 -7
  47. package/dist/detectors/signals.js.map +1 -1
  48. package/dist/detectors/structure.d.ts.map +1 -1
  49. package/dist/detectors/structure.js +2 -1
  50. package/dist/detectors/structure.js.map +1 -1
  51. package/dist/number-name.d.ts +10 -0
  52. package/dist/number-name.d.ts.map +1 -0
  53. package/dist/number-name.js +76 -0
  54. package/dist/number-name.js.map +1 -0
  55. package/dist/orthography.d.ts +6 -2
  56. package/dist/orthography.d.ts.map +1 -1
  57. package/dist/orthography.js +78 -17
  58. package/dist/orthography.js.map +1 -1
  59. package/dist/plugin.d.ts +2 -0
  60. package/dist/plugin.d.ts.map +1 -1
  61. package/dist/preamble-paragraphs.d.ts +7 -0
  62. package/dist/preamble-paragraphs.d.ts.map +1 -0
  63. package/dist/preamble-paragraphs.js +7 -0
  64. package/dist/preamble-paragraphs.js.map +1 -0
  65. package/dist/structure/build.d.ts.map +1 -1
  66. package/dist/structure/build.js +7 -4
  67. package/dist/structure/build.js.map +1 -1
  68. package/dist/structure/heading-leaves.d.ts +7 -0
  69. package/dist/structure/heading-leaves.d.ts.map +1 -0
  70. package/dist/structure/heading-leaves.js +6 -0
  71. package/dist/structure/heading-leaves.js.map +1 -0
  72. package/package.json +3 -3
  73. package/rules/duplicate-definition.yaml +2 -2
  74. package/rules/latin-spacing.yaml +4 -2
  75. package/rules/max-kanji-continuous.yaml +2 -0
  76. package/rules/no-nakaguro-parallel.yaml +2 -2
  77. package/rules/undefined-acronym.yaml +2 -0
  78. package/src/detectors/acronym-context.ts +40 -33
  79. package/src/detectors/acronym-expansion.ts +34 -6
  80. package/src/detectors/char-shape.ts +14 -33
  81. package/src/detectors/conjugated-form.ts +40 -0
  82. package/src/detectors/enumerated-runs.ts +68 -0
  83. package/src/detectors/middle-dot.ts +10 -0
  84. package/src/detectors/orthography.ts +22 -2
  85. package/src/detectors/place-run.ts +63 -0
  86. package/src/detectors/polite-word.ts +14 -0
  87. package/src/detectors/register.ts +87 -0
  88. package/src/detectors/roman-numeral.ts +5 -0
  89. package/src/detectors/sentence-ending.ts +43 -30
  90. package/src/detectors/signals.ts +24 -7
  91. package/src/detectors/structure.ts +2 -1
  92. package/src/number-name.ts +96 -0
  93. package/src/orthography.ts +76 -15
  94. package/src/plugin.ts +2 -0
  95. package/src/preamble-paragraphs.ts +9 -0
  96. package/src/structure/build.ts +13 -10
  97. package/src/structure/heading-leaves.ts +7 -0
@@ -1,41 +1,54 @@
1
- import { endingTokens, hasPredicate, isClosed } from "../sentence-shape.ts";
2
- import type { Detector, Finding, Sentence } from "../plugin.ts";
1
+ import { endingTokens, isClosed } from "../sentence-shape.ts";
2
+ import { enumeratedRuns, enumeratorStarts, numberedStarts } from "./enumerated-runs.ts";
3
+ import { continuesInto, groupOf, registerOf, slipsOf, type Register } from "./register.ts";
4
+ import type { Detector, Finding, LexiconEntry, ProseDocument, Sentence, Span } from "../plugin.ts";
3
5
 
4
6
  /**
5
7
  * 文末の調子が混ざっているかを見る。日本語の「ですます / である」がこれ。
6
8
  *
7
9
  * どちらが正しいかは決めない。決めると、方針の違う書き手にそのまま無視される。
8
10
  * 見るのは文書の中での一貫性だけで、少数派のほうを指摘する。spec §12.3 の方針を日本語にも。
11
+ * 終止符で終わらないもの(見出しの下の `MaaSサービス` のような名前だけの行)は文として数えない。
9
12
  */
10
- /**
11
- * 述語を持たないものは文として数えない。見出しの下の `MaaSサービス` のような
12
- * 名前だけの行がそのまま「である調」の少数派になり、実文書の誤検知はすべてこれだった。
13
- */
14
- const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence) && endingTokens(sentence).length > 0;
13
+ type Entry = { readonly sentence: Sentence; readonly register: Register; readonly group: number | undefined };
15
14
 
16
- /**
17
- * 文末の語だけを見る。文の途中の引用(「ご覧ください」という表現を使う。)は文末の調子ではない。
18
- * 書いた形でも原形でも当てる。「ください」の原形は「くださる」で、原形だけを見ると丁寧な文末を見落とす。
19
- */
20
- const isPolite = (sentence: Sentence, polite: readonly string[]): boolean =>
21
- endingTokens(sentence).some((token) => polite.includes(token.surface) || (token.lemma !== undefined && polite.includes(token.lemma)));
15
+ const registerOfSentence = (sentence: Sentence, polite: readonly LexiconEntry[]): Register | undefined => {
16
+ const ending = endingTokens(sentence);
17
+ const tokens = sentence.tokens ?? [];
18
+ const head = ending[0] === undefined ? -1 : tokens.indexOf(ending[0]);
19
+ return registerOf(ending, head > 0 ? tokens.slice(0, head) : [], polite);
20
+ };
21
+
22
+ /** 番号で始まる段落の並び。木を組むのは高いので、調子が一つだけの文書(混ざりようがない)では組まない。 */
23
+ const runsOf = (doc: ProseDocument, registers: readonly Register[]): Span[] => {
24
+ const tree = new Set(registers).size > 1 ? doc.structure : undefined;
25
+ if (tree === undefined) return [];
26
+ return enumeratedRuns(
27
+ doc.paragraphs.map((paragraph) => paragraph.span),
28
+ enumeratorStarts(numberedStarts(tree), doc.sentences, doc.source),
29
+ doc.source,
30
+ );
31
+ };
22
32
 
23
33
  export const sentenceEnding: Detector = (doc, options): Finding[] => {
24
- const polite = (options.lexicon ?? []).map((entry) => entry.pattern);
25
- const judged = doc.sentences.filter(isSentence).map((sentence) => ({ sentence, polite: isPolite(sentence, polite) }));
26
- const counts = { polite: judged.filter((entry) => entry.polite).length, plain: judged.length - judged.filter((entry) => entry.polite).length };
27
- const minorityIsPolite = counts.polite <= counts.plain;
28
- const few = minorityIsPolite ? counts.polite : counts.plain;
29
- // 片方しか無ければ一貫している。少数派が閾値を超えて多ければ、混在ではなく別の文体。
30
- if (few === 0 || few > options.limit) return [];
31
- return judged
32
- .filter((entry) => entry.polite === minorityIsPolite)
33
- .map(({ sentence }) => ({
34
- rule: "no-mixed-desumasu",
35
- severity: "warning",
36
- line: 0,
37
- column: 0,
38
- quote: sentence.text.trim(),
39
- values: { count: few, limit: options.limit, offset: sentence.span.start },
40
- }));
34
+ const polite = options.lexicon ?? [];
35
+ const lists = doc.lists.map((list) => list.span);
36
+ const found = doc.sentences.flatMap((sentence, index): { sentence: Sentence; register: Register }[] => {
37
+ if (!isClosed(sentence) || continuesInto(sentence, doc.sentences[index + 1])) return [];
38
+ const register = registerOfSentence(sentence, polite);
39
+ return register === undefined ? [] : [{ sentence, register }];
40
+ });
41
+ const runs = runsOf(
42
+ doc,
43
+ found.map((entry) => entry.register),
44
+ );
45
+ const judged = found.map((entry): Entry => ({ ...entry, group: groupOf(entry.sentence.span.start, lists, runs) }));
46
+ return slipsOf(judged, options.limit).map(({ entry: { sentence }, count }) => ({
47
+ rule: "no-mixed-desumasu",
48
+ severity: "warning",
49
+ line: 0,
50
+ column: 0,
51
+ quote: sentence.text.trim(),
52
+ values: { count, limit: options.limit, offset: sentence.span.start },
53
+ }));
41
54
  };
@@ -1,8 +1,9 @@
1
1
  import { proseText } from "../measure.ts";
2
2
  import { wordsOf } from "./structure.ts";
3
3
  import { compacted, placeOf } from "./gram-place.ts";
4
- import { notAcronymSpans } from "./acronym-context.ts";
4
+ import { notAcronymSpansOf, type NotAcronymSpans } from "./acronym-context.ts";
5
5
  import { expansionAt, type ExpandedAt } from "./acronym-expansion.ts";
6
+ import { conjugatedForms } from "./conjugated-form.ts";
6
7
  import type { Detector, Finding, ProseDocument, Section, Sentence, Token } from "../plugin.ts";
7
8
 
8
9
  const PER = 1000;
@@ -183,18 +184,18 @@ type Span = { readonly start: number; readonly end: number };
183
184
 
184
185
  const spanOf = (match: RegExpExecArray): Span => ({ start: match.index, end: match.index + match[0].length });
185
186
 
186
- const spansOf = (text: string): Span[] => [
187
+ const spansOf = (text: string, notation: NotAcronymSpans): Span[] => [
187
188
  ...[...text.matchAll(UNCASED_STRETCH)].filter((match) => SHOUTED_RUN.test(match[0])).map(spanOf),
188
189
  ...[...text.matchAll(IDENTIFIER)].filter((match) => /\d/u.test(match[0])).map(spanOf),
189
190
  ...[QUOTED_CAPS, REQUIREMENT_WORD, LICENCE].flatMap((pattern) => [...text.matchAll(pattern)].map(spanOf)),
190
- ...notAcronymSpans(text),
191
+ ...notation(text),
191
192
  ];
192
193
 
193
194
  type AcronymHit = { readonly word: string; readonly hit: Hit };
194
195
 
195
- const acronymsOf = (doc: ProseDocument): AcronymHit[] =>
196
+ const acronymsOf = (doc: ProseDocument, notation: NotAcronymSpans): AcronymHit[] =>
196
197
  doc.sentences.flatMap((sentence) => {
197
- const excluded = spansOf(sentence.text);
198
+ const excluded = spansOf(sentence.text, notation);
198
199
  return [...sentence.text.matchAll(ACRONYM)]
199
200
  .filter((match) => !excluded.some((span) => span.start <= match.index && match.index + match[0].length <= span.end))
200
201
  .map((match) => ({ word: match[0], hit: { sentence, offset: sentence.span.start + match.index } }));
@@ -209,14 +210,30 @@ const isExpanded = (body: string, acronym: string, expandedAt: ExpandedAt): bool
209
210
 
210
211
  const patternsOf = (doc: ProseDocument, id: string): string[] => (doc.lexicons[id] ?? []).map((entry) => entry.pattern);
211
212
 
213
+ const notationOf = (doc: ProseDocument): NotAcronymSpans =>
214
+ notAcronymSpansOf({
215
+ meridiem: patternsOf(doc, "meridiem"),
216
+ timeZones: patternsOf(doc, "time-zone"),
217
+ currencies: patternsOf(doc, "currency-code"),
218
+ usStates: patternsOf(doc, "us-state-code"),
219
+ emphasis: patternsOf(doc, "emphasis-word"),
220
+ divisions: patternsOf(doc, "numbered-division"),
221
+ });
222
+
223
+ /** 定義の語は、語彙表の形(という)と、この文書で活用して書かれた形(といいます)の両方で照らす。 */
224
+ const definitionVerbsOf = (doc: ProseDocument): string[] => {
225
+ const verbs = patternsOf(doc, "definition-verb");
226
+ return [...verbs, ...conjugatedForms(doc.sentences, verbs)];
227
+ };
228
+
212
229
  export const undefinedAcronym: Detector = (doc, options): Finding[] => {
213
230
  const body = bodyOf(doc);
214
231
  const common = new Set((options.lexicon ?? []).map((entry) => entry.pattern));
215
232
  const seen = new Map<string, Hit>();
216
- acronymsOf(doc).forEach(({ word, hit }) => {
233
+ acronymsOf(doc, notationOf(doc)).forEach(({ word, hit }) => {
217
234
  if (!common.has(word) && !seen.has(word)) seen.set(word, hit);
218
235
  });
219
- const expandedAt = expansionAt({ markers: patternsOf(doc, "definition-marker"), verbs: patternsOf(doc, "definition-verb") });
236
+ const expandedAt = expansionAt({ markers: patternsOf(doc, "definition-marker"), verbs: definitionVerbsOf(doc) });
220
237
  const bare = [...seen.entries()].filter(([acronym]) => !isExpanded(body, acronym, expandedAt));
221
238
  if (bare.length < options.limit) return [];
222
239
  return bare.map(([acronym, hit]) => ({
@@ -2,6 +2,7 @@ import { charLength, lengthOf } from "../measure.ts";
2
2
  import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section, Span } from "../plugin.ts";
3
3
  import { dateStampIndexes, stampCandidates } from "../date-stamp.ts";
4
4
  import { inDocumentOrder } from "../structure/issues.ts";
5
+ import { preambleParagraphs } from "../preamble-paragraphs.ts";
5
6
 
6
7
  /**
7
8
  * ばらつきは変動係数(標準偏差 ÷ 平均)で測る。
@@ -116,7 +117,7 @@ const withoutDateStamps = (doc: ProseDocument, paragraphs: readonly Paragraph[])
116
117
  export const preambleLength: Detector = (doc, options): Finding[] => {
117
118
  const body = doc.sections.find((section) => section.depth >= 2);
118
119
  if (body === undefined) return [];
119
- const all = doc.paragraphs.filter((paragraph) => paragraph.span.start < body.span.start);
120
+ const all = preambleParagraphs(doc.paragraphs, body.span.start);
120
121
  if (all.length <= options.limit) return [];
121
122
  const before = withoutDateStamps(doc, all);
122
123
  if (before.length <= options.limit) return [];
@@ -0,0 +1,96 @@
1
+ import type { Span, Token } from "./plugin.ts";
2
+
3
+ /**
4
+ * 数量ではなく名前として書かれた数(番号・識別子)。鉤括弧で引いた「3.1 リサーチの原則」の節番号、※1 のような注の番号、
5
+ * 「〒102-0094 東京都」の郵便番号は、後ろの空白が番号と題・項目の区切りで、「3回」「3 回」のような空け方の好みではない。
6
+ * 空け方を数える材料にしない。
7
+ *
8
+ * 形で名前と読むのは四つだけ。0 で始まる組を含むハイフンつなぎの番号、※ や鉤括弧のすぐ後ろの番号、文頭の階層つきの節番号(2.1、3.1.2)、
9
+ * 続き番号になった行の頭の番号(白書の注)。文頭でも「223 言語」「1ターン」は数量なので、階層の無い数は文頭というだけでは名前と読まない。
10
+ * そのうえで、すぐ後ろ(空白 1 つまで)の語が数につく語(助数詞・数・助詞・助動詞)なら数量として数え続ける(「3-5 日」「26.7 万行」)。
11
+ * 品詞が無ければ判断せず、名前とは読まない。
12
+ */
13
+
14
+ /** 数と結びついて読まれる語。助数詞は NounType=Class、数は NumType=Card で渡る。 */
15
+ const BOUND_TO_NUMBER = new Set(["ADP", "PART", "AUX", "SCONJ", "CCONJ"]);
16
+
17
+ /** 番号が立つ位置の直前: 題を包む鉤括弧、注の印。 */
18
+ const LABEL_LEAD = /[「『【※]$/u;
19
+
20
+ /** 文頭(前に空白と箇条書きの印だけ)。 */
21
+ const LINE_HEAD = /^[\s\-*+・•]*$/u;
22
+
23
+ /** 階層つきの節番号(2.1、3.1.2)。 */
24
+ const SECTION_NUMBER = /^\d+(?:\.\d+)+$/u;
25
+
26
+ /**
27
+ * 0 で始まる組を含む、ハイフンでつないだ番号(〒102-0094、03-3501)。二つの数だけの「3-5」は範囲(3-5 営業日)とも読めるので
28
+ * 数え続ける。三つ以上つないだもの(073-489-5909)は、orthography.ts が境目を作る前に外している。
29
+ */
30
+ const isHyphenIdentifier = (digits: string): boolean => {
31
+ const parts = digits.split("-").filter((part) => part !== "");
32
+ return parts.length > 1 && parts.some((part) => part.startsWith("0"));
33
+ };
34
+
35
+ const RUN_CHAR = /[\d.-]/u;
36
+
37
+ /** at を含む、数字・小数点・ハイフンの並び。at が並びの中に無ければ undefined。 */
38
+ export const digitRunAround = (text: string, at: number): Span | undefined => {
39
+ if (!RUN_CHAR.test(text[at] ?? "")) return undefined;
40
+ const startAt = (index: number): number => (index > 0 && RUN_CHAR.test(text[index - 1] ?? "") ? startAt(index - 1) : index);
41
+ const endAt = (index: number): number => (index < text.length && RUN_CHAR.test(text[index] ?? "") ? endAt(index + 1) : index);
42
+ return { start: startAt(at), end: endAt(at) };
43
+ };
44
+
45
+ /** 並びの後ろ(空白 1 つまで)で始まる語。 */
46
+ const wordAfter = (tokens: readonly Token[], text: string, run: Span, base: number): Token | undefined => {
47
+ const next = text[run.end] === " " ? run.end + 1 : run.end;
48
+ return tokens.find((token) => token.span.start === base + next);
49
+ };
50
+
51
+ const isBoundToNumber = (token: Token): boolean =>
52
+ token.features?.["NounType"] === "Class" || token.features?.["NumType"] === "Card" || BOUND_TO_NUMBER.has(token.pos);
53
+
54
+ /** 「1 又は 2」の接続詞は数どうしをつなぐが、続き番号の注の本文は「31 ただし、…」のように接続詞でも始まる。 */
55
+ const CONJUNCTIONS = new Set(["CCONJ", "SCONJ"]);
56
+
57
+ /**
58
+ * 白書の注(「9 首相に…」「10 日本経済新聞…」)のように、行の頭に階層の無い番号を置き、次の行の頭の番号が 1 つ大きいもの。
59
+ * 一行だけ見ると「223 言語に対応」と区別がつかないので、前後の番号の行と続き番号になっているものだけを番号と読む。
60
+ * 後ろが英字(「15 Federal Bureau…」)の行も並びには数える。値は行の頭の位置(文書全体の座標)。
61
+ */
62
+ const NUMBERED_LINE = /^[ \t]*(?:[-*+][ \t]+)?(?<number>\d{1,3})[ \t]+\S/gmu;
63
+
64
+ export const sequenceLabelStarts = (text: string): ReadonlySet<number> => {
65
+ const lines = [...text.matchAll(NUMBERED_LINE)].map((match) => {
66
+ const number = match.groups?.["number"] ?? "";
67
+ return { start: match.index + match[0].indexOf(number), value: Number(number) };
68
+ });
69
+ const continues = (index: number): boolean =>
70
+ lines[index - 1]?.value === (lines[index]?.value ?? 0) - 1 || lines[index + 1]?.value === (lines[index]?.value ?? 0) + 1;
71
+ return new Set(lines.flatMap((line, index) => (continues(index) ? [line.start] : [])));
72
+ };
73
+
74
+ /** ハイフンでつないだ識別子か、番号の立つ位置の番号か、文頭の節番号か。 */
75
+ const isNameShaped = (text: string, run: Span): boolean => {
76
+ const [before, digits] = [text.slice(0, run.start), text.slice(run.start, run.end)];
77
+ return isHyphenIdentifier(digits) || LABEL_LEAD.test(before) || (LINE_HEAD.test(before) && SECTION_NUMBER.test(digits));
78
+ };
79
+
80
+ /**
81
+ * text の run が名前として書かれた数か。tokens は文書全体の座標で、base は text の先頭の位置。sequence は sequenceLabelStarts の結果。
82
+ * 後ろの語が読めない(品詞が無い、語の切れ目が合わない)ときは、数量として数え続ける。
83
+ */
84
+ export const isNumberName = (
85
+ text: string,
86
+ run: Span,
87
+ tokens: readonly Token[] | undefined,
88
+ base: number,
89
+ sequence: ReadonlySet<number> = new Set(),
90
+ ): boolean => {
91
+ const inSequence = sequence.has(base + run.start);
92
+ if (tokens === undefined || !(inSequence || isNameShaped(text, run))) return false;
93
+ const next = wordAfter(tokens, text, run, base);
94
+ if (next === undefined) return false;
95
+ return !isBoundToNumber(next) || (inSequence && CONJUNCTIONS.has(next.pos));
96
+ };
@@ -62,23 +62,53 @@ const latinBeside = (japanese: string | undefined, other: string | undefined, di
62
62
  isJapanese(japanese) ? kindOf(other, digitSide) : undefined;
63
63
 
64
64
  /** 日本語の字の隣にある英字・数字との境目。間が半角空白 1 つなら「空けている」、何も無ければ「詰めている」。 */
65
- const ALPHANUMERIC = /[A-Za-z0-9.]/u;
65
+ // 「confidence=0」のような設定の書き方も、= を含めて一つの並びとして読む。
66
+ const ALPHANUMERIC = /[A-Za-z0-9.=]/u;
67
+ // 名前の語は英数字だけ。「API = 0」の空白で囲んだ = は式の記号で、名前の一部ではない。
68
+ const WORD_CHAR = /[A-Za-z0-9.]/u;
69
+
70
+ /** chars[last] を末尾とし、pattern の字が続く並びの、先頭の位置。 */
71
+ const runStartIndex = (chars: readonly string[], last: number, pattern: RegExp = ALPHANUMERIC): number => {
72
+ let first = last;
73
+ while (first > 0 && pattern.test(chars[first - 1] ?? "")) first -= 1;
74
+ return first;
75
+ };
76
+
77
+ /** 並び first の前に、半角空白 1 つずつで続く英数字の語。「Phase 1」の「Phase」、「JIS X 0301」の「JIS X」。無ければ空。 */
78
+ const wordsBefore = (chars: readonly string[], first: number): string => {
79
+ const words: string[] = [];
80
+ let space = first - 1;
81
+ while (chars[space] === " " && WORD_CHAR.test(chars[space - 1] ?? "")) {
82
+ const start = runStartIndex(chars, space - 1, WORD_CHAR);
83
+ words.unshift(chars.slice(start, space).join(""));
84
+ space = start - 1;
85
+ }
86
+ return words.join(" ");
87
+ };
88
+
89
+ // 「1ファイル x 1シート」の x のように、1 字だけの英字は記号か変数で、名前の頭ではない。
90
+ const MIN_NAME_LETTERS = 2;
66
91
 
67
92
  /**
68
- * 日本語の左にある英数字の並びの、先頭の字。「3GBの」の「GB」は数量の単位で、並びは数字で始まる。
69
- * そうした並びと日本語の境目は、英字の空け方ではなく数字の後ろの空け方として数える(「3回」と同じ)。
70
- * 逆に「H30 等」「EC2 で」は英字で始まる名前で、書き手は英単語と同じに空ける。数字の後ろとしては数えない。
93
+ * 「Phase 1 は」「iOS 17以上」の数字は、英字の語に続く名前の一部。空白は名前の中の空白で、日本語との境目は
94
+ * 「H30 等」と同じく英字の空け方として数える。
71
95
  */
72
- const runStart = (chars: readonly string[], last: number): string | undefined => {
73
- let first = last;
74
- while (first > 0 && ALPHANUMERIC.test(chars[first - 1] ?? "")) first -= 1;
75
- return chars[first];
96
+ const isNameTail = (chars: readonly string[], first: number): boolean => {
97
+ const words = wordsBefore(chars, first);
98
+ return LETTER.test(words[0] ?? "") && (words.match(/[A-Za-z]/gu)?.length ?? 0) >= MIN_NAME_LETTERS;
76
99
  };
77
100
 
78
- /** 左の並びの種類は先頭の字で決める。数字で始まる並び(3GB、10ms)は数字、英字で始まる並び(H30、v1.2)は英字。 */
101
+ /**
102
+ * 日本語の左にある英数字の並びの種類は、先頭の字で決める。「3GBの」の「GB」は数量の単位で、並びは数字で始まる。
103
+ * そうした並びと日本語の境目は、英字の空け方ではなく数字の後ろの空け方として数える(「3回」と同じ)。
104
+ * 逆に「H30 等」「EC2 で」「v1.2の」は英字で始まる名前で、書き手は英単語と同じに空ける。
105
+ */
79
106
  const leftRunBeside = (chars: readonly string[], last: number, japanese: string | undefined): SpacingKind | undefined => {
80
107
  const kind = latinBeside(japanese, chars[last], "after-digit");
81
- return kind === undefined ? undefined : (kindOf(runStart(chars, last), "after-digit") ?? kind);
108
+ if (kind === undefined) return undefined;
109
+ const first = runStartIndex(chars, last);
110
+ if (DIGIT.test(chars[first] ?? "") && isNameTail(chars, first)) return "letter";
111
+ return kindOf(chars[first], "after-digit") ?? kind;
82
112
  };
83
113
 
84
114
  /**
@@ -94,10 +124,37 @@ const isOrdinalRun = (chars: readonly string[], index: number): boolean => {
94
124
  return chars[before] === ORDINAL_PREFIX;
95
125
  };
96
126
 
127
+ /**
128
+ * 「073-489-5909」「2026-06-02」「Ⅰ-4-1-3」のように - でつないだ 3 組以上の数字は、電話番号・日付・図の番号で
129
+ * 数量ではない。後ろの空白は欄の区切りで、「3回」の空け方とは別のもの。両側とも数えない。
130
+ * 2 組(「1-3ヶ月」「20-30分」)は幅のある数量なので数える。
131
+ */
132
+ const MIN_CODE_GROUPS = 3;
133
+ const CODE_CHAR = /[\d-]/u;
134
+
135
+ const isCode = (chars: readonly string[], digit: number): boolean => {
136
+ let [first, last] = [digit, digit];
137
+ while (first > 0 && CODE_CHAR.test(chars[first - 1] ?? "")) first -= 1;
138
+ while (last < chars.length - 1 && CODE_CHAR.test(chars[last + 1] ?? "")) last += 1;
139
+ const groups = chars
140
+ .slice(first, last + 1)
141
+ .join("")
142
+ .split("-");
143
+ return groups.filter((group) => group !== "").length >= MIN_CODE_GROUPS;
144
+ };
145
+
146
+ /** 日本語との境目にある数字が、数えない書き方(「第3条」の番地、「073-489-5909」の符号)か。 */
147
+ const isUncountedNumber = (chars: readonly string[], index: number): boolean => {
148
+ const [left, right] = [chars[index], chars[index + 1]];
149
+ if (DIGIT.test(left ?? "")) return isOrdinalRun(chars, index) || isCode(chars, index);
150
+ const next = right === " " ? index + 2 : index + 1;
151
+ if (!DIGIT.test(chars[next] ?? "")) return false;
152
+ return left === ORDINAL_PREFIX || (isJapanese(left) && isCode(chars, next));
153
+ };
154
+
97
155
  const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "offset"> | undefined => {
156
+ if (isUncountedNumber(chars, index)) return undefined;
98
157
  const [left, right, after] = [chars[index], chars[index + 1], chars[index + 2]];
99
- const numberAfterPrefix = left === ORDINAL_PREFIX && DIGIT.test((right === " " ? after : right) ?? "");
100
- if (numberAfterPrefix || (DIGIT.test(left ?? "") && isOrdinalRun(chars, index))) return undefined;
101
158
  const touching = latinBeside(left, right, "before-digit") ?? leftRunBeside(chars, index, right);
102
159
  if (touching !== undefined) return { kind: touching, spaced: false };
103
160
  if (right !== " ") return undefined;
@@ -105,15 +162,19 @@ const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "of
105
162
  return spaced === undefined ? undefined : { kind: spaced, spaced: true };
106
163
  };
107
164
 
108
- /** 文の中の境目を全部。offset は境目の位置(左の字の直後、UTF-16)。配列を作り直さずに一度なめる。 */
109
- export const latinBoundaries = (text: string): Boundary[] => {
165
+ /**
166
+ * 文の中の境目を全部。offset は境目の位置(左の字の直後、UTF-16)。配列を作り直さずに一度なめる。
167
+ * text は本文でないものを空白で覆った文、written は覆う前の同じ範囲。覆ってできた空白(リンクの「[」など)は
168
+ * 書き手が空けたものではないので、「空けている」とは数えない。
169
+ */
170
+ export const latinBoundaries = (text: string, written: string = text): Boundary[] => {
110
171
  const chars = [...text];
111
172
  const found: Boundary[] = [];
112
173
  let offset = 0;
113
174
  chars.forEach((char, index) => {
114
175
  offset += char.length;
115
176
  const boundary = boundaryAt(chars, index);
116
- if (boundary !== undefined) found.push({ ...boundary, offset });
177
+ if (boundary !== undefined && (!boundary.spaced || written[offset] === " ")) found.push({ ...boundary, offset });
117
178
  });
118
179
  return found;
119
180
  };
package/src/plugin.ts CHANGED
@@ -20,6 +20,8 @@ export type Token = {
20
20
  readonly surface: string;
21
21
  readonly pos: string;
22
22
  readonly lemma?: string;
23
+ /** 書いた形の読み(日本語はカタカナ)。表記の違う同じ語(下さい・ください)を同じと見るため。読めない adapter は持たない。 */
24
+ readonly reading?: string;
23
25
  readonly features?: Readonly<Record<string, string>>;
24
26
  };
25
27
 
@@ -0,0 +1,9 @@
1
+ import type { Paragraph } from "./plugin.ts";
2
+ import { isClosed } from "./sentence-shape.ts";
3
+
4
+ /**
5
+ * 本題の前にあって、読み手が読み通す段落。文が一つも終止符で閉じない段落は数えない。
6
+ * 分類の札(「エネルギー・環境」)、著者と所属の行、「印刷」、テンプレートの指示は文ではなく、読ませる前置きではない。
7
+ */
8
+ export const preambleParagraphs = (paragraphs: readonly Paragraph[], bodyStart: number): Paragraph[] =>
9
+ paragraphs.filter((paragraph) => paragraph.span.start < bodyStart && paragraph.sentences.some(isClosed));
@@ -8,6 +8,7 @@ import type { DocumentProfile, Mention, NumberedLine, NumberingContext, Span, St
8
8
  import { lineNumberAt, linesOf, type Line } from "./lines.ts";
9
9
  import { dottedNumber } from "./universal.ts";
10
10
  import { unnumberedUnit, type OpenUnit } from "./unnumbered.ts";
11
+ import { readsOnHeading } from "./heading-leaves.ts";
11
12
 
12
13
  /** 組み立て中の節点。できあがったら StructureNode に固める。 */
13
14
  type Draft = {
@@ -179,17 +180,19 @@ const scopeOf = (state: State, patterns: StructurePatterns, text: string): Reado
179
180
  return within === undefined ? { scope: "local" } : { scope: "local", within: within.address };
180
181
  };
181
182
 
182
- const addLeaves = (state: State, patterns: StructurePatterns, line: Line, text: string, offset: number): void => {
183
+ const addLeaves = (state: State, patterns: StructurePatterns, line: Line, text: string, offset: number, onHeading: boolean): void => {
183
184
  const parent = top(state).draft;
184
185
  const scope = scopeOf(state, patterns, text);
185
- const leaves = LEAVES.flatMap(({ kind, find }) =>
186
- find(patterns, text, state.profile).map((mention) => ({
187
- kind,
188
- start: line.start + offset + mention.start,
189
- end: line.start + offset + mention.end,
190
- attrs: mention.attrs,
191
- })),
192
- ).sort((left, right) => left.start - right.start);
186
+ const leaves = LEAVES.filter(({ kind }) => !onHeading || readsOnHeading(kind))
187
+ .flatMap(({ kind, find }) =>
188
+ find(patterns, text, state.profile).map((mention) => ({
189
+ kind,
190
+ start: line.start + offset + mention.start,
191
+ end: line.start + offset + mention.end,
192
+ attrs: mention.attrs,
193
+ })),
194
+ )
195
+ .sort((left, right) => left.start - right.start);
193
196
  leaves.forEach((leaf) =>
194
197
  parent.children.push({
195
198
  kind: leaf.kind,
@@ -303,7 +306,7 @@ const readLine = (state: State, patterns: StructurePatterns, line: Line, heading
303
306
  // 番号付きの行は番号の後ろだけを読む。「第3条(支払)」の「第3条」を自分への参照として拾わない。
304
307
  const scanned = numbered === undefined ? text : numbered.rest;
305
308
  const offset = Math.max(0, line.text.lastIndexOf(scanned));
306
- if (scanned !== "") addLeaves(state, patterns, line, scanned, offset);
309
+ if (scanned !== "") addLeaves(state, patterns, line, scanned, offset, heading !== undefined);
307
310
  };
308
311
 
309
312
  /**
@@ -0,0 +1,7 @@
1
+ import type { StructureKind } from "../plugin.ts";
2
+
3
+ /**
4
+ * 見出しの行から拾う葉か。見出しは節の題で、「『特別警報』とは」は下の本文が定義する語を掲げているだけで、定義ではない。
5
+ * 定義として数えると、本文の「『特別警報』とは、…」が二度目の定義に見える。参照や数量は見出しにあっても読む。
6
+ */
7
+ export const readsOnHeading = (kind: StructureKind): boolean => kind !== "definition";