chaffjs 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/detectors/acronym-context.d.ts +17 -3
- package/dist/detectors/acronym-context.d.ts.map +1 -1
- package/dist/detectors/acronym-context.js +25 -32
- package/dist/detectors/acronym-context.js.map +1 -1
- package/dist/detectors/acronym-expansion.d.ts.map +1 -1
- package/dist/detectors/acronym-expansion.js +33 -6
- package/dist/detectors/acronym-expansion.js.map +1 -1
- package/dist/detectors/char-shape.d.ts.map +1 -1
- package/dist/detectors/char-shape.js +22 -36
- package/dist/detectors/char-shape.js.map +1 -1
- package/dist/detectors/conjugated-form.d.ts +3 -0
- package/dist/detectors/conjugated-form.d.ts.map +1 -0
- package/dist/detectors/conjugated-form.js +29 -0
- package/dist/detectors/conjugated-form.js.map +1 -0
- package/dist/detectors/enumerated-runs.d.ts +22 -0
- package/dist/detectors/enumerated-runs.d.ts.map +1 -0
- package/dist/detectors/enumerated-runs.js +53 -0
- package/dist/detectors/enumerated-runs.js.map +1 -0
- package/dist/detectors/middle-dot.d.ts +6 -0
- package/dist/detectors/middle-dot.d.ts.map +1 -0
- package/dist/detectors/middle-dot.js +9 -0
- package/dist/detectors/middle-dot.js.map +1 -0
- package/dist/detectors/orthography.d.ts.map +1 -1
- package/dist/detectors/orthography.js +19 -1
- package/dist/detectors/orthography.js.map +1 -1
- package/dist/detectors/place-run.d.ts +8 -0
- package/dist/detectors/place-run.d.ts.map +1 -0
- package/dist/detectors/place-run.js +54 -0
- package/dist/detectors/place-run.js.map +1 -0
- package/dist/detectors/polite-word.d.ts +4 -0
- package/dist/detectors/polite-word.d.ts.map +1 -0
- package/dist/detectors/polite-word.js +10 -0
- package/dist/detectors/polite-word.js.map +1 -0
- package/dist/detectors/register.d.ts +34 -0
- package/dist/detectors/register.d.ts.map +1 -0
- package/dist/detectors/register.js +64 -0
- package/dist/detectors/register.js.map +1 -0
- package/dist/detectors/roman-numeral.d.ts +6 -0
- package/dist/detectors/roman-numeral.d.ts.map +1 -0
- package/dist/detectors/roman-numeral.js +6 -0
- package/dist/detectors/roman-numeral.js.map +1 -0
- package/dist/detectors/sentence-ending.d.ts.map +1 -1
- package/dist/detectors/sentence-ending.js +28 -29
- package/dist/detectors/sentence-ending.js.map +1 -1
- package/dist/detectors/signals.d.ts.map +1 -1
- package/dist/detectors/signals.js +21 -7
- package/dist/detectors/signals.js.map +1 -1
- package/dist/detectors/structure.d.ts.map +1 -1
- package/dist/detectors/structure.js +2 -1
- package/dist/detectors/structure.js.map +1 -1
- package/dist/number-name.d.ts +10 -0
- package/dist/number-name.d.ts.map +1 -0
- package/dist/number-name.js +76 -0
- package/dist/number-name.js.map +1 -0
- package/dist/orthography.d.ts +6 -2
- package/dist/orthography.d.ts.map +1 -1
- package/dist/orthography.js +78 -17
- package/dist/orthography.js.map +1 -1
- package/dist/plugin.d.ts +2 -0
- package/dist/plugin.d.ts.map +1 -1
- package/dist/preamble-paragraphs.d.ts +7 -0
- package/dist/preamble-paragraphs.d.ts.map +1 -0
- package/dist/preamble-paragraphs.js +7 -0
- package/dist/preamble-paragraphs.js.map +1 -0
- package/dist/structure/build.d.ts.map +1 -1
- package/dist/structure/build.js +7 -4
- package/dist/structure/build.js.map +1 -1
- package/dist/structure/heading-leaves.d.ts +7 -0
- package/dist/structure/heading-leaves.d.ts.map +1 -0
- package/dist/structure/heading-leaves.js +6 -0
- package/dist/structure/heading-leaves.js.map +1 -0
- package/package.json +3 -3
- package/rules/duplicate-definition.yaml +2 -2
- package/rules/latin-spacing.yaml +4 -2
- package/rules/max-kanji-continuous.yaml +2 -0
- package/rules/no-nakaguro-parallel.yaml +2 -2
- package/rules/undefined-acronym.yaml +2 -0
- package/src/detectors/acronym-context.ts +40 -33
- package/src/detectors/acronym-expansion.ts +34 -6
- package/src/detectors/char-shape.ts +14 -33
- package/src/detectors/conjugated-form.ts +40 -0
- package/src/detectors/enumerated-runs.ts +68 -0
- package/src/detectors/middle-dot.ts +10 -0
- package/src/detectors/orthography.ts +22 -2
- package/src/detectors/place-run.ts +63 -0
- package/src/detectors/polite-word.ts +14 -0
- package/src/detectors/register.ts +87 -0
- package/src/detectors/roman-numeral.ts +5 -0
- package/src/detectors/sentence-ending.ts +43 -30
- package/src/detectors/signals.ts +24 -7
- package/src/detectors/structure.ts +2 -1
- package/src/number-name.ts +96 -0
- package/src/orthography.ts +76 -15
- package/src/plugin.ts +2 -0
- package/src/preamble-paragraphs.ts +9 -0
- package/src/structure/build.ts +13 -10
- package/src/structure/heading-leaves.ts +7 -0
|
@@ -1,41 +1,54 @@
|
|
|
1
|
-
import { endingTokens,
|
|
2
|
-
import
|
|
1
|
+
import { endingTokens, isClosed } from "../sentence-shape.ts";
|
|
2
|
+
import { enumeratedRuns, enumeratorStarts, numberedStarts } from "./enumerated-runs.ts";
|
|
3
|
+
import { continuesInto, groupOf, registerOf, slipsOf, type Register } from "./register.ts";
|
|
4
|
+
import type { Detector, Finding, LexiconEntry, ProseDocument, Sentence, Span } from "../plugin.ts";
|
|
3
5
|
|
|
4
6
|
/**
|
|
5
7
|
* 文末の調子が混ざっているかを見る。日本語の「ですます / である」がこれ。
|
|
6
8
|
*
|
|
7
9
|
* どちらが正しいかは決めない。決めると、方針の違う書き手にそのまま無視される。
|
|
8
10
|
* 見るのは文書の中での一貫性だけで、少数派のほうを指摘する。spec §12.3 の方針を日本語にも。
|
|
11
|
+
* 終止符で終わらないもの(見出しの下の `MaaSサービス` のような名前だけの行)は文として数えない。
|
|
9
12
|
*/
|
|
10
|
-
|
|
11
|
-
* 述語を持たないものは文として数えない。見出しの下の `MaaSサービス` のような
|
|
12
|
-
* 名前だけの行がそのまま「である調」の少数派になり、実文書の誤検知はすべてこれだった。
|
|
13
|
-
*/
|
|
14
|
-
const isSentence = (sentence: Sentence): boolean => isClosed(sentence) && hasPredicate(sentence) && endingTokens(sentence).length > 0;
|
|
13
|
+
type Entry = { readonly sentence: Sentence; readonly register: Register; readonly group: number | undefined };
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
15
|
+
const registerOfSentence = (sentence: Sentence, polite: readonly LexiconEntry[]): Register | undefined => {
|
|
16
|
+
const ending = endingTokens(sentence);
|
|
17
|
+
const tokens = sentence.tokens ?? [];
|
|
18
|
+
const head = ending[0] === undefined ? -1 : tokens.indexOf(ending[0]);
|
|
19
|
+
return registerOf(ending, head > 0 ? tokens.slice(0, head) : [], polite);
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
/** 番号で始まる段落の並び。木を組むのは高いので、調子が一つだけの文書(混ざりようがない)では組まない。 */
|
|
23
|
+
const runsOf = (doc: ProseDocument, registers: readonly Register[]): Span[] => {
|
|
24
|
+
const tree = new Set(registers).size > 1 ? doc.structure : undefined;
|
|
25
|
+
if (tree === undefined) return [];
|
|
26
|
+
return enumeratedRuns(
|
|
27
|
+
doc.paragraphs.map((paragraph) => paragraph.span),
|
|
28
|
+
enumeratorStarts(numberedStarts(tree), doc.sentences, doc.source),
|
|
29
|
+
doc.source,
|
|
30
|
+
);
|
|
31
|
+
};
|
|
22
32
|
|
|
23
33
|
export const sentenceEnding: Detector = (doc, options): Finding[] => {
|
|
24
|
-
const polite =
|
|
25
|
-
const
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
.map((
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
34
|
+
const polite = options.lexicon ?? [];
|
|
35
|
+
const lists = doc.lists.map((list) => list.span);
|
|
36
|
+
const found = doc.sentences.flatMap((sentence, index): { sentence: Sentence; register: Register }[] => {
|
|
37
|
+
if (!isClosed(sentence) || continuesInto(sentence, doc.sentences[index + 1])) return [];
|
|
38
|
+
const register = registerOfSentence(sentence, polite);
|
|
39
|
+
return register === undefined ? [] : [{ sentence, register }];
|
|
40
|
+
});
|
|
41
|
+
const runs = runsOf(
|
|
42
|
+
doc,
|
|
43
|
+
found.map((entry) => entry.register),
|
|
44
|
+
);
|
|
45
|
+
const judged = found.map((entry): Entry => ({ ...entry, group: groupOf(entry.sentence.span.start, lists, runs) }));
|
|
46
|
+
return slipsOf(judged, options.limit).map(({ entry: { sentence }, count }) => ({
|
|
47
|
+
rule: "no-mixed-desumasu",
|
|
48
|
+
severity: "warning",
|
|
49
|
+
line: 0,
|
|
50
|
+
column: 0,
|
|
51
|
+
quote: sentence.text.trim(),
|
|
52
|
+
values: { count, limit: options.limit, offset: sentence.span.start },
|
|
53
|
+
}));
|
|
41
54
|
};
|
package/src/detectors/signals.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { proseText } from "../measure.ts";
|
|
2
2
|
import { wordsOf } from "./structure.ts";
|
|
3
3
|
import { compacted, placeOf } from "./gram-place.ts";
|
|
4
|
-
import {
|
|
4
|
+
import { notAcronymSpansOf, type NotAcronymSpans } from "./acronym-context.ts";
|
|
5
5
|
import { expansionAt, type ExpandedAt } from "./acronym-expansion.ts";
|
|
6
|
+
import { conjugatedForms } from "./conjugated-form.ts";
|
|
6
7
|
import type { Detector, Finding, ProseDocument, Section, Sentence, Token } from "../plugin.ts";
|
|
7
8
|
|
|
8
9
|
const PER = 1000;
|
|
@@ -183,18 +184,18 @@ type Span = { readonly start: number; readonly end: number };
|
|
|
183
184
|
|
|
184
185
|
const spanOf = (match: RegExpExecArray): Span => ({ start: match.index, end: match.index + match[0].length });
|
|
185
186
|
|
|
186
|
-
const spansOf = (text: string): Span[] => [
|
|
187
|
+
const spansOf = (text: string, notation: NotAcronymSpans): Span[] => [
|
|
187
188
|
...[...text.matchAll(UNCASED_STRETCH)].filter((match) => SHOUTED_RUN.test(match[0])).map(spanOf),
|
|
188
189
|
...[...text.matchAll(IDENTIFIER)].filter((match) => /\d/u.test(match[0])).map(spanOf),
|
|
189
190
|
...[QUOTED_CAPS, REQUIREMENT_WORD, LICENCE].flatMap((pattern) => [...text.matchAll(pattern)].map(spanOf)),
|
|
190
|
-
...
|
|
191
|
+
...notation(text),
|
|
191
192
|
];
|
|
192
193
|
|
|
193
194
|
type AcronymHit = { readonly word: string; readonly hit: Hit };
|
|
194
195
|
|
|
195
|
-
const acronymsOf = (doc: ProseDocument): AcronymHit[] =>
|
|
196
|
+
const acronymsOf = (doc: ProseDocument, notation: NotAcronymSpans): AcronymHit[] =>
|
|
196
197
|
doc.sentences.flatMap((sentence) => {
|
|
197
|
-
const excluded = spansOf(sentence.text);
|
|
198
|
+
const excluded = spansOf(sentence.text, notation);
|
|
198
199
|
return [...sentence.text.matchAll(ACRONYM)]
|
|
199
200
|
.filter((match) => !excluded.some((span) => span.start <= match.index && match.index + match[0].length <= span.end))
|
|
200
201
|
.map((match) => ({ word: match[0], hit: { sentence, offset: sentence.span.start + match.index } }));
|
|
@@ -209,14 +210,30 @@ const isExpanded = (body: string, acronym: string, expandedAt: ExpandedAt): bool
|
|
|
209
210
|
|
|
210
211
|
const patternsOf = (doc: ProseDocument, id: string): string[] => (doc.lexicons[id] ?? []).map((entry) => entry.pattern);
|
|
211
212
|
|
|
213
|
+
const notationOf = (doc: ProseDocument): NotAcronymSpans =>
|
|
214
|
+
notAcronymSpansOf({
|
|
215
|
+
meridiem: patternsOf(doc, "meridiem"),
|
|
216
|
+
timeZones: patternsOf(doc, "time-zone"),
|
|
217
|
+
currencies: patternsOf(doc, "currency-code"),
|
|
218
|
+
usStates: patternsOf(doc, "us-state-code"),
|
|
219
|
+
emphasis: patternsOf(doc, "emphasis-word"),
|
|
220
|
+
divisions: patternsOf(doc, "numbered-division"),
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
/** 定義の語は、語彙表の形(という)と、この文書で活用して書かれた形(といいます)の両方で照らす。 */
|
|
224
|
+
const definitionVerbsOf = (doc: ProseDocument): string[] => {
|
|
225
|
+
const verbs = patternsOf(doc, "definition-verb");
|
|
226
|
+
return [...verbs, ...conjugatedForms(doc.sentences, verbs)];
|
|
227
|
+
};
|
|
228
|
+
|
|
212
229
|
export const undefinedAcronym: Detector = (doc, options): Finding[] => {
|
|
213
230
|
const body = bodyOf(doc);
|
|
214
231
|
const common = new Set((options.lexicon ?? []).map((entry) => entry.pattern));
|
|
215
232
|
const seen = new Map<string, Hit>();
|
|
216
|
-
acronymsOf(doc).forEach(({ word, hit }) => {
|
|
233
|
+
acronymsOf(doc, notationOf(doc)).forEach(({ word, hit }) => {
|
|
217
234
|
if (!common.has(word) && !seen.has(word)) seen.set(word, hit);
|
|
218
235
|
});
|
|
219
|
-
const expandedAt = expansionAt({ markers: patternsOf(doc, "definition-marker"), verbs:
|
|
236
|
+
const expandedAt = expansionAt({ markers: patternsOf(doc, "definition-marker"), verbs: definitionVerbsOf(doc) });
|
|
220
237
|
const bare = [...seen.entries()].filter(([acronym]) => !isExpanded(body, acronym, expandedAt));
|
|
221
238
|
if (bare.length < options.limit) return [];
|
|
222
239
|
return bare.map(([acronym, hit]) => ({
|
|
@@ -2,6 +2,7 @@ import { charLength, lengthOf } from "../measure.ts";
|
|
|
2
2
|
import type { BulletList, Detector, Finding, Paragraph, ProseDocument, Section, Span } from "../plugin.ts";
|
|
3
3
|
import { dateStampIndexes, stampCandidates } from "../date-stamp.ts";
|
|
4
4
|
import { inDocumentOrder } from "../structure/issues.ts";
|
|
5
|
+
import { preambleParagraphs } from "../preamble-paragraphs.ts";
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* ばらつきは変動係数(標準偏差 ÷ 平均)で測る。
|
|
@@ -116,7 +117,7 @@ const withoutDateStamps = (doc: ProseDocument, paragraphs: readonly Paragraph[])
|
|
|
116
117
|
export const preambleLength: Detector = (doc, options): Finding[] => {
|
|
117
118
|
const body = doc.sections.find((section) => section.depth >= 2);
|
|
118
119
|
if (body === undefined) return [];
|
|
119
|
-
const all = doc.paragraphs
|
|
120
|
+
const all = preambleParagraphs(doc.paragraphs, body.span.start);
|
|
120
121
|
if (all.length <= options.limit) return [];
|
|
121
122
|
const before = withoutDateStamps(doc, all);
|
|
122
123
|
if (before.length <= options.limit) return [];
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import type { Span, Token } from "./plugin.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 数量ではなく名前として書かれた数(番号・識別子)。鉤括弧で引いた「3.1 リサーチの原則」の節番号、※1 のような注の番号、
|
|
5
|
+
* 「〒102-0094 東京都」の郵便番号は、後ろの空白が番号と題・項目の区切りで、「3回」「3 回」のような空け方の好みではない。
|
|
6
|
+
* 空け方を数える材料にしない。
|
|
7
|
+
*
|
|
8
|
+
* 形で名前と読むのは四つだけ。0 で始まる組を含むハイフンつなぎの番号、※ や鉤括弧のすぐ後ろの番号、文頭の階層つきの節番号(2.1、3.1.2)、
|
|
9
|
+
* 続き番号になった行の頭の番号(白書の注)。文頭でも「223 言語」「1ターン」は数量なので、階層の無い数は文頭というだけでは名前と読まない。
|
|
10
|
+
* そのうえで、すぐ後ろ(空白 1 つまで)の語が数につく語(助数詞・数・助詞・助動詞)なら数量として数え続ける(「3-5 日」「26.7 万行」)。
|
|
11
|
+
* 品詞が無ければ判断せず、名前とは読まない。
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** 数と結びついて読まれる語。助数詞は NounType=Class、数は NumType=Card で渡る。 */
|
|
15
|
+
const BOUND_TO_NUMBER = new Set(["ADP", "PART", "AUX", "SCONJ", "CCONJ"]);
|
|
16
|
+
|
|
17
|
+
/** 番号が立つ位置の直前: 題を包む鉤括弧、注の印。 */
|
|
18
|
+
const LABEL_LEAD = /[「『【※]$/u;
|
|
19
|
+
|
|
20
|
+
/** 文頭(前に空白と箇条書きの印だけ)。 */
|
|
21
|
+
const LINE_HEAD = /^[\s\-*+・•]*$/u;
|
|
22
|
+
|
|
23
|
+
/** 階層つきの節番号(2.1、3.1.2)。 */
|
|
24
|
+
const SECTION_NUMBER = /^\d+(?:\.\d+)+$/u;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* 0 で始まる組を含む、ハイフンでつないだ番号(〒102-0094、03-3501)。二つの数だけの「3-5」は範囲(3-5 営業日)とも読めるので
|
|
28
|
+
* 数え続ける。三つ以上つないだもの(073-489-5909)は、orthography.ts が境目を作る前に外している。
|
|
29
|
+
*/
|
|
30
|
+
const isHyphenIdentifier = (digits: string): boolean => {
|
|
31
|
+
const parts = digits.split("-").filter((part) => part !== "");
|
|
32
|
+
return parts.length > 1 && parts.some((part) => part.startsWith("0"));
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const RUN_CHAR = /[\d.-]/u;
|
|
36
|
+
|
|
37
|
+
/** at を含む、数字・小数点・ハイフンの並び。at が並びの中に無ければ undefined。 */
|
|
38
|
+
export const digitRunAround = (text: string, at: number): Span | undefined => {
|
|
39
|
+
if (!RUN_CHAR.test(text[at] ?? "")) return undefined;
|
|
40
|
+
const startAt = (index: number): number => (index > 0 && RUN_CHAR.test(text[index - 1] ?? "") ? startAt(index - 1) : index);
|
|
41
|
+
const endAt = (index: number): number => (index < text.length && RUN_CHAR.test(text[index] ?? "") ? endAt(index + 1) : index);
|
|
42
|
+
return { start: startAt(at), end: endAt(at) };
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** 並びの後ろ(空白 1 つまで)で始まる語。 */
|
|
46
|
+
const wordAfter = (tokens: readonly Token[], text: string, run: Span, base: number): Token | undefined => {
|
|
47
|
+
const next = text[run.end] === " " ? run.end + 1 : run.end;
|
|
48
|
+
return tokens.find((token) => token.span.start === base + next);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const isBoundToNumber = (token: Token): boolean =>
|
|
52
|
+
token.features?.["NounType"] === "Class" || token.features?.["NumType"] === "Card" || BOUND_TO_NUMBER.has(token.pos);
|
|
53
|
+
|
|
54
|
+
/** 「1 又は 2」の接続詞は数どうしをつなぐが、続き番号の注の本文は「31 ただし、…」のように接続詞でも始まる。 */
|
|
55
|
+
const CONJUNCTIONS = new Set(["CCONJ", "SCONJ"]);
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* 白書の注(「9 首相に…」「10 日本経済新聞…」)のように、行の頭に階層の無い番号を置き、次の行の頭の番号が 1 つ大きいもの。
|
|
59
|
+
* 一行だけ見ると「223 言語に対応」と区別がつかないので、前後の番号の行と続き番号になっているものだけを番号と読む。
|
|
60
|
+
* 後ろが英字(「15 Federal Bureau…」)の行も並びには数える。値は行の頭の位置(文書全体の座標)。
|
|
61
|
+
*/
|
|
62
|
+
const NUMBERED_LINE = /^[ \t]*(?:[-*+][ \t]+)?(?<number>\d{1,3})[ \t]+\S/gmu;
|
|
63
|
+
|
|
64
|
+
export const sequenceLabelStarts = (text: string): ReadonlySet<number> => {
|
|
65
|
+
const lines = [...text.matchAll(NUMBERED_LINE)].map((match) => {
|
|
66
|
+
const number = match.groups?.["number"] ?? "";
|
|
67
|
+
return { start: match.index + match[0].indexOf(number), value: Number(number) };
|
|
68
|
+
});
|
|
69
|
+
const continues = (index: number): boolean =>
|
|
70
|
+
lines[index - 1]?.value === (lines[index]?.value ?? 0) - 1 || lines[index + 1]?.value === (lines[index]?.value ?? 0) + 1;
|
|
71
|
+
return new Set(lines.flatMap((line, index) => (continues(index) ? [line.start] : [])));
|
|
72
|
+
};
|
|
73
|
+
|
|
74
|
+
/** ハイフンでつないだ識別子か、番号の立つ位置の番号か、文頭の節番号か。 */
|
|
75
|
+
const isNameShaped = (text: string, run: Span): boolean => {
|
|
76
|
+
const [before, digits] = [text.slice(0, run.start), text.slice(run.start, run.end)];
|
|
77
|
+
return isHyphenIdentifier(digits) || LABEL_LEAD.test(before) || (LINE_HEAD.test(before) && SECTION_NUMBER.test(digits));
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* text の run が名前として書かれた数か。tokens は文書全体の座標で、base は text の先頭の位置。sequence は sequenceLabelStarts の結果。
|
|
82
|
+
* 後ろの語が読めない(品詞が無い、語の切れ目が合わない)ときは、数量として数え続ける。
|
|
83
|
+
*/
|
|
84
|
+
export const isNumberName = (
|
|
85
|
+
text: string,
|
|
86
|
+
run: Span,
|
|
87
|
+
tokens: readonly Token[] | undefined,
|
|
88
|
+
base: number,
|
|
89
|
+
sequence: ReadonlySet<number> = new Set(),
|
|
90
|
+
): boolean => {
|
|
91
|
+
const inSequence = sequence.has(base + run.start);
|
|
92
|
+
if (tokens === undefined || !(inSequence || isNameShaped(text, run))) return false;
|
|
93
|
+
const next = wordAfter(tokens, text, run, base);
|
|
94
|
+
if (next === undefined) return false;
|
|
95
|
+
return !isBoundToNumber(next) || (inSequence && CONJUNCTIONS.has(next.pos));
|
|
96
|
+
};
|
package/src/orthography.ts
CHANGED
|
@@ -62,23 +62,53 @@ const latinBeside = (japanese: string | undefined, other: string | undefined, di
|
|
|
62
62
|
isJapanese(japanese) ? kindOf(other, digitSide) : undefined;
|
|
63
63
|
|
|
64
64
|
/** 日本語の字の隣にある英字・数字との境目。間が半角空白 1 つなら「空けている」、何も無ければ「詰めている」。 */
|
|
65
|
-
|
|
65
|
+
// 「confidence=0」のような設定の書き方も、= を含めて一つの並びとして読む。
|
|
66
|
+
const ALPHANUMERIC = /[A-Za-z0-9.=]/u;
|
|
67
|
+
// 名前の語は英数字だけ。「API = 0」の空白で囲んだ = は式の記号で、名前の一部ではない。
|
|
68
|
+
const WORD_CHAR = /[A-Za-z0-9.]/u;
|
|
69
|
+
|
|
70
|
+
/** chars[last] を末尾とし、pattern の字が続く並びの、先頭の位置。 */
|
|
71
|
+
const runStartIndex = (chars: readonly string[], last: number, pattern: RegExp = ALPHANUMERIC): number => {
|
|
72
|
+
let first = last;
|
|
73
|
+
while (first > 0 && pattern.test(chars[first - 1] ?? "")) first -= 1;
|
|
74
|
+
return first;
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
/** 並び first の前に、半角空白 1 つずつで続く英数字の語。「Phase 1」の「Phase」、「JIS X 0301」の「JIS X」。無ければ空。 */
|
|
78
|
+
const wordsBefore = (chars: readonly string[], first: number): string => {
|
|
79
|
+
const words: string[] = [];
|
|
80
|
+
let space = first - 1;
|
|
81
|
+
while (chars[space] === " " && WORD_CHAR.test(chars[space - 1] ?? "")) {
|
|
82
|
+
const start = runStartIndex(chars, space - 1, WORD_CHAR);
|
|
83
|
+
words.unshift(chars.slice(start, space).join(""));
|
|
84
|
+
space = start - 1;
|
|
85
|
+
}
|
|
86
|
+
return words.join(" ");
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
// 「1ファイル x 1シート」の x のように、1 字だけの英字は記号か変数で、名前の頭ではない。
|
|
90
|
+
const MIN_NAME_LETTERS = 2;
|
|
66
91
|
|
|
67
92
|
/**
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
* 逆に「H30 等」「EC2 で」は英字で始まる名前で、書き手は英単語と同じに空ける。数字の後ろとしては数えない。
|
|
93
|
+
* 「Phase 1 は」「iOS 17以上」の数字は、英字の語に続く名前の一部。空白は名前の中の空白で、日本語との境目は
|
|
94
|
+
* 「H30 等」と同じく英字の空け方として数える。
|
|
71
95
|
*/
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
return chars[first];
|
|
96
|
+
const isNameTail = (chars: readonly string[], first: number): boolean => {
|
|
97
|
+
const words = wordsBefore(chars, first);
|
|
98
|
+
return LETTER.test(words[0] ?? "") && (words.match(/[A-Za-z]/gu)?.length ?? 0) >= MIN_NAME_LETTERS;
|
|
76
99
|
};
|
|
77
100
|
|
|
78
|
-
/**
|
|
101
|
+
/**
|
|
102
|
+
* 日本語の左にある英数字の並びの種類は、先頭の字で決める。「3GBの」の「GB」は数量の単位で、並びは数字で始まる。
|
|
103
|
+
* そうした並びと日本語の境目は、英字の空け方ではなく数字の後ろの空け方として数える(「3回」と同じ)。
|
|
104
|
+
* 逆に「H30 等」「EC2 で」「v1.2の」は英字で始まる名前で、書き手は英単語と同じに空ける。
|
|
105
|
+
*/
|
|
79
106
|
const leftRunBeside = (chars: readonly string[], last: number, japanese: string | undefined): SpacingKind | undefined => {
|
|
80
107
|
const kind = latinBeside(japanese, chars[last], "after-digit");
|
|
81
|
-
|
|
108
|
+
if (kind === undefined) return undefined;
|
|
109
|
+
const first = runStartIndex(chars, last);
|
|
110
|
+
if (DIGIT.test(chars[first] ?? "") && isNameTail(chars, first)) return "letter";
|
|
111
|
+
return kindOf(chars[first], "after-digit") ?? kind;
|
|
82
112
|
};
|
|
83
113
|
|
|
84
114
|
/**
|
|
@@ -94,10 +124,37 @@ const isOrdinalRun = (chars: readonly string[], index: number): boolean => {
|
|
|
94
124
|
return chars[before] === ORDINAL_PREFIX;
|
|
95
125
|
};
|
|
96
126
|
|
|
127
|
+
/**
|
|
128
|
+
* 「073-489-5909」「2026-06-02」「Ⅰ-4-1-3」のように - でつないだ 3 組以上の数字は、電話番号・日付・図の番号で
|
|
129
|
+
* 数量ではない。後ろの空白は欄の区切りで、「3回」の空け方とは別のもの。両側とも数えない。
|
|
130
|
+
* 2 組(「1-3ヶ月」「20-30分」)は幅のある数量なので数える。
|
|
131
|
+
*/
|
|
132
|
+
const MIN_CODE_GROUPS = 3;
|
|
133
|
+
const CODE_CHAR = /[\d-]/u;
|
|
134
|
+
|
|
135
|
+
const isCode = (chars: readonly string[], digit: number): boolean => {
|
|
136
|
+
let [first, last] = [digit, digit];
|
|
137
|
+
while (first > 0 && CODE_CHAR.test(chars[first - 1] ?? "")) first -= 1;
|
|
138
|
+
while (last < chars.length - 1 && CODE_CHAR.test(chars[last + 1] ?? "")) last += 1;
|
|
139
|
+
const groups = chars
|
|
140
|
+
.slice(first, last + 1)
|
|
141
|
+
.join("")
|
|
142
|
+
.split("-");
|
|
143
|
+
return groups.filter((group) => group !== "").length >= MIN_CODE_GROUPS;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
/** 日本語との境目にある数字が、数えない書き方(「第3条」の番地、「073-489-5909」の符号)か。 */
|
|
147
|
+
const isUncountedNumber = (chars: readonly string[], index: number): boolean => {
|
|
148
|
+
const [left, right] = [chars[index], chars[index + 1]];
|
|
149
|
+
if (DIGIT.test(left ?? "")) return isOrdinalRun(chars, index) || isCode(chars, index);
|
|
150
|
+
const next = right === " " ? index + 2 : index + 1;
|
|
151
|
+
if (!DIGIT.test(chars[next] ?? "")) return false;
|
|
152
|
+
return left === ORDINAL_PREFIX || (isJapanese(left) && isCode(chars, next));
|
|
153
|
+
};
|
|
154
|
+
|
|
97
155
|
const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "offset"> | undefined => {
|
|
156
|
+
if (isUncountedNumber(chars, index)) return undefined;
|
|
98
157
|
const [left, right, after] = [chars[index], chars[index + 1], chars[index + 2]];
|
|
99
|
-
const numberAfterPrefix = left === ORDINAL_PREFIX && DIGIT.test((right === " " ? after : right) ?? "");
|
|
100
|
-
if (numberAfterPrefix || (DIGIT.test(left ?? "") && isOrdinalRun(chars, index))) return undefined;
|
|
101
158
|
const touching = latinBeside(left, right, "before-digit") ?? leftRunBeside(chars, index, right);
|
|
102
159
|
if (touching !== undefined) return { kind: touching, spaced: false };
|
|
103
160
|
if (right !== " ") return undefined;
|
|
@@ -105,15 +162,19 @@ const boundaryAt = (chars: readonly string[], index: number): Omit<Boundary, "of
|
|
|
105
162
|
return spaced === undefined ? undefined : { kind: spaced, spaced: true };
|
|
106
163
|
};
|
|
107
164
|
|
|
108
|
-
/**
|
|
109
|
-
|
|
165
|
+
/**
|
|
166
|
+
* 文の中の境目を全部。offset は境目の位置(左の字の直後、UTF-16)。配列を作り直さずに一度なめる。
|
|
167
|
+
* text は本文でないものを空白で覆った文、written は覆う前の同じ範囲。覆ってできた空白(リンクの「[」など)は
|
|
168
|
+
* 書き手が空けたものではないので、「空けている」とは数えない。
|
|
169
|
+
*/
|
|
170
|
+
export const latinBoundaries = (text: string, written: string = text): Boundary[] => {
|
|
110
171
|
const chars = [...text];
|
|
111
172
|
const found: Boundary[] = [];
|
|
112
173
|
let offset = 0;
|
|
113
174
|
chars.forEach((char, index) => {
|
|
114
175
|
offset += char.length;
|
|
115
176
|
const boundary = boundaryAt(chars, index);
|
|
116
|
-
if (boundary !== undefined) found.push({ ...boundary, offset });
|
|
177
|
+
if (boundary !== undefined && (!boundary.spaced || written[offset] === " ")) found.push({ ...boundary, offset });
|
|
117
178
|
});
|
|
118
179
|
return found;
|
|
119
180
|
};
|
package/src/plugin.ts
CHANGED
|
@@ -20,6 +20,8 @@ export type Token = {
|
|
|
20
20
|
readonly surface: string;
|
|
21
21
|
readonly pos: string;
|
|
22
22
|
readonly lemma?: string;
|
|
23
|
+
/** 書いた形の読み(日本語はカタカナ)。表記の違う同じ語(下さい・ください)を同じと見るため。読めない adapter は持たない。 */
|
|
24
|
+
readonly reading?: string;
|
|
23
25
|
readonly features?: Readonly<Record<string, string>>;
|
|
24
26
|
};
|
|
25
27
|
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Paragraph } from "./plugin.ts";
|
|
2
|
+
import { isClosed } from "./sentence-shape.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* 本題の前にあって、読み手が読み通す段落。文が一つも終止符で閉じない段落は数えない。
|
|
6
|
+
* 分類の札(「エネルギー・環境」)、著者と所属の行、「印刷」、テンプレートの指示は文ではなく、読ませる前置きではない。
|
|
7
|
+
*/
|
|
8
|
+
export const preambleParagraphs = (paragraphs: readonly Paragraph[], bodyStart: number): Paragraph[] =>
|
|
9
|
+
paragraphs.filter((paragraph) => paragraph.span.start < bodyStart && paragraph.sentences.some(isClosed));
|
package/src/structure/build.ts
CHANGED
|
@@ -8,6 +8,7 @@ import type { DocumentProfile, Mention, NumberedLine, NumberingContext, Span, St
|
|
|
8
8
|
import { lineNumberAt, linesOf, type Line } from "./lines.ts";
|
|
9
9
|
import { dottedNumber } from "./universal.ts";
|
|
10
10
|
import { unnumberedUnit, type OpenUnit } from "./unnumbered.ts";
|
|
11
|
+
import { readsOnHeading } from "./heading-leaves.ts";
|
|
11
12
|
|
|
12
13
|
/** 組み立て中の節点。できあがったら StructureNode に固める。 */
|
|
13
14
|
type Draft = {
|
|
@@ -179,17 +180,19 @@ const scopeOf = (state: State, patterns: StructurePatterns, text: string): Reado
|
|
|
179
180
|
return within === undefined ? { scope: "local" } : { scope: "local", within: within.address };
|
|
180
181
|
};
|
|
181
182
|
|
|
182
|
-
const addLeaves = (state: State, patterns: StructurePatterns, line: Line, text: string, offset: number): void => {
|
|
183
|
+
const addLeaves = (state: State, patterns: StructurePatterns, line: Line, text: string, offset: number, onHeading: boolean): void => {
|
|
183
184
|
const parent = top(state).draft;
|
|
184
185
|
const scope = scopeOf(state, patterns, text);
|
|
185
|
-
const leaves = LEAVES.
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
186
|
+
const leaves = LEAVES.filter(({ kind }) => !onHeading || readsOnHeading(kind))
|
|
187
|
+
.flatMap(({ kind, find }) =>
|
|
188
|
+
find(patterns, text, state.profile).map((mention) => ({
|
|
189
|
+
kind,
|
|
190
|
+
start: line.start + offset + mention.start,
|
|
191
|
+
end: line.start + offset + mention.end,
|
|
192
|
+
attrs: mention.attrs,
|
|
193
|
+
})),
|
|
194
|
+
)
|
|
195
|
+
.sort((left, right) => left.start - right.start);
|
|
193
196
|
leaves.forEach((leaf) =>
|
|
194
197
|
parent.children.push({
|
|
195
198
|
kind: leaf.kind,
|
|
@@ -303,7 +306,7 @@ const readLine = (state: State, patterns: StructurePatterns, line: Line, heading
|
|
|
303
306
|
// 番号付きの行は番号の後ろだけを読む。「第3条(支払)」の「第3条」を自分への参照として拾わない。
|
|
304
307
|
const scanned = numbered === undefined ? text : numbered.rest;
|
|
305
308
|
const offset = Math.max(0, line.text.lastIndexOf(scanned));
|
|
306
|
-
if (scanned !== "") addLeaves(state, patterns, line, scanned, offset);
|
|
309
|
+
if (scanned !== "") addLeaves(state, patterns, line, scanned, offset, heading !== undefined);
|
|
307
310
|
};
|
|
308
311
|
|
|
309
312
|
/**
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { StructureKind } from "../plugin.ts";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 見出しの行から拾う葉か。見出しは節の題で、「『特別警報』とは」は下の本文が定義する語を掲げているだけで、定義ではない。
|
|
5
|
+
* 定義として数えると、本文の「『特別警報』とは、…」が二度目の定義に見える。参照や数量は見出しにあっても読む。
|
|
6
|
+
*/
|
|
7
|
+
export const readsOnHeading = (kind: StructureKind): boolean => kind !== "definition";
|