@chaffjs/lang-ja 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/citation.d.ts +6 -3
- package/dist/citation.d.ts.map +1 -1
- package/dist/citation.js +31 -9
- package/dist/citation.js.map +1 -1
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +4 -0
- package/dist/lexicons.js.map +1 -1
- package/dist/pos.d.ts +17 -1
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +69 -36
- package/dist/pos.js.map +1 -1
- package/dist/predicate-frame.d.ts +14 -0
- package/dist/predicate-frame.d.ts.map +1 -0
- package/dist/predicate-frame.js +22 -0
- package/dist/predicate-frame.js.map +1 -0
- package/dist/ra-dropped.d.ts +12 -0
- package/dist/ra-dropped.d.ts.map +1 -0
- package/dist/ra-dropped.js +18 -0
- package/dist/ra-dropped.js.map +1 -0
- package/dist/reduplication.d.ts +17 -5
- package/dist/reduplication.d.ts.map +1 -1
- package/dist/reduplication.js +85 -6
- package/dist/reduplication.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +41 -21
- package/dist/structure.js.map +1 -1
- package/dist/well-formed.d.ts +4 -1
- package/dist/well-formed.d.ts.map +1 -1
- package/dist/well-formed.js +5 -2
- package/dist/well-formed.js.map +1 -1
- package/lexicons/absolute-word.yaml +12 -0
- package/lexicons/abstract-heading.yaml +11 -0
- package/lexicons/adversative-particle.yaml +6 -0
- package/lexicons/age-unit.yaml +6 -0
- package/lexicons/ai-tell.yaml +36 -0
- package/lexicons/approximate-marker.yaml +32 -0
- package/lexicons/birth-origin.yaml +5 -0
- package/lexicons/bookend-heading.yaml +33 -0
- package/lexicons/chat-citation-marker.yaml +11 -0
- package/lexicons/citation-author-joiner.yaml +14 -0
- package/lexicons/citation-not-author.yaml +15 -0
- package/lexicons/comparison-marker.yaml +7 -0
- package/lexicons/demonstrative-opener.yaml +19 -0
- package/lexicons/distributive-noun.yaml +14 -2
- package/lexicons/double-negative.yaml +32 -0
- package/lexicons/doubled-nado.yaml +15 -0
- package/lexicons/doubled-word.yaml +12 -0
- package/lexicons/duration-day.yaml +6 -0
- package/lexicons/duration-month.yaml +16 -0
- package/lexicons/duration-week.yaml +6 -0
- package/lexicons/duration-year.yaml +5 -0
- package/lexicons/elapsed-link.yaml +8 -0
- package/lexicons/elapsed-origin.yaml +12 -0
- package/lexicons/elapsed-unit.yaml +7 -0
- package/lexicons/enumeration-frame.yaml +32 -0
- package/lexicons/enumeration-joiner.yaml +15 -0
- package/lexicons/enumeration-negation.yaml +10 -0
- package/lexicons/exception-marker.yaml +11 -0
- package/lexicons/fact-attribute.yaml +6 -0
- package/lexicons/fact-label-vague.yaml +13 -0
- package/lexicons/fact-separator.yaml +8 -0
- package/lexicons/fact-value-end.yaml +16 -0
- package/lexicons/figure-in-text.yaml +7 -0
- package/lexicons/heading-form.yaml +64 -0
- package/lexicons/heading-number-label.yaml +30 -0
- package/lexicons/idiom-misuse.yaml +69 -0
- package/lexicons/item-number.yaml +38 -0
- package/lexicons/known-misspelling.yaml +72 -0
- package/lexicons/label-affix.yaml +9 -0
- package/lexicons/light-verb.yaml +10 -0
- package/lexicons/long-vowel-distinct.yaml +25 -0
- package/lexicons/long-vowel-drop-after.yaml +25 -0
- package/lexicons/misconversion.yaml +117 -0
- package/lexicons/month-name.yaml +29 -0
- package/lexicons/name-note.yaml +5 -1
- package/lexicons/nominalization-event-noun.yaml +13 -0
- package/lexicons/nominalization-light-verb.yaml +10 -0
- package/lexicons/numeral-counter.yaml +42 -0
- package/lexicons/numeral-kanji.yaml +20 -0
- package/lexicons/orthographic-variant-distinct.yaml +20 -0
- package/lexicons/paper-section-heading.yaml +18 -0
- package/lexicons/passive-predicate-frame.yaml +13 -0
- package/lexicons/person-form.yaml +23 -0
- package/lexicons/person-word.yaml +20 -0
- package/lexicons/placeholder-word.yaml +19 -0
- package/lexicons/pro-con-heading.yaml +19 -0
- package/lexicons/ra-dropped-verb.yaml +7 -0
- package/lexicons/redundant-expression.yaml +67 -0
- package/lexicons/reference-aside.yaml +18 -0
- package/lexicons/reference-list-heading.yaml +13 -0
- package/lexicons/reference-topic-link.yaml +14 -0
- package/lexicons/reference-topic-vague.yaml +16 -0
- package/lexicons/relative-after.yaml +5 -0
- package/lexicons/relative-before.yaml +5 -0
- package/lexicons/relative-link.yaml +8 -0
- package/lexicons/relative-year-unit.yaml +5 -0
- package/lexicons/requirement-either.yaml +13 -0
- package/lexicons/requirement-loophole.yaml +13 -0
- package/lexicons/requirement-marker.yaml +15 -0
- package/lexicons/requirement-modal-jis-z8301-2019.yaml +14 -0
- package/lexicons/requirement-not-marker.yaml +8 -0
- package/lexicons/requirement-open-end.yaml +6 -0
- package/lexicons/restating-opener.yaml +18 -0
- package/lexicons/sasete-itadaku.yaml +11 -0
- package/lexicons/summary-heading.yaml +11 -0
- package/lexicons/teki-suffix.yaml +6 -0
- package/lexicons/ten-mark.yaml +6 -0
- package/lexicons/translationese.yaml +38 -0
- package/lexicons/unit-data.yaml +28 -0
- package/lexicons/unit-length.yaml +24 -0
- package/lexicons/unit-mass.yaml +22 -0
- package/lexicons/unit-time.yaml +24 -0
- package/lexicons/unit-volume.yaml +16 -0
- package/lexicons/unnumbered-heading.yaml +28 -0
- package/lexicons/vague-clause-pointer.yaml +14 -0
- package/lexicons/vague-figure-pointer.yaml +23 -0
- package/lexicons/vague-link-text.yaml +23 -0
- package/lexicons/vague-word.yaml +32 -0
- package/lexicons/value-unit.yaml +61 -0
- package/lexicons/year-reference.yaml +19 -0
- package/package.json +1 -1
- package/src/citation.ts +35 -10
- package/src/lexicons.ts +4 -0
- package/src/pos.ts +78 -29
- package/src/predicate-frame.ts +28 -0
- package/src/ra-dropped.ts +29 -0
- package/src/reduplication.ts +119 -8
- package/src/structure.ts +42 -19
- package/src/well-formed.ts +5 -2
package/src/pos.ts
CHANGED
|
@@ -4,7 +4,9 @@ import { analyserPieces } from "./analyser-pieces.ts";
|
|
|
4
4
|
import { readCounterTsu, type Morpheme } from "./counter-tsu.ts";
|
|
5
5
|
import { outsideTheReport, isPassiveForm, passiveVocabulary, readsAsPassive } from "./passive-reading.ts";
|
|
6
6
|
import { loadLexicons } from "./lexicons.ts";
|
|
7
|
-
import {
|
|
7
|
+
import { predicateFrameAfter } from "./predicate-frame.ts";
|
|
8
|
+
import { distributiveVocabulary, isEchoAt, type Inflection } from "./reduplication.ts";
|
|
9
|
+
import { isRaDroppedAt, raDroppedVocabulary } from "./ra-dropped.ts";
|
|
8
10
|
import { createRequire } from "node:module";
|
|
9
11
|
import { dirname, join } from "node:path";
|
|
10
12
|
import type { Token } from "chaffjs/plugin";
|
|
@@ -113,7 +115,15 @@ export const upos = (pos: string, detail: string): string => BY_DETAIL[detail] ?
|
|
|
113
115
|
* (自発や状態を言う動詞、名付けの「と呼ばれる」、尊敬の形と決まり文句、ら抜きと可能)と、文が報告する動作の外にあるもの
|
|
114
116
|
* (仮定の節、「〜されやすい」)は passive-reading.ts が外す。残りは「受動の形」として印を付ける。
|
|
115
117
|
*/
|
|
116
|
-
const
|
|
118
|
+
const LEXICONS = loadLexicons();
|
|
119
|
+
|
|
120
|
+
const PASSIVE_VOCABULARY = passiveVocabulary(LEXICONS);
|
|
121
|
+
|
|
122
|
+
const RA_DROPPED_VOCABULARY = raDroppedVocabulary(LEXICONS);
|
|
123
|
+
|
|
124
|
+
const PREDICATE_FRAMES = (LEXICONS["passive-predicate-frame"] ?? []).map((entry) => entry.pattern);
|
|
125
|
+
|
|
126
|
+
const REDUPLICATING_NOUNS = distributiveVocabulary(LEXICONS).nouns;
|
|
117
127
|
|
|
118
128
|
/**
|
|
119
129
|
* 非自立名詞(の・こと・もの・ため・はず)。品詞は名詞だが、単独では何も指さない。
|
|
@@ -146,14 +156,17 @@ export const isReady = (): boolean => state.ready !== undefined;
|
|
|
146
156
|
* span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
|
|
147
157
|
* 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
|
|
148
158
|
*/
|
|
149
|
-
|
|
159
|
+
/** 形態素を語にするときに、前後の形態素から決まる印。 */
|
|
160
|
+
type Marks = { readonly passive: boolean; readonly echo: boolean; readonly light: boolean; readonly raDropped: boolean };
|
|
161
|
+
|
|
162
|
+
const toToken = (morpheme: Morpheme, start: number, { passive, echo, light, raDropped }: Marks): Token => ({
|
|
150
163
|
span: { start, end: start + morpheme.surface_form.length },
|
|
151
164
|
surface: morpheme.surface_form,
|
|
152
165
|
// UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
|
|
153
166
|
pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
|
|
154
167
|
...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
|
|
155
168
|
...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
|
|
156
|
-
...withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo),
|
|
169
|
+
...withRaDropped(withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo), raDropped),
|
|
157
170
|
});
|
|
158
171
|
|
|
159
172
|
/**
|
|
@@ -183,6 +196,10 @@ const withWordStatus = (
|
|
|
183
196
|
const withEcho = (found: { features?: Readonly<Record<string, string>> }, echo: boolean): { features?: Readonly<Record<string, string>> } =>
|
|
184
197
|
echo ? { features: { ...found.features, Echo: "Rdp" } } : found;
|
|
185
198
|
|
|
199
|
+
/** ら抜き言葉の一部(「食べれる」の 食べ と れる、「見れる」)。日本語に固有の印で、UD の FEATS の言語別拡張。 */
|
|
200
|
+
const withRaDropped = (found: { features?: Readonly<Record<string, string>> }, raDropped: boolean): { features?: Readonly<Record<string, string>> } =>
|
|
201
|
+
raDropped ? { features: { ...found.features, PotentialRa: "Dropped" } } : found;
|
|
202
|
+
|
|
186
203
|
/**
|
|
187
204
|
* 数(名詞,数)。UPOS では名詞に寄せるので、数であることは UD の NumType=Card で渡す。
|
|
188
205
|
* 決まった言い回し(二人三脚、三日坊主、一人ひとり、十分)は辞書が一語として持つので、数にはならない。
|
|
@@ -222,9 +239,17 @@ const featuresOf = (morpheme: Morpheme, passive: boolean): { features?: Readonly
|
|
|
222
239
|
const name = placeType(morpheme) ?? personOrOrganisation(morpheme);
|
|
223
240
|
if (name !== undefined) return { features: { NameType: name } };
|
|
224
241
|
if (isCounter(morpheme)) return { features: { NounType: "Class" } };
|
|
242
|
+
if (isVerbalNoun(morpheme)) return { features: { VerbForm: "Vnoun" } };
|
|
225
243
|
return {};
|
|
226
244
|
};
|
|
227
245
|
|
|
246
|
+
/**
|
|
247
|
+
* サ変名詞(調査・確認)。「する」を付ければ動詞になる名詞。UD の VerbForm=Vnoun。
|
|
248
|
+
* 辞書は知らない記号(「(VM)を」の「)」)もサ変接続に入れるので、字(漢字・かな・英字)を含む語だけ。
|
|
249
|
+
*/
|
|
250
|
+
const LETTER = /\p{L}/u;
|
|
251
|
+
const isVerbalNoun = (morpheme: Morpheme): boolean => morpheme.pos === "名詞" && morpheme.pos_detail_1 === "サ変接続" && LETTER.test(morpheme.surface_form);
|
|
252
|
+
|
|
228
253
|
/**
|
|
229
254
|
* 名詞を修飾しているだけの受動から印を外す。「使用されるフレームワーク」
|
|
230
255
|
* 「開催される BootCamp」は動作主を隠しているのではなく、名前の付けかたで、
|
|
@@ -244,13 +269,27 @@ const BREAK = new Set(["、", ",", ",", "。", ".", "."]);
|
|
|
244
269
|
const clauseEnd = (tokens: readonly Token[], from: number): number =>
|
|
245
270
|
tokens.find((token) => token.span.start >= from && BREAK.has(token.surface))?.span.start ?? Number.MAX_SAFE_INTEGER;
|
|
246
271
|
|
|
272
|
+
const unmarked = (token: Token): Token => ({
|
|
273
|
+
span: token.span,
|
|
274
|
+
surface: token.surface,
|
|
275
|
+
pos: token.pos,
|
|
276
|
+
...(token.lemma === undefined ? {} : { lemma: token.lemma }),
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
const isDependentToken = (token: Token): boolean => token.features?.["NounType"] === "Dependent";
|
|
280
|
+
|
|
281
|
+
/** 受動の後ろ(from から節の終わりまで)に、受動が修飾する名詞があるか。型があれば、型の「こと」と後ろの「ため」のような非自立名詞は数えない。 */
|
|
282
|
+
const nounModifiedAfter = (tokens: readonly Token[], from: number, framed: boolean): boolean => {
|
|
283
|
+
const end = clauseEnd(tokens, from);
|
|
284
|
+
return tokens.some((other) => other.span.start >= from && other.span.end <= end && NOMINAL.has(other.pos) && !(framed && isDependentToken(other)));
|
|
285
|
+
};
|
|
286
|
+
|
|
247
287
|
export const predicateOnly = (tokens: readonly Token[]): Token[] =>
|
|
248
|
-
tokens.map((token) => {
|
|
288
|
+
tokens.map((token, index) => {
|
|
249
289
|
if (token.features?.["Voice"] !== "Pass") return token;
|
|
250
|
-
const
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
return { span: token.span, surface: token.surface, pos: token.pos, ...(token.lemma === undefined ? {} : { lemma: token.lemma }) };
|
|
290
|
+
const frame = predicateFrameAfter(tokens, index + 1, PREDICATE_FRAMES);
|
|
291
|
+
if (frame === undefined) return nounModifiedAfter(tokens, token.span.end, false) ? unmarked(token) : token;
|
|
292
|
+
return frame.conditional || nounModifiedAfter(tokens, token.span.end, true) ? unmarked(token) : token;
|
|
254
293
|
});
|
|
255
294
|
|
|
256
295
|
const inflectionOf = (morpheme: Morpheme, start: number): Inflection => ({
|
|
@@ -262,20 +301,24 @@ const inflectionOf = (morpheme: Morpheme, start: number): Inflection => ({
|
|
|
262
301
|
start,
|
|
263
302
|
});
|
|
264
303
|
|
|
304
|
+
/** 解析器を読み込んでいないか、読めなかったら undefined。 */
|
|
305
|
+
const readReady = (text: string): Placed[] | undefined => (state.ready === undefined ? undefined : readWith(state.ready, text));
|
|
306
|
+
|
|
265
307
|
export const tokenize = (text: string): Token[] | undefined => {
|
|
266
|
-
const
|
|
267
|
-
|
|
268
|
-
|
|
308
|
+
const read = readReady(text);
|
|
309
|
+
return read === undefined ? undefined : tokensOf(read);
|
|
310
|
+
};
|
|
311
|
+
|
|
312
|
+
const tokensOf = (read: readonly Placed[]): Token[] => {
|
|
269
313
|
const sequence = read.map(({ morpheme }) => morpheme);
|
|
270
314
|
const inflections = read.map(({ morpheme, start }) => inflectionOf(morpheme, start));
|
|
271
315
|
return read.map(({ morpheme, start }, index) =>
|
|
272
|
-
toToken(
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
),
|
|
316
|
+
toToken(morpheme, start, {
|
|
317
|
+
passive: readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index),
|
|
318
|
+
echo: isEchoAt(inflections, index, REDUPLICATING_NOUNS),
|
|
319
|
+
light: isLightVerbAt(sequence, index),
|
|
320
|
+
raDropped: isRaDroppedAt(sequence, index, RA_DROPPED_VOCABULARY),
|
|
321
|
+
}),
|
|
279
322
|
);
|
|
280
323
|
};
|
|
281
324
|
|
|
@@ -285,14 +328,23 @@ const PIECE_LIMIT = 1000;
|
|
|
285
328
|
const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] =>
|
|
286
329
|
readCounterTsu(analyserPieces(text, PIECE_LIMIT).flatMap((piece) => toArray(callMethod(tokenizer, "tokenize", [piece])).flatMap(toMorpheme)));
|
|
287
330
|
|
|
288
|
-
|
|
289
|
-
|
|
331
|
+
type Placed = { readonly morpheme: Morpheme; readonly start: number };
|
|
332
|
+
|
|
333
|
+
/**
|
|
334
|
+
* 読めない文字は先に置き換える(wellFormed)。それでも解析器が投げたら undefined を返し、品詞の無い文として渡す。
|
|
335
|
+
* 一つの文書のために run 全体を止めない。品詞が要る rule は、core が「読めなかった」として動かさない。
|
|
336
|
+
*/
|
|
337
|
+
export const readWith = (tokenizer: Tokenizer, text: string): Placed[] | undefined => {
|
|
290
338
|
const readable = wellFormed(text);
|
|
291
|
-
|
|
339
|
+
try {
|
|
340
|
+
return placed(readable, analyse(tokenizer, readable));
|
|
341
|
+
} catch {
|
|
342
|
+
return undefined;
|
|
343
|
+
}
|
|
292
344
|
};
|
|
293
345
|
|
|
294
346
|
/** 形態素と、本文の中での始まり。本文に見つからないものは落とす。 */
|
|
295
|
-
const placed = (text: string, raws: readonly Morpheme[]):
|
|
347
|
+
const placed = (text: string, raws: readonly Morpheme[]): Placed[] => {
|
|
296
348
|
const starts = surfaceStarts(
|
|
297
349
|
text,
|
|
298
350
|
raws.map((raw) => raw.surface_form),
|
|
@@ -316,11 +368,9 @@ export type Morph = {
|
|
|
316
368
|
readonly detail2: string;
|
|
317
369
|
};
|
|
318
370
|
|
|
319
|
-
/**
|
|
320
|
-
export const morphemes = (text: string): Morph[] | undefined =>
|
|
321
|
-
|
|
322
|
-
if (tokenizer === undefined) return undefined;
|
|
323
|
-
return readAll(tokenizer, text).map(({ morpheme: raw, start }) => ({
|
|
371
|
+
/** 解析器を読み込んでいないか、読めなければ undefined。呼ぶ側は、形態素なしの読み方に戻る。 */
|
|
372
|
+
export const morphemes = (text: string): Morph[] | undefined =>
|
|
373
|
+
readReady(text)?.map(({ morpheme: raw, start }) => ({
|
|
324
374
|
start,
|
|
325
375
|
end: start + raw.surface_form.length,
|
|
326
376
|
surface: raw.surface_form,
|
|
@@ -328,7 +378,6 @@ export const morphemes = (text: string): Morph[] | undefined => {
|
|
|
328
378
|
detail1: raw.pos_detail_1,
|
|
329
379
|
detail2: raw.pos_detail_2,
|
|
330
380
|
}));
|
|
331
|
-
};
|
|
332
381
|
|
|
333
382
|
const isCounterMorph = (morph: Morph): boolean => morph.pos === "名詞" && morph.detail1 === "接尾" && morph.detail2 === "助数詞";
|
|
334
383
|
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Token } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
const lemmaOf = (token: Token): string => token.lemma ?? token.surface;
|
|
4
|
+
|
|
5
|
+
/** at から読んだ語の原形をつないで、frame とちょうど同じになる語の数。ならなければ 0。語は 1 字以上なので、frame の字数より多くは読まない。 */
|
|
6
|
+
const lengthOf = (tokens: readonly Token[], at: number, frame: string): number => {
|
|
7
|
+
const lemmas = tokens.slice(at, at + frame.length).map(lemmaOf);
|
|
8
|
+
return lemmas.map((_lemma, index) => lemmas.slice(0, index + 1).join("")).indexOf(frame) + 1;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* at から始まる述語の型(「ことになる」「こととなる」)の語の数。型の名詞は修飾される名詞ではなく、述語の一部。
|
|
13
|
+
* 「検討されることとなった」の「こと」は、受動を名詞の修飾にしない。原形で照らすので、活用した形(なりました)にも当たる。
|
|
14
|
+
*/
|
|
15
|
+
export const predicateFrameAt = (tokens: readonly Token[], at: number, frames: readonly string[]): number =>
|
|
16
|
+
Math.max(0, ...frames.map((frame) => lengthOf(tokens, at, frame)));
|
|
17
|
+
|
|
18
|
+
/** 型のすぐ後ろで仮定の節を作る語(「検討されることとなれば」「〜ことになると」)。起きたことではなく、その前提を言う。 */
|
|
19
|
+
const CONDITIONAL = new Set(["ば", "と"]);
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* at から始まる述語の型と、その後ろが仮定の節かどうか。型が無ければ undefined。
|
|
23
|
+
* 型の名詞(こと・事)は非自立名詞なので、型の後ろの名詞を読む側は非自立名詞を数えなければ型を読み飛ばせる。
|
|
24
|
+
*/
|
|
25
|
+
export const predicateFrameAfter = (tokens: readonly Token[], at: number, frames: readonly string[]): { readonly conditional: boolean } | undefined => {
|
|
26
|
+
const length = predicateFrameAt(tokens, at, frames);
|
|
27
|
+
return length === 0 ? undefined : { conditional: CONDITIONAL.has(tokens[at + length]?.surface ?? "") };
|
|
28
|
+
};
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Lexicon } from "chaffjs/plugin";
|
|
2
|
+
import type { Morpheme } from "./counter-tsu.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* ら抜き言葉(「見れる」「食べれる」「来れる」)。一段動詞とカ変動詞の可能は「られる」で作る(食べられる・来られる)。
|
|
6
|
+
* 一段動詞・カ変動詞の未然形に「れる」が直に付けば「ら」が抜けている(食べ+れる、こ+れる)。五段動詞の可能動詞(走れる・帰れる)は解析器が一語の一段動詞と読むので当たらない。
|
|
7
|
+
* 解析器が一語として持つら抜きの形(見れる・来れる)は、語彙表 ra-dropped-verb に並べる。
|
|
8
|
+
*/
|
|
9
|
+
export type RaDroppedVocabulary = ReadonlySet<string>;
|
|
10
|
+
|
|
11
|
+
export const raDroppedVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): RaDroppedVocabulary =>
|
|
12
|
+
new Set((lexicons["ra-dropped-verb"] ?? []).map((entry) => entry.pattern));
|
|
13
|
+
|
|
14
|
+
/** 可能を「られる」で作る動詞(一段動詞とカ変動詞)。「れる」はその未然形にしか付かない。 */
|
|
15
|
+
const isRareruStem = (morpheme: Morpheme | undefined): boolean =>
|
|
16
|
+
morpheme?.pos === "動詞" && (morpheme.conjugated_type === "一段" || (morpheme.conjugated_type?.startsWith("カ変") ?? false));
|
|
17
|
+
|
|
18
|
+
/** 動詞に付いた「れる」(IPADIC の動詞,接尾)。 */
|
|
19
|
+
const isBareReru = (morpheme: Morpheme | undefined): boolean => morpheme?.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && morpheme.basic_form === "れる";
|
|
20
|
+
|
|
21
|
+
/** sequence[index] が、ら抜き言葉の一部(一段・カ変動詞の未然形とそれに付いた「れる」、または一語で持つら抜きの形)か。 */
|
|
22
|
+
export const isRaDroppedAt = (sequence: readonly Morpheme[], index: number, vocabulary: RaDroppedVocabulary): boolean => {
|
|
23
|
+
const morpheme = sequence[index];
|
|
24
|
+
if (morpheme === undefined) return false;
|
|
25
|
+
if (morpheme.pos === "動詞" && vocabulary.has(morpheme.basic_form)) return true;
|
|
26
|
+
// 「れ」も一段の活用をするので(こ・れ・ない の れ は一段の未然形)、先に「れる」かを見る。
|
|
27
|
+
if (isBareReru(morpheme)) return isRareruStem(sequence[index - 1]);
|
|
28
|
+
return isRareruStem(morpheme) && isBareReru(sequence[index + 1]);
|
|
29
|
+
};
|
package/src/reduplication.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { Lexicon, Token } from "chaffjs/plugin";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
* 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp
|
|
4
|
+
* 名詞を重ねて「それぞれの」を言う形(会社会社で、チームチームの)。重ねられるのは、まとまりや場合を指す名詞だけで、
|
|
5
|
+
* 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp を付ける。二字以上の和語・漢語の重なりは isWholeWordEcho も拾うので、
|
|
6
|
+
* ここが要るのは外来語(チーム)と一字の語(国)。
|
|
6
7
|
*/
|
|
7
8
|
export type Distributive = { readonly nouns: ReadonlySet<string>; readonly particles: ReadonlySet<string> };
|
|
8
9
|
|
|
@@ -153,8 +154,8 @@ const FREE_WORD = new Set(["名詞", "動詞", "形容詞"]);
|
|
|
153
154
|
/** 接尾語が付く語。 */
|
|
154
155
|
const HOST = new Set(["名詞", "動詞", "形容詞", "助動詞"]);
|
|
155
156
|
|
|
156
|
-
/**
|
|
157
|
-
const SMALL_KANA = /[
|
|
157
|
+
/** 拍に数えない小さい仮名(ちょ・しゃ・ティ)。 */
|
|
158
|
+
const SMALL_KANA = /[ぁぃぅぇぉゃゅょゎァィゥェォャュョヮ]/gu;
|
|
158
159
|
|
|
159
160
|
/** 擬音・擬態語の根は二拍(すう・きし・ちょん)。動詞に読まれた重なりは二拍までが擬音で、長い動詞の重なり(できるできる)は書き損じ。 */
|
|
160
161
|
const MIMETIC_ROOT_MORAE = 2;
|
|
@@ -177,7 +178,7 @@ const isAttachedSuffix = (before: Inflection | undefined, word: Inflection): boo
|
|
|
177
178
|
|
|
178
179
|
/**
|
|
179
180
|
* 擬音・擬態語は副詞として立つ。後ろに「と」「に」が続くか、行が終わる(歌の行末。解析器は改行を「記号,空白」と読む)。
|
|
180
|
-
*
|
|
181
|
+
* 句読点の前は、動詞の書き損じ(できるできる、)と見分けられない。
|
|
181
182
|
*/
|
|
182
183
|
const ADVERB_MARK = new Set(["と", "に"]);
|
|
183
184
|
|
|
@@ -187,7 +188,7 @@ const endsAsAdverb = (second: Inflection, next: Inflection | undefined): boolean
|
|
|
187
188
|
/**
|
|
188
189
|
* 平仮名だけの語を丸ごと重ねた形(きしきし・ちょんちょん・すうすう・しだいしだい・あはれあはれ)は擬音・擬態語か感動詞。
|
|
189
190
|
* 解析器は辞書に無い擬音を、表層の合う名詞や動詞に切って読む(すう = 吸う)ので、品詞ではなく平仮名であることと、副詞の位置に立つことで見分ける。
|
|
190
|
-
*
|
|
191
|
+
* 前の語に付いた接尾語(田中さんさん)と、副詞の位置に立たない動詞の重なり(できるできるように)は書き損じのまま。名詞の重なりは isWholeWordEcho が見る。
|
|
191
192
|
*/
|
|
192
193
|
export const isKanaEcho = (words: readonly Inflection[], index: number): boolean => {
|
|
193
194
|
const [before, first, second] = [words[index - 2], words[index - 1], words[index]];
|
|
@@ -201,5 +202,115 @@ export const isKanaEcho = (words: readonly Inflection[], index: number): boolean
|
|
|
201
202
|
);
|
|
202
203
|
};
|
|
203
204
|
|
|
204
|
-
/**
|
|
205
|
-
|
|
205
|
+
/** 名詞の細分類のうち、それだけでは語にならないもの(こと・の、さん・的、数)。 */
|
|
206
|
+
const DEPENDENT_NOUN = new Set(["非自立", "接尾", "数"]);
|
|
207
|
+
|
|
208
|
+
const isContentNoun = (word: Inflection): boolean => word.pos === "名詞" && !DEPENDENT_NOUN.has(word.detail);
|
|
209
|
+
|
|
210
|
+
/** 言い切りの形の形容詞(えらい・若い)。続く形(長く長く)は isInflectedEcho が見る。 */
|
|
211
|
+
const isPlainAdjective = (word: Inflection): boolean => word.pos === "形容詞" && word.detail === "自立" && word.form === "基本形";
|
|
212
|
+
|
|
213
|
+
/** 漢字だけで書いた名詞。漢語(確認・資料)の重なりはたいてい書き損じ。 */
|
|
214
|
+
const KANJI_ONLY = /^\p{Script=Han}+$/u;
|
|
215
|
+
|
|
216
|
+
/** 漢字の名詞でも重ねて言うもの。時を言う副詞的な名詞(毎年・各自・一瞬)と、形容動詞の語幹(駄目・大変)。 */
|
|
217
|
+
const REDUPLICATING_DETAIL = new Set(["副詞可能", "形容動詞語幹"]);
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* 重ねて言える名詞か。漢字だけの名詞は、人・単位・時・場所のまとまりを言うもの(個人・一行・時代・地域。語彙表 distributive-noun)と
|
|
221
|
+
* REDUPLICATING_DETAIL の細分類だけ。仮名や送り仮名のある名詞(それ・もちもち・早め)は丸ごと重ねれば畳語か強め。
|
|
222
|
+
*/
|
|
223
|
+
const reduplicates = (noun: Inflection, kanjiNouns: ReadonlySet<string>): boolean =>
|
|
224
|
+
!KANJI_ONLY.test(noun.surface) || REDUPLICATING_DETAIL.has(noun.detail) || kanjiNouns.has(noun.surface);
|
|
225
|
+
|
|
226
|
+
/** 二つ目の名詞は、解析器が前の名詞に付く接尾語と読むことがある(好き好き・嫌い嫌い の二つ目)。 */
|
|
227
|
+
const isWholeWordPair = (first: Inflection, second: Inflection, kanjiNouns: ReadonlySet<string>): boolean =>
|
|
228
|
+
(isContentNoun(first) && second.pos === "名詞" && reduplicates(first, kanjiNouns)) || (isPlainAdjective(first) && isPlainAdjective(second));
|
|
229
|
+
|
|
230
|
+
const KATAKANA = /^[\p{Script=Katakana}ー]+$/u;
|
|
231
|
+
|
|
232
|
+
/** 擬音・擬態語の根の拍の数(ムク・ブイ・ゴロン・ブー)。 */
|
|
233
|
+
const MIMETIC_UNIT = { fewest: 2, most: 3 } as const;
|
|
234
|
+
|
|
235
|
+
/** 伸ばす音が根の途中にある語(データ)は外来語。擬音の伸ばす音は根の終わりに来る(ブーブー)。 */
|
|
236
|
+
const INNER_LONG_VOWEL = /ー./u;
|
|
237
|
+
|
|
238
|
+
const isMimeticUnit = (surface: string): boolean => {
|
|
239
|
+
const morae = moraeOf(surface);
|
|
240
|
+
return morae >= MIMETIC_UNIT.fewest && morae <= MIMETIC_UNIT.most && !INNER_LONG_VOWEL.test(surface);
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
/** 「する」は名詞を動詞にするだけで、擬音の後ろの動詞(ブイブイ言わせる)とは違い、外来語の重なり(テストテストする)にも付く。 */
|
|
244
|
+
const LIGHT_VERB = "する";
|
|
245
|
+
|
|
246
|
+
/** 擬音は副詞として立つ。後ろに「と」「に」が続くか、行が終わるか(endsAsAdverb)、動詞に直接かかる(ブイブイ言わせる)。 */
|
|
247
|
+
const standsAsMimetic = (second: Inflection, next: Inflection | undefined): boolean =>
|
|
248
|
+
endsAsAdverb(second, next) || (next !== undefined && next.pos === "動詞" && next.detail === "自立" && next.surface !== LIGHT_VERB && touches(second, next));
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* 片仮名の語はたいてい外来語(ユーザー・テスト・メモ)で、重ねれば書き損じ。擬音(ムクムクと・ブイブイ言わせる)は、根が擬音の形で、
|
|
252
|
+
* 副詞の位置に立つものだけ。解析器は擬音の根も名詞(椋・ブイ)と読むので、品詞では見分けられない。
|
|
253
|
+
*/
|
|
254
|
+
const isLoanwordPair = (second: Inflection, next: Inflection | undefined): boolean =>
|
|
255
|
+
KATAKANA.test(second.surface) && !(isMimeticUnit(second.surface) && standsAsMimetic(second, next));
|
|
256
|
+
|
|
257
|
+
const sameAndTouching = (first: Inflection | undefined, second: Inflection | undefined): boolean =>
|
|
258
|
+
first !== undefined && second !== undefined && first.surface === second.surface && touches(first, second);
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* 内容語(名詞・形容詞)を丸ごと重ねた形は畳語か強め(個人個人・一行一行・駄目駄目・えらいえらい・それそれ・もちもち)。
|
|
262
|
+
* 漢語の名詞は reduplicates が認めるものだけで、ほかの重なり(確認確認・資料資料)は書き損じ。kanjiNouns は pos.ts が語彙表から渡す。
|
|
263
|
+
* 一字の漢字(法法・金金)は isIteratedKanji、外来語(ユーザーユーザー)と三つ続く重なり(早め早め早め)は書き損じのまま。
|
|
264
|
+
*/
|
|
265
|
+
export const isWholeWordEcho = (words: readonly Inflection[], index: number, kanjiNouns: ReadonlySet<string>): boolean => {
|
|
266
|
+
const [before, first, second] = [words[index - 2], words[index - 1], words[index]];
|
|
267
|
+
return (
|
|
268
|
+
first !== undefined &&
|
|
269
|
+
second !== undefined &&
|
|
270
|
+
isWholeWordPair(first, second, kanjiNouns) &&
|
|
271
|
+
sameAndTouching(first, second) &&
|
|
272
|
+
first.surface.length >= MIN_ECHO_LENGTH &&
|
|
273
|
+
!isLoanwordPair(second, words[index + 1]) &&
|
|
274
|
+
!sameAndTouching(before, first)
|
|
275
|
+
);
|
|
276
|
+
};
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* 重ねて笑い声や擬音になる一字の仮名。平仮名は笑い声の行(ははは・ふふふ・へへへ)だけで、助詞(ををを・ににに・よよよ)は書き損じ。
|
|
280
|
+
* 片仮名の一字(ババババ・ドドド)は擬音で、片仮名の助詞は書かない。
|
|
281
|
+
*/
|
|
282
|
+
const REPEATED_KANA = /^[はひふへほ\p{Script=Katakana}]$/u;
|
|
283
|
+
|
|
284
|
+
/** 前の語に付かずに立つ並びの、前に来てよい語。記号(「・、)と、笑い声の頭の「あ」(フィラー・感動詞)。 */
|
|
285
|
+
const FREE_BEFORE = new Set(["記号", "フィラー", "感動詞"]);
|
|
286
|
+
|
|
287
|
+
/** index の語を含む、同じ語が接して続く並びの最初の位置。 */
|
|
288
|
+
const runStart = (words: readonly Inflection[], index: number): number => {
|
|
289
|
+
const before = words.slice(0, index + 1).findLastIndex((word, at) => !sameAndTouching(words[at - 1], word));
|
|
290
|
+
return Math.max(before, 0);
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
/** 並びが前の語に付いていない(文の頭、空白の後ろ、記号やフィラーの後ろ)。付いていれば助詞・助動詞の書き損じ(資料ををを・行ったたた)。 */
|
|
294
|
+
const standsFree = (words: readonly Inflection[], start: number): boolean => {
|
|
295
|
+
const [before, first] = [words[start - 1], words[start]];
|
|
296
|
+
return before === undefined || first === undefined || !touches(before, first) || FREE_BEFORE.has(before.pos);
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
/**
|
|
300
|
+
* 笑い声・擬音の一字の仮名が三つ以上続き、前の語に付かずに立つ並び(ははは・あははは・ふふふ・ドドド)は笑い声か擬音。解析器は一字ずつ助詞などに切る。
|
|
301
|
+
* 二つだけの重なり(をを・たた・よよ・かか)、助詞の仮名の並び(ををを)、前の語に付いた並び(私ははは)は書き損じのまま。
|
|
302
|
+
*/
|
|
303
|
+
export const isKanaRepeat = (words: readonly Inflection[], index: number): boolean => {
|
|
304
|
+
const [before, first, second, after] = [words[index - 2], words[index - 1], words[index], words[index + 1]];
|
|
305
|
+
return (
|
|
306
|
+
second !== undefined &&
|
|
307
|
+
REPEATED_KANA.test(second.surface) &&
|
|
308
|
+
sameAndTouching(first, second) &&
|
|
309
|
+
(sameAndTouching(before, first) || sameAndTouching(second, after)) &&
|
|
310
|
+
standsFree(words, runStart(words, index))
|
|
311
|
+
);
|
|
312
|
+
};
|
|
313
|
+
|
|
314
|
+
/** 活用した語・仮名の語・内容語の重なりか、一字の仮名の三つ以上の並び。pos.ts が二つ目に Echo=Rdp を付ける。 */
|
|
315
|
+
export const isEchoAt = (words: readonly Inflection[], index: number, kanjiNouns: ReadonlySet<string>): boolean =>
|
|
316
|
+
isInflectedEcho(words, index) || isKanaEcho(words, index) || isWholeWordEcho(words, index, kanjiNouns) || isKanaRepeat(words, index);
|
package/src/structure.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Mention, NumberedLine, NumberingContext, StructurePatterns } from "chaffjs/plugin";
|
|
2
2
|
import { parseJapaneseNumber, toHalfWidth } from "./numbers.ts";
|
|
3
|
-
import { citationVocabulary, citedDocument } from "./citation.ts";
|
|
3
|
+
import { citationVocabulary, citedDocument, namedDocument } from "./citation.ts";
|
|
4
4
|
import { loadLexicons } from "./lexicons.ts";
|
|
5
5
|
import { countedAfter, dates, quantities } from "./quantities.ts";
|
|
6
6
|
import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
|
|
@@ -13,6 +13,12 @@ const SPACE = "[ \\t\\u3000]";
|
|
|
13
13
|
|
|
14
14
|
/** 見出しとして読めるのは、番号の直後が空白・括弧・行末のとき。「第3条に定める」は本文。 */
|
|
15
15
|
const ARTICLE = new RegExp(`^${SPACE}*第(?<n>${NUMBER})条(?:の(?<sub>${NUMBER}))?(?<rest>(?:${SPACE}|(|\\().*|)$`, "u");
|
|
16
|
+
/**
|
|
17
|
+
* 見出しの行では、題を詰めて書いた「第7条委託」「第2章概要」も読む。後ろが平仮名(「第3条に定める」「第2章では」)、
|
|
18
|
+
* 次の番号(「第2章第1節」)、並べる語(「第4条及び第5条」「第2条若しくは第3条」「第2条乃至第5条」)なら、番号は題の札でないので読まない。
|
|
19
|
+
*/
|
|
20
|
+
const TIGHT_TITLE = "(?<rest>[^\\s\\p{Script=Hiragana}0-90-9第及又並若乃].*)";
|
|
21
|
+
const ARTICLE_TIGHT = new RegExp(`^${SPACE}*第(?<n>${NUMBER})条(?:の(?<sub>${NUMBER}))?${TIGHT_TITLE}$`, "u");
|
|
16
22
|
/** 「第四十三条から第五十五条まで 削除」「第五百十六条及び第五百十七条 削除」。削られた条を一行でまとめる法令の書き方。 */
|
|
17
23
|
const ARTICLE_RANGE = new RegExp(
|
|
18
24
|
`^${SPACE}*第(?<n>${NUMBER})条(?:の(?<sub>${NUMBER}))?(?:から第(?<m>${NUMBER})条(?:の(?<msub>${NUMBER}))?まで|及び第(?<m2>${NUMBER})条(?:の(?<msub2>${NUMBER}))?)(?<rest>(?:${SPACE}|(|\\().*|)$`,
|
|
@@ -37,6 +43,9 @@ const headingOf = (rest: string, whole: boolean): string => {
|
|
|
37
43
|
return bracketed ?? (whole && !trimmed.includes("。") ? trimmed : "");
|
|
38
44
|
};
|
|
39
45
|
|
|
46
|
+
const headingGroups = (pattern: RegExp, line: string, context: NumberingContext): Readonly<Record<string, string | undefined>> | undefined =>
|
|
47
|
+
context.isHeading ? pattern.exec(line)?.groups : undefined;
|
|
48
|
+
|
|
40
49
|
const numberOf = (text: string | undefined): string | undefined => {
|
|
41
50
|
const value = text === undefined ? undefined : parseJapaneseNumber(text);
|
|
42
51
|
return value === undefined ? undefined : String(value);
|
|
@@ -88,8 +97,8 @@ const articleRange = (line: string): NumberedLine | undefined => {
|
|
|
88
97
|
};
|
|
89
98
|
};
|
|
90
99
|
|
|
91
|
-
const article = (line: string): NumberedLine | undefined => {
|
|
92
|
-
const groups = ARTICLE.exec(line)?.groups;
|
|
100
|
+
const article = (line: string, context: NumberingContext): NumberedLine | undefined => {
|
|
101
|
+
const groups = ARTICLE.exec(line)?.groups ?? headingGroups(ARTICLE_TIGHT, line, context);
|
|
93
102
|
const main = numberOf(groups?.["n"]);
|
|
94
103
|
if (groups === undefined || main === undefined) return undefined;
|
|
95
104
|
const sub = numberOf(groups["sub"]);
|
|
@@ -155,14 +164,15 @@ const item = (line: string, context: NumberingContext): NumberedLine | undefined
|
|
|
155
164
|
* 章と節は親の番地に続ける(第2編第1章 → pt2.ch1、その第3節 → pt2.ch1.3)。条は通し番号のまま。
|
|
156
165
|
*/
|
|
157
166
|
const CHAPTER = new RegExp(`^${SPACE}*第(?<n>${NUMBER})(?<unit>[編章節])(?<rest>(?:${SPACE}|(|\\().*|)$`, "u");
|
|
167
|
+
const CHAPTER_TIGHT = new RegExp(`^${SPACE}*第(?<n>${NUMBER})(?<unit>[編章節])${TIGHT_TITLE}$`, "u");
|
|
158
168
|
const CHAPTER_SHAPES: Readonly<Record<string, { readonly depth: number; readonly prefix: string }>> = {
|
|
159
169
|
編: { depth: -2, prefix: "pt" },
|
|
160
170
|
章: { depth: -1, prefix: "ch" },
|
|
161
171
|
節: { depth: 0, prefix: "" },
|
|
162
172
|
};
|
|
163
173
|
|
|
164
|
-
const chapter = (line: string): NumberedLine | undefined => {
|
|
165
|
-
const groups = CHAPTER.exec(line)?.groups;
|
|
174
|
+
const chapter = (line: string, context: NumberingContext): NumberedLine | undefined => {
|
|
175
|
+
const groups = CHAPTER.exec(line)?.groups ?? headingGroups(CHAPTER_TIGHT, line, context);
|
|
166
176
|
const number = numberOf(groups?.["n"]);
|
|
167
177
|
const shape = CHAPTER_SHAPES[groups?.["unit"] ?? ""];
|
|
168
178
|
if (groups === undefined || number === undefined || shape === undefined) return undefined;
|
|
@@ -181,7 +191,7 @@ const chapter = (line: string): NumberedLine | undefined => {
|
|
|
181
191
|
};
|
|
182
192
|
|
|
183
193
|
const numbered = (line: string, context: NumberingContext): NumberedLine | undefined =>
|
|
184
|
-
chapter(line) ?? articleRange(line) ?? article(line) ?? item(line, context);
|
|
194
|
+
chapter(line, context) ?? articleRange(line) ?? article(line, context) ?? item(line, context);
|
|
185
195
|
|
|
186
196
|
/** 正規表現の一致を Mention にする。g フラグ付きのものだけを渡す。 */
|
|
187
197
|
const mentions = (
|
|
@@ -208,10 +218,11 @@ const scopedLocally = (before: string): boolean => THIS_PART.test(before) || NEA
|
|
|
208
218
|
|
|
209
219
|
const definitions = (text: string): Mention[] =>
|
|
210
220
|
DEFINITIONS.flatMap((pattern) =>
|
|
211
|
-
mentions(pattern, text, (groups,
|
|
221
|
+
mentions(pattern, text, (groups, whole, start) => {
|
|
212
222
|
if (groups["term"] === undefined) return undefined;
|
|
213
223
|
const local = scopedLocally(text.slice(Math.max(0, start - 30), start));
|
|
214
|
-
|
|
224
|
+
const inline = whole.startsWith("以下");
|
|
225
|
+
return { term: groups["term"], ...(local ? { scope: "local" } : {}), ...(inline ? { placement: "inline" } : {}) };
|
|
215
226
|
}),
|
|
216
227
|
);
|
|
217
228
|
|
|
@@ -227,11 +238,20 @@ const REFERENCE = new RegExp(`第(?<a>${NUMBER})条(?:の(?<s>${NUMBER}))?(?:第
|
|
|
227
238
|
*/
|
|
228
239
|
/**
|
|
229
240
|
* 他の文書の参照に続く並び(「民事訴訟法第百条第一項、第百一条、第百二条の二」)は、同じ文書の条を指す。
|
|
230
|
-
*
|
|
241
|
+
* 間が読点・接続の語・号や項の断片・括弧書きだけなら、前の参照の文書を引き継ぐ。接続の語は、規約や案内が仮名で書く形(および)も同じ。
|
|
231
242
|
*/
|
|
232
|
-
const CONNECTORS =
|
|
243
|
+
const CONNECTORS = /及び|並びに|若しくは|又は|および|ならびに|もしくは|または|から|まで|ただし書|前段|後段|[、,\s]/gu;
|
|
233
244
|
const FRAGMENT = new RegExp(`第${NUMBER}[項号](?:の${NUMBER})*`, "gu");
|
|
234
|
-
|
|
245
|
+
/** 括弧書きの括弧。全角でも半角でもよい(厚生労働省法令等データベースは半角で書く)が、開きと閉じは同じ幅で組む。 */
|
|
246
|
+
const OPENER_OF: ReadonlyMap<string, string> = new Map([
|
|
247
|
+
[")", "("],
|
|
248
|
+
[")", "("],
|
|
249
|
+
]);
|
|
250
|
+
const isOpening = (char: string): boolean => char === "(" || char === "(";
|
|
251
|
+
const PARENTHESES = /([^()()]*)|\([^()()]*\)/gu;
|
|
252
|
+
|
|
253
|
+
/** 閉じの括弧が、開いたままの最後の括弧と組になるか。 */
|
|
254
|
+
const closes = (char: string, opened: string | undefined): boolean => opened !== undefined && OPENER_OF.get(char) === opened;
|
|
235
255
|
|
|
236
256
|
/** 間の文字列が、接続の語・読点・項や号の断片・括弧書きだけでできているか。 */
|
|
237
257
|
const isContinuation = (gap: string): boolean => gap.replace(PARENTHESES, "").replace(FRAGMENT, "").replace(CONNECTORS, "") === "";
|
|
@@ -239,11 +259,11 @@ const isContinuation = (gap: string): boolean => gap.replace(PARENTHESES, "").re
|
|
|
239
259
|
/** 行を一度だけ読んで、位置ごとの括弧の深さを出す。参照ごとに行を読み直さない。 */
|
|
240
260
|
const depthsOf = (text: string): Int32Array => {
|
|
241
261
|
const depth = new Int32Array(text.length + 1);
|
|
242
|
-
const
|
|
262
|
+
const opened: string[] = [];
|
|
243
263
|
text.split("").forEach((char, at) => {
|
|
244
|
-
depth[at] =
|
|
245
|
-
if (char
|
|
246
|
-
if (char
|
|
264
|
+
depth[at] = opened.length;
|
|
265
|
+
if (isOpening(char)) opened.push(char);
|
|
266
|
+
else if (closes(char, opened.at(-1))) opened.pop();
|
|
247
267
|
});
|
|
248
268
|
return depth;
|
|
249
269
|
};
|
|
@@ -262,10 +282,12 @@ const withoutClosedParentheses = (gap: string): string => {
|
|
|
262
282
|
const kept: string[] = [];
|
|
263
283
|
const opens: number[] = [];
|
|
264
284
|
for (const char of gap) {
|
|
265
|
-
const
|
|
266
|
-
if (
|
|
267
|
-
|
|
268
|
-
|
|
285
|
+
const last = opens.at(-1);
|
|
286
|
+
if (last !== undefined && closes(char, kept[last])) {
|
|
287
|
+
opens.pop();
|
|
288
|
+
kept.splice(last);
|
|
289
|
+
} else {
|
|
290
|
+
if (isOpening(char)) opens.push(kept.length);
|
|
269
291
|
kept.push(char);
|
|
270
292
|
}
|
|
271
293
|
}
|
|
@@ -398,4 +420,5 @@ export const structure: StructurePatterns = {
|
|
|
398
420
|
countedAfter: countedAfterNumber,
|
|
399
421
|
continuesSentence: startsWithParticle,
|
|
400
422
|
number,
|
|
423
|
+
namedDocument: (text, at) => namedDocument(text, at, CITATION),
|
|
401
424
|
};
|
package/src/well-formed.ts
CHANGED
|
@@ -1,2 +1,5 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
1
|
+
/**
|
|
2
|
+
* 解析器(kuromoji)が例外を投げる文字を U+FFFD に置き換える。対になっていないサロゲートと NUL(U+0000)。
|
|
3
|
+
* 長さを変えないので、位置はそのまま同じ文字を指す。BMP の全文字を一つずつ渡して、投げたのはこの二つだけだった。
|
|
4
|
+
*/
|
|
5
|
+
export const wellFormed = (text: string): string => text.replace(/[\p{Surrogate}\0]/gu, "�");
|