@chaffjs/lang-ja 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyser-pieces.d.ts +6 -0
- package/dist/analyser-pieces.d.ts.map +1 -0
- package/dist/analyser-pieces.js +37 -0
- package/dist/analyser-pieces.js.map +1 -0
- package/dist/citation.d.ts +3 -1
- package/dist/citation.d.ts.map +1 -1
- package/dist/citation.js +8 -3
- package/dist/citation.js.map +1 -1
- package/dist/counter-tsu.d.ts +5 -0
- package/dist/counter-tsu.d.ts.map +1 -1
- package/dist/counter-tsu.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -8
- package/dist/index.js.map +1 -1
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +5 -6
- package/dist/lexicons.js.map +1 -1
- package/dist/particle-after.d.ts +7 -0
- package/dist/particle-after.d.ts.map +1 -0
- package/dist/particle-after.js +9 -0
- package/dist/particle-after.js.map +1 -0
- package/dist/passive-reading.d.ts +29 -0
- package/dist/passive-reading.d.ts.map +1 -0
- package/dist/passive-reading.js +143 -0
- package/dist/passive-reading.js.map +1 -0
- package/dist/pos.d.ts +4 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +75 -19
- package/dist/pos.js.map +1 -1
- package/dist/reduplication.d.ts +33 -1
- package/dist/reduplication.d.ts.map +1 -1
- package/dist/reduplication.js +112 -1
- package/dist/reduplication.js.map +1 -1
- package/dist/sentence-split.js +1 -1
- package/dist/sentence-split.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +28 -13
- package/dist/structure.js.map +1 -1
- package/dist/tokens-within.d.ts +7 -0
- package/dist/tokens-within.d.ts.map +1 -0
- package/dist/tokens-within.js +29 -0
- package/dist/tokens-within.js.map +1 -0
- package/dist/well-formed.d.ts +3 -0
- package/dist/well-formed.d.ts.map +1 -0
- package/dist/well-formed.js +3 -0
- package/dist/well-formed.js.map +1 -0
- package/lexicons/calendar-unit.yaml +8 -0
- package/lexicons/calendar-year-unit.yaml +7 -0
- package/lexicons/common-acronym.yaml +4 -0
- package/lexicons/date-time-unit.yaml +12 -0
- package/lexicons/distributive-noun.yaml +2 -1
- package/lexicons/doubled-word.yaml +6 -0
- package/lexicons/emphasis-word.yaml +1 -0
- package/lexicons/excessive-hedging.yaml +9 -5
- package/lexicons/hedge-frame.yaml +14 -0
- package/lexicons/hedge-scope.yaml +7 -0
- package/lexicons/honorific-formula.yaml +10 -0
- package/lexicons/honorific.yaml +15 -0
- package/lexicons/http-method.yaml +14 -0
- package/lexicons/intransitive-verb.yaml +18 -0
- package/lexicons/iteration-kanji.yaml +18 -0
- package/lexicons/name-joiner.yaml +5 -0
- package/lexicons/naming-verb.yaml +9 -0
- package/lexicons/quantity-noun.yaml +42 -0
- package/lexicons/self-prefix.yaml +2 -1
- package/lexicons/spontaneous-verb.yaml +18 -0
- package/lexicons/stative-passive-verb.yaml +44 -0
- package/package.json +1 -1
- package/src/analyser-pieces.ts +38 -0
- package/src/citation.ts +11 -3
- package/src/counter-tsu.ts +5 -0
- package/src/index.ts +10 -11
- package/src/lexicons.ts +9 -8
- package/src/particle-after.ts +11 -0
- package/src/passive-reading.ts +195 -0
- package/src/pos.ts +84 -19
- package/src/reduplication.ts +175 -2
- package/src/sentence-split.ts +1 -1
- package/src/structure.ts +29 -14
- package/src/tokens-within.ts +27 -0
- package/src/well-formed.ts +2 -0
package/src/index.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { loadLexicons } from "./lexicons.ts";
|
|
2
2
|
import { sentenceSpans } from "./sentence-split.ts";
|
|
3
3
|
import { structure } from "./structure.ts";
|
|
4
|
-
import { isReady, predicateOnly, prepare, readsAsCounter, tokenize } from "./pos.ts";
|
|
4
|
+
import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.ts";
|
|
5
5
|
import { markSpacedCounters } from "./spaced-counter.ts";
|
|
6
|
-
import {
|
|
6
|
+
import { tokensWithin } from "./tokens-within.ts";
|
|
7
|
+
import { distributiveVocabulary, iterationMarkReading, markReduplication } from "./reduplication.ts";
|
|
7
8
|
import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
|
|
8
9
|
|
|
9
10
|
// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
|
|
@@ -37,10 +38,9 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
|
|
|
37
38
|
// 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
|
|
38
39
|
// 英文と訳文のように別の行のものまで 1 文に繋がる。
|
|
39
40
|
const acrossLines = last !== undefined && source.slice(last.end, span.start).includes("\n");
|
|
40
|
-
if (last !== undefined && !acrossLines && isOpen(source.slice(last.start, last.end))) {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
return [...acc, span];
|
|
41
|
+
if (last !== undefined && !acrossLines && isOpen(source.slice(last.start, last.end))) acc[acc.length - 1] = { start: last.start, end: span.end };
|
|
42
|
+
else acc.push(span);
|
|
43
|
+
return acc;
|
|
44
44
|
}, [])
|
|
45
45
|
.map((span) => ({ span, text: source.slice(span.start, span.end) }));
|
|
46
46
|
|
|
@@ -48,7 +48,9 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
|
|
|
48
48
|
* token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
|
|
49
49
|
* 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
|
|
50
50
|
*/
|
|
51
|
-
const
|
|
51
|
+
const LEXICONS = loadLexicons();
|
|
52
|
+
const DISTRIBUTIVE = distributiveVocabulary(LEXICONS);
|
|
53
|
+
const TAKES_ITERATION_MARK = iterationMarkReading(LEXICONS, readsAsOneWord);
|
|
52
54
|
|
|
53
55
|
const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
|
|
54
56
|
const read = tokenize(source);
|
|
@@ -57,10 +59,7 @@ const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[]
|
|
|
57
59
|
return sentences.map((sentence) => ({
|
|
58
60
|
...sentence,
|
|
59
61
|
// 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
|
|
60
|
-
tokens: markReduplication(
|
|
61
|
-
predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
|
|
62
|
-
DISTRIBUTIVE,
|
|
63
|
-
),
|
|
62
|
+
tokens: markReduplication(predicateOnly(tokensWithin(tokens, sentence.span)), DISTRIBUTIVE, readsAsOneAdverb, TAKES_ITERATION_MARK),
|
|
64
63
|
}));
|
|
65
64
|
};
|
|
66
65
|
|
package/src/lexicons.ts
CHANGED
|
@@ -27,11 +27,12 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
|
27
27
|
* 新しい言語のサポートは、ここを書くところから始まる。
|
|
28
28
|
*/
|
|
29
29
|
export const loadLexicons = (dir: string = DIR): Record<string, Lexicon> =>
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
30
|
+
Object.fromEntries(
|
|
31
|
+
readdirSync(dir)
|
|
32
|
+
.filter((file) => file.endsWith(".yaml"))
|
|
33
|
+
.flatMap((file): [string, Lexicon][] => {
|
|
34
|
+
const raw: unknown = parse(readFileSync(join(dir, file), "utf8"));
|
|
35
|
+
if (!isRecord(raw) || typeof raw["id"] !== "string" || !Array.isArray(raw["entries"])) return [];
|
|
36
|
+
return [[raw["id"], raw["entries"].map(toEntry).filter((entry) => entry !== undefined)]];
|
|
37
|
+
}),
|
|
38
|
+
);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { morphemes } from "./pos.ts";
|
|
2
|
+
|
|
3
|
+
const PARTICLE = "助詞";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* 行頭の番号の後ろが助詞で始まる(「3.11.0 を公開しました。」「4.2 は廃止した。」)なら、番号は文の中の語。
|
|
7
|
+
* 節の題(「4.2 設定」)は名詞で始まる。助詞は前の語が無いと接続詞と読まれる(「が」「で」)ので、番号に続けて読む。
|
|
8
|
+
* 解析器を読み込んでいなければ決めない。
|
|
9
|
+
*/
|
|
10
|
+
export const startsWithParticle = (number: string, rest: string): boolean =>
|
|
11
|
+
morphemes(`${number}${rest}`)?.find((morph) => morph.start >= number.length)?.pos === PARTICLE;
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import type { Lexicon } from "chaffjs/plugin";
|
|
2
|
+
import type { Morpheme } from "./counter-tsu.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* 「れる/られる」は受動・自発・尊敬・可能のどれにもなる。IPADIC はその区別を付けないので、
|
|
6
|
+
* 形のうえで受動でないと言えるものだけを外す。どれにも当たらなければ受動の形として残す。
|
|
7
|
+
*/
|
|
8
|
+
export type PassiveVocabulary = {
|
|
9
|
+
/** 自発に読む動詞の原形(考える・思う)。 */
|
|
10
|
+
readonly spontaneous: ReadonlySet<string>;
|
|
11
|
+
/** 状態・決まり・文書の中身を言う動詞の原形とサ変名詞(含む・定める・適用・記載)。 */
|
|
12
|
+
readonly stative: ReadonlySet<string>;
|
|
13
|
+
/** 尊敬の決まり文句(におかれましては)。 */
|
|
14
|
+
readonly formulas: readonly string[];
|
|
15
|
+
/** 受動を作らない自動詞の原形(来る・取り組む)と、「する」を付けて自動詞になるサ変名詞(参加・辞任)。 */
|
|
16
|
+
readonly intransitive: ReadonlySet<string>;
|
|
17
|
+
/** 「と」を受けて名前を言う動詞の原形(呼ぶ)。 */
|
|
18
|
+
readonly naming: ReadonlySet<string>;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const patternsOf = (lexicon: Lexicon | undefined): string[] => (lexicon ?? []).map((entry) => entry.pattern).filter((pattern) => pattern !== "");
|
|
22
|
+
|
|
23
|
+
export const passiveVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): PassiveVocabulary => ({
|
|
24
|
+
spontaneous: new Set(patternsOf(lexicons["spontaneous-verb"])),
|
|
25
|
+
stative: new Set(patternsOf(lexicons["stative-passive-verb"])),
|
|
26
|
+
formulas: patternsOf(lexicons["honorific-formula"]),
|
|
27
|
+
intransitive: new Set(patternsOf(lexicons["intransitive-verb"])),
|
|
28
|
+
naming: new Set(patternsOf(lexicons["naming-verb"])),
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const PASSIVE_LEMMA = new Set(["れる", "られる"]);
|
|
32
|
+
|
|
33
|
+
/** IPADIC では「れる/られる」は動詞の接尾として出る。 */
|
|
34
|
+
export const isPassiveForm = (morpheme: Morpheme): boolean =>
|
|
35
|
+
morpheme.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && PASSIVE_LEMMA.has(morpheme.basic_form);
|
|
36
|
+
|
|
37
|
+
/** 述語の続き(ます・ている・ておる・てはいない・た)。ここに「た」があれば過去。 */
|
|
38
|
+
const LINKING_TE = new Set(["て", "で"]);
|
|
39
|
+
|
|
40
|
+
/** IPADIC は「てもいた」の「い」を自立の動詞と読む。 */
|
|
41
|
+
const ASPECT_VERB = new Set(["いる", "おる"]);
|
|
42
|
+
|
|
43
|
+
/** 「られ・て・おり・まし・た」のように続く語の数の上限。文書全体の残りを毎回切り出さない。 */
|
|
44
|
+
const PREDICATE_REACH = 8;
|
|
45
|
+
|
|
46
|
+
const continuesPredicate = (morpheme: Morpheme): boolean =>
|
|
47
|
+
morpheme.pos === "助動詞" ||
|
|
48
|
+
(morpheme.pos === "動詞" && (morpheme.pos_detail_1 === "非自立" || ASPECT_VERB.has(morpheme.basic_form))) ||
|
|
49
|
+
(morpheme.pos_detail_1 === "接続助詞" && LINKING_TE.has(morpheme.surface_form)) ||
|
|
50
|
+
morpheme.pos_detail_1 === "係助詞";
|
|
51
|
+
|
|
52
|
+
const isPast = (morpheme: Morpheme): boolean => morpheme.pos === "助動詞" && morpheme.basic_form === "た";
|
|
53
|
+
|
|
54
|
+
/** at の「れる/られる」に続く述語の語(ます・ている・た)。述語の外の最初の語は含まない。 */
|
|
55
|
+
const predicateAfter = (morphemes: readonly Morpheme[], at: number): readonly Morpheme[] => {
|
|
56
|
+
const rest = morphemes.slice(at + 1, at + 1 + PREDICATE_REACH);
|
|
57
|
+
const end = rest.findIndex((morpheme) => !continuesPredicate(morpheme));
|
|
58
|
+
return rest.slice(0, end === -1 ? rest.length : end);
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
const pastFollows = (morphemes: readonly Morpheme[], at: number): boolean => predicateAfter(morphemes, at).some(isPast);
|
|
62
|
+
|
|
63
|
+
const isAspect = (morpheme: Morpheme): boolean => morpheme.pos === "動詞" && ASPECT_VERB.has(morpheme.basic_form);
|
|
64
|
+
|
|
65
|
+
/** 「た」で終わる出来事の述語。「適用された」は誰かが適用した動作、「記載されていた」は過去の状態。 */
|
|
66
|
+
const eventPastFollows = (morphemes: readonly Morpheme[], at: number): boolean => {
|
|
67
|
+
const predicate = predicateAfter(morphemes, at);
|
|
68
|
+
return predicate.some(isPast) && !predicate.some(isAspect);
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* 自発は書き手がいま思うことを言う。過去の「考えられた」「解された」は、会議や裁判所など誰かの考えた動作の受動になる。
|
|
73
|
+
* 補助動詞(動詞,非自立)の後ろは見ない。「務めてこられた」は尊敬でも、「連れてこられた」「持っていかれた」は受動で、形では分けられない。
|
|
74
|
+
*/
|
|
75
|
+
const verbReadsOtherwise = (morphemes: readonly Morpheme[], at: number, verb: Morpheme, vocabulary: PassiveVocabulary): boolean =>
|
|
76
|
+
(vocabulary.spontaneous.has(verb.basic_form) && !pastFollows(morphemes, at)) ||
|
|
77
|
+
statesSomething(morphemes, at, vocabulary) ||
|
|
78
|
+
namesSomething(morphemes, at, verb, vocabulary) ||
|
|
79
|
+
withoutRa(verb, morphemes[at]) ||
|
|
80
|
+
cannotBeDone(morphemes, at, verb);
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* 状態・決まり・文書の中身を言う受動(「適用される」「定められている」「記載されている」)。サ変名詞は「する」の前で見る。
|
|
84
|
+
* 「た」で終わる出来事(「適用された」「定められた」)は、誰かがした動作の受動として残す。
|
|
85
|
+
*/
|
|
86
|
+
const statesSomething = (morphemes: readonly Morpheme[], at: number, vocabulary: PassiveVocabulary): boolean =>
|
|
87
|
+
lemmasBefore(morphemes, at).some((lemma) => vocabulary.stative.has(lemma)) && !eventPastFollows(morphemes, at);
|
|
88
|
+
|
|
89
|
+
const isSuru = (morpheme: Morpheme | undefined): boolean => morpheme?.pos === "動詞" && morpheme.basic_form === "する";
|
|
90
|
+
|
|
91
|
+
/** 「れる/られる」の直前の動詞の原形と、それが「する」ならその前のサ変名詞(「適用される」の適用)。 */
|
|
92
|
+
const lemmasBefore = (morphemes: readonly Morpheme[], at: number): string[] => {
|
|
93
|
+
const verb = morphemes[at - 1];
|
|
94
|
+
const noun = morphemes[at - 2];
|
|
95
|
+
if (verb === undefined) return [];
|
|
96
|
+
return isSuru(verb) && noun?.pos_detail_1 === "サ変接続" ? [verb.basic_form, noun.basic_form] : [verb.basic_form];
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* 一段動詞の受動は「られる」で作る(「変えられる」)。一段動詞に直に付いた「れる」は、ら抜きの可能(「見れる」)か、
|
|
101
|
+
* 「とらえれいただければ」のような誤字を解析器が受動と読んだもの。
|
|
102
|
+
*/
|
|
103
|
+
const withoutRa = (verb: Morpheme, passive: Morpheme | undefined): boolean => verb.conjugated_type === "一段" && passive?.basic_form === "れる";
|
|
104
|
+
|
|
105
|
+
const NEGATION = new Set(["ない", "ぬ"]);
|
|
106
|
+
|
|
107
|
+
const isNegation = (morpheme: Morpheme | undefined): boolean => morpheme?.pos === "助動詞" && NEGATION.has(morpheme.basic_form);
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* 一段動詞の「られる」は受動と可能が同じ形。打ち消しが直に続くと可能に読む(「他人は変えられない」「質問に答えられず」)。
|
|
111
|
+
* 「ている」を挟んだ「変えられていない」は、まだ変えていない状態の受動として残す。
|
|
112
|
+
*/
|
|
113
|
+
const cannotBeDone = (morphemes: readonly Morpheme[], at: number, verb: Morpheme): boolean => verb.conjugated_type === "一段" && isNegation(morphemes[at + 1]);
|
|
114
|
+
|
|
115
|
+
/** 「〜と呼ばれる」「〜とも呼ばれています」は名前を言うもので、呼んだ誰かを隠していない。「と」の無い「会議に呼ばれた」は受動。 */
|
|
116
|
+
const namesSomething = (morphemes: readonly Morpheme[], at: number, verb: Morpheme, vocabulary: PassiveVocabulary): boolean => {
|
|
117
|
+
if (!vocabulary.naming.has(verb.basic_form)) return false;
|
|
118
|
+
const before = morphemes[at - 2]?.pos_detail_1 === "係助詞" ? morphemes[at - 3] : morphemes[at - 2];
|
|
119
|
+
return before?.pos === "助詞" && before.surface_form === "と";
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* 自動詞には、動作を受ける側を主語にする受動が無い。「来られ」「取り組まれ」「辞任され」は尊敬か可能。
|
|
124
|
+
* 本動詞(動詞,自立)だけを見る。補助動詞の「連れてこられた」は「連れてくる」全体の受動。
|
|
125
|
+
*/
|
|
126
|
+
const intransitiveVerb = (morphemes: readonly Morpheme[], at: number, vocabulary: PassiveVocabulary): boolean => {
|
|
127
|
+
const verb = morphemes[at - 1];
|
|
128
|
+
if (verb?.pos !== "動詞" || verb.pos_detail_1 !== "自立") return false;
|
|
129
|
+
return lemmasBefore(morphemes, at).some((lemma) => vocabulary.intransitive.has(lemma));
|
|
130
|
+
};
|
|
131
|
+
|
|
132
|
+
/** 「おる」は受動を作らないので、「しておられる」の「れる」は尊敬。 */
|
|
133
|
+
const HONORIFIC_BASE = "おる";
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* 形のうえで尊敬と言える「れる/られる」。主語が動作をする人なので、隠れた動作主はいない。
|
|
137
|
+
* 「ご用意された」「お会いされた」の お・ご は見ない。謙譲の「ご用意する」の受動(「資料がご用意されました」)と形が同じ。
|
|
138
|
+
*/
|
|
139
|
+
const honoursTheDoer = (morphemes: readonly Morpheme[], at: number, vocabulary: PassiveVocabulary): boolean =>
|
|
140
|
+
(morphemes[at - 1]?.pos === "動詞" && morphemes[at - 1]?.basic_form === HONORIFIC_BASE) || intransitiveVerb(morphemes, at, vocabulary);
|
|
141
|
+
|
|
142
|
+
const textOf = (morphemes: readonly Morpheme[]): string => morphemes.map((morpheme) => morpheme.surface_form).join("");
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* at の語を覆う決まり文句が、語の区切りから始まっているか。語は 1 文字以上あるので、
|
|
146
|
+
* 決まり文句の文字数だけ前の語までさかのぼれば足りる。文書全体を毎回つなげない。
|
|
147
|
+
*/
|
|
148
|
+
const insideFormula = (morphemes: readonly Morpheme[], at: number, formulas: readonly string[]): boolean => {
|
|
149
|
+
const reach = Math.max(0, ...formulas.map((formula) => formula.length));
|
|
150
|
+
const from = Math.max(0, at - reach + 1);
|
|
151
|
+
return morphemes.slice(from, at + 1).some((_, offset) => {
|
|
152
|
+
const start = from + offset;
|
|
153
|
+
const lead = textOf(morphemes.slice(start, at)).length;
|
|
154
|
+
const text = textOf(morphemes.slice(start, start + reach));
|
|
155
|
+
return formulas.some((formula) => lead < formula.length && text.startsWith(formula));
|
|
156
|
+
});
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
/** morphemes[at] が受動と読める「れる/られる」か。直前の動詞、尊敬の形、囲む決まり文句を見る。 */
|
|
160
|
+
export const readsAsPassive = (morphemes: readonly Morpheme[], at: number, vocabulary: PassiveVocabulary): boolean => {
|
|
161
|
+
const morpheme = morphemes[at];
|
|
162
|
+
if (morpheme === undefined || !isPassiveForm(morpheme)) return false;
|
|
163
|
+
const verb = morphemes[at - 1];
|
|
164
|
+
if (verb?.pos === "動詞" && verbReadsOtherwise(morphemes, at, verb, vocabulary)) return false;
|
|
165
|
+
if (honoursTheDoer(morphemes, at, vocabulary)) return false;
|
|
166
|
+
return !insideFormula(morphemes, at, vocabulary.formulas);
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* 述語を従える接続助詞のうち、仮定の節を作るもの。逆接の「が」「けど」は入れない。「確認されていないが、」は起きたことを言い切っている。
|
|
171
|
+
*/
|
|
172
|
+
const CONDITIONAL = new Set(["ば", "と"]);
|
|
173
|
+
|
|
174
|
+
/** 「見直されなければならない」「図られなければなりません」の「ば」は仮定ではなく義務の言い方の一部。 */
|
|
175
|
+
const OBLIGATION = new Set(["なる", "いける"]);
|
|
176
|
+
|
|
177
|
+
const isObligation = (morpheme: Morpheme | undefined): boolean => morpheme?.pos === "動詞" && OBLIGATION.has(morpheme.basic_form);
|
|
178
|
+
|
|
179
|
+
/** 受動が仮定の節の述語か。「立証されれば」「整理されていると」は起きたことではなく、その前提。 */
|
|
180
|
+
const inConditionalClause = (morphemes: readonly Morpheme[], at: number): boolean => {
|
|
181
|
+
const end = at + 1 + predicateAfter(morphemes, at).length;
|
|
182
|
+
const next = morphemes[end];
|
|
183
|
+
return next?.pos_detail_1 === "接続助詞" && CONDITIONAL.has(next.basic_form) && !isObligation(morphemes[end + 1]);
|
|
184
|
+
};
|
|
185
|
+
|
|
186
|
+
/** 受動の連用形に付いて、起きやすさ・起こりうることを言う語(「理解されやすい」「開催されづらい」「解釈され得る」「放置されがち」)。 */
|
|
187
|
+
const TENDENCY = new Set(["やすい", "にくい", "づらい", "得る", "うる", "がち"]);
|
|
188
|
+
|
|
189
|
+
const isTendency = (morpheme: Morpheme | undefined): boolean => morpheme !== undefined && TENDENCY.has(morpheme.basic_form);
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* 受動の「れる/られる」が、文の報告する動作の外にあるか。仮定の節と、起きやすさを言う形は、誰かが実際にした動作を言っていない。
|
|
193
|
+
* 隠れた動作主を問うのは、文末や「〜され、」で続く述語の受動のほう。
|
|
194
|
+
*/
|
|
195
|
+
export const outsideTheReport = (morphemes: readonly Morpheme[], at: number): boolean => isTendency(morphemes[at + 1]) || inConditionalClause(morphemes, at);
|
package/src/pos.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { surfaceStarts } from "./surface-starts.ts";
|
|
2
|
+
import { wellFormed } from "./well-formed.ts";
|
|
3
|
+
import { analyserPieces } from "./analyser-pieces.ts";
|
|
2
4
|
import { readCounterTsu, type Morpheme } from "./counter-tsu.ts";
|
|
5
|
+
import { outsideTheReport, isPassiveForm, passiveVocabulary, readsAsPassive } from "./passive-reading.ts";
|
|
6
|
+
import { loadLexicons } from "./lexicons.ts";
|
|
7
|
+
import { isEchoAt, type Inflection } from "./reduplication.ts";
|
|
3
8
|
import { createRequire } from "node:module";
|
|
4
9
|
import { dirname, join } from "node:path";
|
|
5
10
|
import type { Token } from "chaffjs/plugin";
|
|
@@ -22,20 +27,35 @@ const isCallable = (value: unknown): value is (...args: readonly unknown[]) => u
|
|
|
22
27
|
|
|
23
28
|
const isTokenizer = (value: unknown): value is Tokenizer => isRecord(value) && isCallable(value["tokenize"]);
|
|
24
29
|
|
|
25
|
-
/** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ
|
|
30
|
+
/** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ *。三段目(人名の姓・名)は無ければ持たない。 */
|
|
26
31
|
const toMorpheme = (value: unknown): Morpheme[] => {
|
|
27
32
|
if (!isRecord(value)) return [];
|
|
28
|
-
const [surface, pos, detail1, detail2, basic, reading] = [
|
|
33
|
+
const [surface, pos, detail1, detail2, detail3, basic, reading, form, conjugation] = [
|
|
29
34
|
value["surface_form"],
|
|
30
35
|
value["pos"],
|
|
31
36
|
value["pos_detail_1"],
|
|
32
37
|
value["pos_detail_2"],
|
|
38
|
+
value["pos_detail_3"],
|
|
33
39
|
value["basic_form"],
|
|
34
40
|
value["reading"],
|
|
41
|
+
value["conjugated_form"],
|
|
42
|
+
value["conjugated_type"],
|
|
35
43
|
];
|
|
36
44
|
if (typeof surface !== "string" || typeof pos !== "string" || typeof detail1 !== "string" || typeof basic !== "string") return [];
|
|
37
45
|
const detail = typeof detail2 === "string" ? detail2 : "*";
|
|
38
|
-
return [
|
|
46
|
+
return [
|
|
47
|
+
{
|
|
48
|
+
surface_form: surface,
|
|
49
|
+
pos,
|
|
50
|
+
pos_detail_1: detail1,
|
|
51
|
+
pos_detail_2: detail,
|
|
52
|
+
...(typeof detail3 === "string" ? { pos_detail_3: detail3 } : {}),
|
|
53
|
+
basic_form: basic,
|
|
54
|
+
...(typeof reading === "string" ? { reading } : {}),
|
|
55
|
+
...(typeof form === "string" && form !== "*" ? { conjugated_form: form } : {}),
|
|
56
|
+
...(typeof conjugation === "string" && conjugation !== "*" ? { conjugated_type: conjugation } : {}),
|
|
57
|
+
},
|
|
58
|
+
];
|
|
39
59
|
};
|
|
40
60
|
|
|
41
61
|
const dictionaryPath = (): string => join(dirname(require.resolve("@sglkc/kuromoji/package.json")), "dict");
|
|
@@ -89,13 +109,11 @@ const BY_POS: Readonly<Record<string, string>> = {
|
|
|
89
109
|
export const upos = (pos: string, detail: string): string => BY_DETAIL[detail] ?? BY_POS[pos] ?? "X";
|
|
90
110
|
|
|
91
111
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
112
|
+
* 受動の「れる/られる」。この語は可能・尊敬・自発も表す。形のうえで受動でないと言えるもの
|
|
113
|
+
* (自発や状態を言う動詞、名付けの「と呼ばれる」、尊敬の形と決まり文句、ら抜きと可能)と、文が報告する動作の外にあるもの
|
|
114
|
+
* (仮定の節、「〜されやすい」)は passive-reading.ts が外す。残りは「受動の形」として印を付ける。
|
|
95
115
|
*/
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
const isPassive = (morpheme: Morpheme): boolean => morpheme.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && PASSIVE_LEMMA.has(morpheme.basic_form);
|
|
116
|
+
const PASSIVE_VOCABULARY = passiveVocabulary(loadLexicons());
|
|
99
117
|
|
|
100
118
|
/**
|
|
101
119
|
* 非自立名詞(の・こと・もの・ため・はず)。品詞は名詞だが、単独では何も指さない。
|
|
@@ -128,16 +146,20 @@ export const isReady = (): boolean => state.ready !== undefined;
|
|
|
128
146
|
* span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
|
|
129
147
|
* 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
|
|
130
148
|
*/
|
|
131
|
-
const toToken = (morpheme: Morpheme, start: number): Token => ({
|
|
149
|
+
const toToken = (morpheme: Morpheme, start: number, passive: boolean, echo: boolean): Token => ({
|
|
132
150
|
span: { start, end: start + morpheme.surface_form.length },
|
|
133
151
|
surface: morpheme.surface_form,
|
|
134
152
|
// UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
|
|
135
|
-
pos:
|
|
153
|
+
pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
|
|
136
154
|
...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
|
|
137
155
|
...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
|
|
138
|
-
...featuresOf(morpheme),
|
|
156
|
+
...withEcho(featuresOf(morpheme, passive), echo),
|
|
139
157
|
});
|
|
140
158
|
|
|
159
|
+
/** 重ね言葉の二つ目(UD の Echo=Rdp)。ほかの印は残す。 */
|
|
160
|
+
const withEcho = (found: { features?: Readonly<Record<string, string>> }, echo: boolean): { features?: Readonly<Record<string, string>> } =>
|
|
161
|
+
echo ? { features: { ...found.features, Echo: "Rdp" } } : found;
|
|
162
|
+
|
|
141
163
|
/**
|
|
142
164
|
* 数(名詞,数)。UPOS では名詞に寄せるので、数であることは UD の NumType=Card で渡す。
|
|
143
165
|
* 決まった言い回し(二人三脚、三日坊主、一人ひとり、十分)は辞書が一語として持つので、数にはならない。
|
|
@@ -157,15 +179,25 @@ const placeType = (morpheme: Morpheme): string | undefined => {
|
|
|
157
179
|
return morpheme.pos_detail_1 === "接尾" ? "GeoUnit" : undefined;
|
|
158
180
|
};
|
|
159
181
|
|
|
182
|
+
/** 人名(IPADIC の「固有名詞,人名」)の三段目。姓は Sur、名は Giv、どちらとも言えない人名は Prs(UD の NameType)。 */
|
|
183
|
+
const PERSON: Readonly<Record<string, string>> = { 姓: "Sur", 名: "Giv", 一般: "Prs" };
|
|
184
|
+
|
|
185
|
+
/** 人名と組織名(「固有名詞,組織」は UD の NameType=Com)。地名は placeType が見る。 */
|
|
186
|
+
const personOrOrganisation = (morpheme: Morpheme): string | undefined => {
|
|
187
|
+
if (morpheme.pos !== "名詞" || morpheme.pos_detail_1 !== "固有名詞") return undefined;
|
|
188
|
+
if (morpheme.pos_detail_2 === "人名") return PERSON[morpheme.pos_detail_3 ?? ""];
|
|
189
|
+
return morpheme.pos_detail_2 === "組織" ? "Com" : undefined;
|
|
190
|
+
};
|
|
191
|
+
|
|
160
192
|
/** 数を数える単位(IPADIC の「接尾,助数詞」: 丁目・件・人)。UD では NounType=Class。 */
|
|
161
193
|
const isCounter = (morpheme: Morpheme): boolean => morpheme.pos === "名詞" && morpheme.pos_detail_1 === "接尾" && morpheme.pos_detail_2 === "助数詞";
|
|
162
194
|
|
|
163
|
-
const featuresOf = (morpheme: Morpheme): { features?: Readonly<Record<string, string>> } => {
|
|
164
|
-
if (
|
|
195
|
+
const featuresOf = (morpheme: Morpheme, passive: boolean): { features?: Readonly<Record<string, string>> } => {
|
|
196
|
+
if (passive) return { features: { Voice: "Pass" } };
|
|
165
197
|
if (isDependentNoun(morpheme)) return { features: { NounType: "Dependent" } };
|
|
166
198
|
if (isNumeral(morpheme)) return { features: { NumType: "Card" } };
|
|
167
|
-
const
|
|
168
|
-
if (
|
|
199
|
+
const name = placeType(morpheme) ?? personOrOrganisation(morpheme);
|
|
200
|
+
if (name !== undefined) return { features: { NameType: name } };
|
|
169
201
|
if (isCounter(morpheme)) return { features: { NounType: "Class" } };
|
|
170
202
|
return {};
|
|
171
203
|
};
|
|
@@ -198,13 +230,37 @@ export const predicateOnly = (tokens: readonly Token[]): Token[] =>
|
|
|
198
230
|
return { span: token.span, surface: token.surface, pos: token.pos, ...(token.lemma === undefined ? {} : { lemma: token.lemma }) };
|
|
199
231
|
});
|
|
200
232
|
|
|
233
|
+
const inflectionOf = (morpheme: Morpheme, start: number): Inflection => ({
|
|
234
|
+
surface: morpheme.surface_form,
|
|
235
|
+
pos: morpheme.pos,
|
|
236
|
+
detail: morpheme.pos_detail_1,
|
|
237
|
+
form: morpheme.conjugated_form ?? "*",
|
|
238
|
+
conjugation: morpheme.conjugated_type ?? "*",
|
|
239
|
+
start,
|
|
240
|
+
});
|
|
241
|
+
|
|
201
242
|
export const tokenize = (text: string): Token[] | undefined => {
|
|
202
243
|
const tokenizer = state.ready;
|
|
203
244
|
if (tokenizer === undefined) return undefined;
|
|
204
|
-
|
|
245
|
+
const read = readAll(tokenizer, text);
|
|
246
|
+
const sequence = read.map(({ morpheme }) => morpheme);
|
|
247
|
+
const inflections = read.map(({ morpheme, start }) => inflectionOf(morpheme, start));
|
|
248
|
+
return read.map(({ morpheme, start }, index) =>
|
|
249
|
+
toToken(morpheme, start, readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index), isEchoAt(inflections, index)),
|
|
250
|
+
);
|
|
205
251
|
};
|
|
206
252
|
|
|
207
|
-
|
|
253
|
+
/** 句読点の無い並びをこれより長く渡さない。解析器は並びの長さの二乗で遅くなり、20 万字の 1 行では何分も返らない。 */
|
|
254
|
+
const PIECE_LIMIT = 1000;
|
|
255
|
+
|
|
256
|
+
const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] =>
|
|
257
|
+
readCounterTsu(analyserPieces(text, PIECE_LIMIT).flatMap((piece) => toArray(callMethod(tokenizer, "tokenize", [piece])).flatMap(toMorpheme)));
|
|
258
|
+
|
|
259
|
+
/** 解析器は片割れのサロゲートで例外を投げる。数量の後ろを数文字だけ読み直すと、絵文字を半分に切ることがある。 */
|
|
260
|
+
const readAll = (tokenizer: Tokenizer, text: string): { readonly morpheme: Morpheme; readonly start: number }[] => {
|
|
261
|
+
const readable = wellFormed(text);
|
|
262
|
+
return placed(readable, analyse(tokenizer, readable));
|
|
263
|
+
};
|
|
208
264
|
|
|
209
265
|
/** 形態素と、本文の中での始まり。本文に見つからないものは落とす。 */
|
|
210
266
|
const placed = (text: string, raws: readonly Morpheme[]): { readonly morpheme: Morpheme; readonly start: number }[] => {
|
|
@@ -235,7 +291,7 @@ export type Morph = {
|
|
|
235
291
|
export const morphemes = (text: string): Morph[] | undefined => {
|
|
236
292
|
const tokenizer = state.ready;
|
|
237
293
|
if (tokenizer === undefined) return undefined;
|
|
238
|
-
return
|
|
294
|
+
return readAll(tokenizer, text).map(({ morpheme: raw, start }) => ({
|
|
239
295
|
start,
|
|
240
296
|
end: start + raw.surface_form.length,
|
|
241
297
|
surface: raw.surface_form,
|
|
@@ -250,6 +306,15 @@ const isCounterMorph = (morph: Morph): boolean => morph.pos === "名詞" && morp
|
|
|
250
306
|
/** 桁の語(万・億)。「26.7 万行」の「万行」は、詰めれば 26.7万 と 行 に分かれる。 */
|
|
251
307
|
const isNumeralMorph = (morph: Morph): boolean => morph.pos === "名詞" && morph.detail1 === "数";
|
|
252
308
|
|
|
309
|
+
/** 一語の副詞と読むか(がんがん)。前後の語に引かれない、その文字列だけの読み。 */
|
|
310
|
+
export const readsAsOneAdverb = (text: string): boolean => {
|
|
311
|
+
const read = tokenize(text);
|
|
312
|
+
return read?.length === 1 && read[0]?.pos === "ADV";
|
|
313
|
+
};
|
|
314
|
+
|
|
315
|
+
/** 一語と読むか(家々・時々)。品詞は問わない。 */
|
|
316
|
+
export const readsAsOneWord = (text: string): boolean => tokenize(text)?.length === 1;
|
|
317
|
+
|
|
253
318
|
/**
|
|
254
319
|
* 数のすぐ後ろに詰めて書いたとき、word が数につく語として読まれるか(spaced-counter.ts が使う)。
|
|
255
320
|
* word がそのまま一語の助数詞になるか、桁の語で始まるとき。「件名」は詰めると 件 と 名 に割れるので、助数詞とは読まない。
|