@chaffjs/lang-ja 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyser-pieces.d.ts +6 -0
- package/dist/analyser-pieces.d.ts.map +1 -0
- package/dist/analyser-pieces.js +37 -0
- package/dist/analyser-pieces.js.map +1 -0
- package/dist/counter-tsu.d.ts +1 -0
- package/dist/counter-tsu.d.ts.map +1 -1
- package/dist/counter-tsu.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/particle-after.d.ts +7 -0
- package/dist/particle-after.d.ts.map +1 -0
- package/dist/particle-after.js +9 -0
- package/dist/particle-after.js.map +1 -0
- package/dist/passive-reading.d.ts +24 -0
- package/dist/passive-reading.d.ts.map +1 -0
- package/dist/passive-reading.js +90 -0
- package/dist/passive-reading.js.map +1 -0
- package/dist/pos.d.ts +2 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +56 -19
- package/dist/pos.js.map +1 -1
- package/dist/reduplication.d.ts +3 -1
- package/dist/reduplication.d.ts.map +1 -1
- package/dist/reduplication.js +16 -1
- package/dist/reduplication.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +5 -1
- package/dist/structure.js.map +1 -1
- package/dist/tokens-within.d.ts +7 -0
- package/dist/tokens-within.d.ts.map +1 -0
- package/dist/tokens-within.js +29 -0
- package/dist/tokens-within.js.map +1 -0
- package/dist/well-formed.d.ts +3 -0
- package/dist/well-formed.d.ts.map +1 -0
- package/dist/well-formed.js +3 -0
- package/dist/well-formed.js.map +1 -0
- package/lexicons/common-acronym.yaml +4 -0
- package/lexicons/date-time-unit.yaml +11 -0
- package/lexicons/distributive-noun.yaml +2 -1
- package/lexicons/doubled-word.yaml +5 -0
- package/lexicons/emphasis-word.yaml +1 -0
- package/lexicons/honorific-formula.yaml +10 -0
- package/lexicons/honorific.yaml +15 -0
- package/lexicons/http-method.yaml +14 -0
- package/lexicons/intransitive-verb.yaml +18 -0
- package/lexicons/naming-verb.yaml +9 -0
- package/lexicons/quantity-noun.yaml +42 -0
- package/lexicons/spontaneous-verb.yaml +18 -0
- package/lexicons/stative-passive-verb.yaml +8 -0
- package/package.json +1 -1
- package/src/analyser-pieces.ts +38 -0
- package/src/counter-tsu.ts +1 -0
- package/src/index.ts +3 -5
- package/src/particle-after.ts +11 -0
- package/src/passive-reading.ts +125 -0
- package/src/pos.ts +59 -19
- package/src/reduplication.ts +24 -4
- package/src/structure.ts +6 -1
- package/src/tokens-within.ts +27 -0
- package/src/well-formed.ts +2 -0
package/src/pos.ts
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
import { surfaceStarts } from "./surface-starts.ts";
|
|
2
|
+
import { wellFormed } from "./well-formed.ts";
|
|
3
|
+
import { analyserPieces } from "./analyser-pieces.ts";
|
|
2
4
|
import { readCounterTsu, type Morpheme } from "./counter-tsu.ts";
|
|
5
|
+
import { isPassiveForm, passiveVocabulary, readsAsPassive } from "./passive-reading.ts";
|
|
6
|
+
import { loadLexicons } from "./lexicons.ts";
|
|
3
7
|
import { createRequire } from "node:module";
|
|
4
8
|
import { dirname, join } from "node:path";
|
|
5
9
|
import type { Token } from "chaffjs/plugin";
|
|
@@ -22,20 +26,31 @@ const isCallable = (value: unknown): value is (...args: readonly unknown[]) => u
|
|
|
22
26
|
|
|
23
27
|
const isTokenizer = (value: unknown): value is Tokenizer => isRecord(value) && isCallable(value["tokenize"]);
|
|
24
28
|
|
|
25
|
-
/** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ
|
|
29
|
+
/** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ *。三段目(人名の姓・名)は無ければ持たない。 */
|
|
26
30
|
const toMorpheme = (value: unknown): Morpheme[] => {
|
|
27
31
|
if (!isRecord(value)) return [];
|
|
28
|
-
const [surface, pos, detail1, detail2, basic, reading] = [
|
|
32
|
+
const [surface, pos, detail1, detail2, detail3, basic, reading] = [
|
|
29
33
|
value["surface_form"],
|
|
30
34
|
value["pos"],
|
|
31
35
|
value["pos_detail_1"],
|
|
32
36
|
value["pos_detail_2"],
|
|
37
|
+
value["pos_detail_3"],
|
|
33
38
|
value["basic_form"],
|
|
34
39
|
value["reading"],
|
|
35
40
|
];
|
|
36
41
|
if (typeof surface !== "string" || typeof pos !== "string" || typeof detail1 !== "string" || typeof basic !== "string") return [];
|
|
37
42
|
const detail = typeof detail2 === "string" ? detail2 : "*";
|
|
38
|
-
return [
|
|
43
|
+
return [
|
|
44
|
+
{
|
|
45
|
+
surface_form: surface,
|
|
46
|
+
pos,
|
|
47
|
+
pos_detail_1: detail1,
|
|
48
|
+
pos_detail_2: detail,
|
|
49
|
+
...(typeof detail3 === "string" ? { pos_detail_3: detail3 } : {}),
|
|
50
|
+
basic_form: basic,
|
|
51
|
+
...(typeof reading === "string" ? { reading } : {}),
|
|
52
|
+
},
|
|
53
|
+
];
|
|
39
54
|
};
|
|
40
55
|
|
|
41
56
|
const dictionaryPath = (): string => join(dirname(require.resolve("@sglkc/kuromoji/package.json")), "dict");
|
|
@@ -89,13 +104,10 @@ const BY_POS: Readonly<Record<string, string>> = {
|
|
|
89
104
|
export const upos = (pos: string, detail: string): string => BY_DETAIL[detail] ?? BY_POS[pos] ?? "X";
|
|
90
105
|
|
|
91
106
|
/**
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
* 受動と言い切らず「受動の形」として印を付け、どう扱うかは rule 側に委ねる。
|
|
107
|
+
* 受動の「れる/られる」。この語は可能・尊敬・自発も表す。形のうえで受動でないと言えるもの
|
|
108
|
+
* (自発や関係を表す動詞、名付けの「と呼ばれる」、尊敬の形と決まり文句)は passive-reading.ts が外す。残りは「受動の形」として印を付ける。
|
|
95
109
|
*/
|
|
96
|
-
const
|
|
97
|
-
|
|
98
|
-
const isPassive = (morpheme: Morpheme): boolean => morpheme.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && PASSIVE_LEMMA.has(morpheme.basic_form);
|
|
110
|
+
const PASSIVE_VOCABULARY = passiveVocabulary(loadLexicons());
|
|
99
111
|
|
|
100
112
|
/**
|
|
101
113
|
* 非自立名詞(の・こと・もの・ため・はず)。品詞は名詞だが、単独では何も指さない。
|
|
@@ -128,14 +140,14 @@ export const isReady = (): boolean => state.ready !== undefined;
|
|
|
128
140
|
* span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
|
|
129
141
|
* 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
|
|
130
142
|
*/
|
|
131
|
-
const toToken = (morpheme: Morpheme, start: number): Token => ({
|
|
143
|
+
const toToken = (morpheme: Morpheme, start: number, passive: boolean): Token => ({
|
|
132
144
|
span: { start, end: start + morpheme.surface_form.length },
|
|
133
145
|
surface: morpheme.surface_form,
|
|
134
146
|
// UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
|
|
135
|
-
pos:
|
|
147
|
+
pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
|
|
136
148
|
...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
|
|
137
149
|
...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
|
|
138
|
-
...featuresOf(morpheme),
|
|
150
|
+
...featuresOf(morpheme, passive),
|
|
139
151
|
});
|
|
140
152
|
|
|
141
153
|
/**
|
|
@@ -157,15 +169,25 @@ const placeType = (morpheme: Morpheme): string | undefined => {
|
|
|
157
169
|
return morpheme.pos_detail_1 === "接尾" ? "GeoUnit" : undefined;
|
|
158
170
|
};
|
|
159
171
|
|
|
172
|
+
/** 人名(IPADIC の「固有名詞,人名」)の三段目。姓は Sur、名は Giv、どちらとも言えない人名は Prs(UD の NameType)。 */
|
|
173
|
+
const PERSON: Readonly<Record<string, string>> = { 姓: "Sur", 名: "Giv", 一般: "Prs" };
|
|
174
|
+
|
|
175
|
+
/** 人名と組織名(「固有名詞,組織」は UD の NameType=Com)。地名は placeType が見る。 */
|
|
176
|
+
const personOrOrganisation = (morpheme: Morpheme): string | undefined => {
|
|
177
|
+
if (morpheme.pos !== "名詞" || morpheme.pos_detail_1 !== "固有名詞") return undefined;
|
|
178
|
+
if (morpheme.pos_detail_2 === "人名") return PERSON[morpheme.pos_detail_3 ?? ""];
|
|
179
|
+
return morpheme.pos_detail_2 === "組織" ? "Com" : undefined;
|
|
180
|
+
};
|
|
181
|
+
|
|
160
182
|
/** 数を数える単位(IPADIC の「接尾,助数詞」: 丁目・件・人)。UD では NounType=Class。 */
|
|
161
183
|
const isCounter = (morpheme: Morpheme): boolean => morpheme.pos === "名詞" && morpheme.pos_detail_1 === "接尾" && morpheme.pos_detail_2 === "助数詞";
|
|
162
184
|
|
|
163
|
-
const featuresOf = (morpheme: Morpheme): { features?: Readonly<Record<string, string>> } => {
|
|
164
|
-
if (
|
|
185
|
+
const featuresOf = (morpheme: Morpheme, passive: boolean): { features?: Readonly<Record<string, string>> } => {
|
|
186
|
+
if (passive) return { features: { Voice: "Pass" } };
|
|
165
187
|
if (isDependentNoun(morpheme)) return { features: { NounType: "Dependent" } };
|
|
166
188
|
if (isNumeral(morpheme)) return { features: { NumType: "Card" } };
|
|
167
|
-
const
|
|
168
|
-
if (
|
|
189
|
+
const name = placeType(morpheme) ?? personOrOrganisation(morpheme);
|
|
190
|
+
if (name !== undefined) return { features: { NameType: name } };
|
|
169
191
|
if (isCounter(morpheme)) return { features: { NounType: "Class" } };
|
|
170
192
|
return {};
|
|
171
193
|
};
|
|
@@ -201,10 +223,22 @@ export const predicateOnly = (tokens: readonly Token[]): Token[] =>
|
|
|
201
223
|
export const tokenize = (text: string): Token[] | undefined => {
|
|
202
224
|
const tokenizer = state.ready;
|
|
203
225
|
if (tokenizer === undefined) return undefined;
|
|
204
|
-
|
|
226
|
+
const read = readAll(tokenizer, text);
|
|
227
|
+
const sequence = read.map(({ morpheme }) => morpheme);
|
|
228
|
+
return read.map(({ morpheme, start }, index) => toToken(morpheme, start, readsAsPassive(sequence, index, PASSIVE_VOCABULARY)));
|
|
205
229
|
};
|
|
206
230
|
|
|
207
|
-
|
|
231
|
+
/** 句読点の無い並びをこれより長く渡さない。解析器は並びの長さの二乗で遅くなり、20 万字の 1 行では何分も返らない。 */
|
|
232
|
+
const PIECE_LIMIT = 1000;
|
|
233
|
+
|
|
234
|
+
const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] =>
|
|
235
|
+
readCounterTsu(analyserPieces(text, PIECE_LIMIT).flatMap((piece) => toArray(callMethod(tokenizer, "tokenize", [piece])).flatMap(toMorpheme)));
|
|
236
|
+
|
|
237
|
+
/** 解析器は片割れのサロゲートで例外を投げる。数量の後ろを数文字だけ読み直すと、絵文字を半分に切ることがある。 */
|
|
238
|
+
const readAll = (tokenizer: Tokenizer, text: string): { readonly morpheme: Morpheme; readonly start: number }[] => {
|
|
239
|
+
const readable = wellFormed(text);
|
|
240
|
+
return placed(readable, analyse(tokenizer, readable));
|
|
241
|
+
};
|
|
208
242
|
|
|
209
243
|
/** 形態素と、本文の中での始まり。本文に見つからないものは落とす。 */
|
|
210
244
|
const placed = (text: string, raws: readonly Morpheme[]): { readonly morpheme: Morpheme; readonly start: number }[] => {
|
|
@@ -235,7 +269,7 @@ export type Morph = {
|
|
|
235
269
|
export const morphemes = (text: string): Morph[] | undefined => {
|
|
236
270
|
const tokenizer = state.ready;
|
|
237
271
|
if (tokenizer === undefined) return undefined;
|
|
238
|
-
return
|
|
272
|
+
return readAll(tokenizer, text).map(({ morpheme: raw, start }) => ({
|
|
239
273
|
start,
|
|
240
274
|
end: start + raw.surface_form.length,
|
|
241
275
|
surface: raw.surface_form,
|
|
@@ -250,6 +284,12 @@ const isCounterMorph = (morph: Morph): boolean => morph.pos === "名詞" && morp
|
|
|
250
284
|
/** 桁の語(万・億)。「26.7 万行」の「万行」は、詰めれば 26.7万 と 行 に分かれる。 */
|
|
251
285
|
const isNumeralMorph = (morph: Morph): boolean => morph.pos === "名詞" && morph.detail1 === "数";
|
|
252
286
|
|
|
287
|
+
/** 一語の副詞と読むか(がんがん)。前後の語に引かれない、その文字列だけの読み。 */
|
|
288
|
+
export const readsAsOneAdverb = (text: string): boolean => {
|
|
289
|
+
const read = tokenize(text);
|
|
290
|
+
return read?.length === 1 && read[0]?.pos === "ADV";
|
|
291
|
+
};
|
|
292
|
+
|
|
253
293
|
/**
|
|
254
294
|
* 数のすぐ後ろに詰めて書いたとき、word が数につく語として読まれるか(spaced-counter.ts が使う)。
|
|
255
295
|
* word がそのまま一語の助数詞になるか、桁の語で始まるとき。「件名」は詰めると 件 と 名 に割れるので、助数詞とは読まない。
|
package/src/reduplication.ts
CHANGED
|
@@ -26,7 +26,27 @@ const isParticleAfter = (next: Token | undefined, second: Token, particles: Read
|
|
|
26
26
|
|
|
27
27
|
const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
const isDistributive = (tokens: readonly Token[], index: number, vocabulary: Distributive): boolean => {
|
|
30
|
+
const token = tokens[index];
|
|
31
|
+
return token !== undefined && isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles);
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** 文字列を解析器が一語の副詞と読むか。解析器を持つ index.ts が渡す。 */
|
|
35
|
+
export type ReadsAsAdverb = (text: string) => boolean;
|
|
36
|
+
|
|
37
|
+
const CONTENT_WORD = new Set(["NOUN", "VERB", "ADJ"]);
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 重ね言葉の副詞(がんがん)は、前の語に引かれると同じ名詞二つに切られる(労働者ががんがん → が・がん・がん)。
|
|
41
|
+
* 二語を続けて読み直して一語の副詞なら、重ね言葉。機能語の重なりは読み直さない。書き損じの「行ったたので」の「たた」も一語の副詞に読める。
|
|
42
|
+
*/
|
|
43
|
+
const isSplitAdverb = (first: Token | undefined, second: Token, readsAsAdverb: ReadsAsAdverb): boolean =>
|
|
44
|
+
first !== undefined &&
|
|
45
|
+
first.surface === second.surface &&
|
|
46
|
+
first.pos === second.pos &&
|
|
47
|
+
CONTENT_WORD.has(second.pos) &&
|
|
48
|
+
first.span.end === second.span.start &&
|
|
49
|
+
readsAsAdverb(`${first.surface}${second.surface}`);
|
|
50
|
+
|
|
51
|
+
export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive, readsAsAdverb: ReadsAsAdverb): Token[] =>
|
|
52
|
+
tokens.map((token, index) => (isDistributive(tokens, index, vocabulary) || isSplitAdverb(tokens[index - 1], token, readsAsAdverb) ? echoed(token) : token));
|
package/src/structure.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { citationVocabulary, citedDocument } from "./citation.ts";
|
|
|
4
4
|
import { loadLexicons } from "./lexicons.ts";
|
|
5
5
|
import { countedAfter, dates, quantities } from "./quantities.ts";
|
|
6
6
|
import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
|
|
7
|
+
import { startsWithParticle } from "./particle-after.ts";
|
|
7
8
|
|
|
8
9
|
// 契約書・規程・法令の番号の書き方。core は番号の書き方を知らず、ここで読んだものを入れ子にする。
|
|
9
10
|
|
|
@@ -202,6 +203,9 @@ const definitions = (text: string): Mention[] =>
|
|
|
202
203
|
}),
|
|
203
204
|
);
|
|
204
205
|
|
|
206
|
+
/** 条を数える語。見出しに「1 目的」と番号を振っただけの文書には、この語で数えた条が無い。 */
|
|
207
|
+
const ARTICLE_UNIT = "条";
|
|
208
|
+
|
|
205
209
|
const REFERENCE = new RegExp(`第(?<a>${NUMBER})条(?:の(?<s>${NUMBER}))?(?:第(?<p>${NUMBER})項)?(?:第(?<i>${NUMBER})号)?`, "gu");
|
|
206
210
|
|
|
207
211
|
/**
|
|
@@ -278,7 +282,7 @@ const articlesOf = (text: string): Mention[] =>
|
|
|
278
282
|
const address = addressOfReference(match.groups ?? {});
|
|
279
283
|
if (address === undefined) return [];
|
|
280
284
|
const { target, fallback } = address;
|
|
281
|
-
const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }) };
|
|
285
|
+
const attrs = { target, label: match[0], unitWord: ARTICLE_UNIT, ...(fallback === undefined ? {} : { fallback }) };
|
|
282
286
|
return [{ start: match.index, end: match.index + match[0].length, attrs }];
|
|
283
287
|
});
|
|
284
288
|
|
|
@@ -360,5 +364,6 @@ export const structure: StructurePatterns = {
|
|
|
360
364
|
quantities: quantitiesOutsideSections,
|
|
361
365
|
dates,
|
|
362
366
|
countedAfter: countedAfterNumber,
|
|
367
|
+
continuesSentence: startsWithParticle,
|
|
363
368
|
number,
|
|
364
369
|
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Span, Token } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
/** start が from 以上になる最初の位置。tokens は start の昇順。 */
|
|
4
|
+
const firstFrom = (tokens: readonly Token[], from: number): number => {
|
|
5
|
+
let low = 0;
|
|
6
|
+
let high = tokens.length;
|
|
7
|
+
while (low < high) {
|
|
8
|
+
const middle = (low + high) >> 1;
|
|
9
|
+
if ((tokens[middle]?.span.start ?? Number.POSITIVE_INFINITY) < from) low = middle + 1;
|
|
10
|
+
else high = middle;
|
|
11
|
+
}
|
|
12
|
+
return low;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* span に収まる token。tokens は start の昇順(解析器が本文の頭から出した順)。
|
|
17
|
+
* 文ごとに全 token をなめると、空行の無い何千行もの .txt で文の数 × token の数になる。
|
|
18
|
+
*/
|
|
19
|
+
export const tokensWithin = (tokens: readonly Token[], span: Span): Token[] => {
|
|
20
|
+
const found: Token[] = [];
|
|
21
|
+
for (let at = firstFrom(tokens, span.start); at < tokens.length; at += 1) {
|
|
22
|
+
const token = tokens[at];
|
|
23
|
+
if (token === undefined || token.span.start > span.end) break;
|
|
24
|
+
if (token.span.end <= span.end) found.push(token);
|
|
25
|
+
}
|
|
26
|
+
return found;
|
|
27
|
+
};
|