@chaffjs/lang-ja 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analyser-pieces.d.ts +6 -0
- package/dist/analyser-pieces.d.ts.map +1 -0
- package/dist/analyser-pieces.js +37 -0
- package/dist/analyser-pieces.js.map +1 -0
- package/dist/counter-tsu.d.ts +18 -0
- package/dist/counter-tsu.d.ts.map +1 -0
- package/dist/counter-tsu.js +29 -0
- package/dist/counter-tsu.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -2
- package/dist/index.js.map +1 -1
- package/dist/particle-after.d.ts +7 -0
- package/dist/particle-after.d.ts.map +1 -0
- package/dist/particle-after.js +9 -0
- package/dist/particle-after.js.map +1 -0
- package/dist/passive-reading.d.ts +24 -0
- package/dist/passive-reading.d.ts.map +1 -0
- package/dist/passive-reading.js +90 -0
- package/dist/passive-reading.js.map +1 -0
- package/dist/pos.d.ts +2 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +75 -29
- package/dist/pos.js.map +1 -1
- package/dist/quantities.d.ts +7 -1
- package/dist/quantities.d.ts.map +1 -1
- package/dist/quantities.js +47 -9
- package/dist/quantities.js.map +1 -1
- package/dist/reduplication.d.ts +3 -1
- package/dist/reduplication.d.ts.map +1 -1
- package/dist/reduplication.js +16 -1
- package/dist/reduplication.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +9 -2
- package/dist/structure.js.map +1 -1
- package/dist/tokens-within.d.ts +7 -0
- package/dist/tokens-within.d.ts.map +1 -0
- package/dist/tokens-within.js +29 -0
- package/dist/tokens-within.js.map +1 -0
- package/dist/well-formed.d.ts +3 -0
- package/dist/well-formed.d.ts.map +1 -0
- package/dist/well-formed.js +3 -0
- package/dist/well-formed.js.map +1 -0
- package/lexicons/common-acronym.yaml +4 -0
- package/lexicons/date-time-unit.yaml +11 -0
- package/lexicons/definition-verb.yaml +5 -1
- package/lexicons/distributive-noun.yaml +2 -1
- package/lexicons/doubled-word.yaml +5 -0
- package/lexicons/emphasis-word.yaml +1 -0
- package/lexicons/honorific-formula.yaml +10 -0
- package/lexicons/honorific.yaml +15 -0
- package/lexicons/http-method.yaml +14 -0
- package/lexicons/intransitive-verb.yaml +18 -0
- package/lexicons/naming-verb.yaml +9 -0
- package/lexicons/not-magnitude.yaml +12 -0
- package/lexicons/numbered-division.yaml +24 -0
- package/lexicons/quantity-noun.yaml +42 -0
- package/lexicons/spontaneous-verb.yaml +18 -0
- package/lexicons/stative-passive-verb.yaml +8 -0
- package/package.json +1 -1
- package/src/analyser-pieces.ts +38 -0
- package/src/counter-tsu.ts +60 -0
- package/src/index.ts +3 -5
- package/src/particle-after.ts +11 -0
- package/src/passive-reading.ts +125 -0
- package/src/pos.ts +77 -39
- package/src/quantities.ts +62 -10
- package/src/reduplication.ts +24 -4
- package/src/structure.ts +11 -2
- package/src/tokens-within.ts +27 -0
- package/src/well-formed.ts +2 -0
package/src/quantities.ts
CHANGED
|
@@ -29,6 +29,7 @@ const UNITS = [
|
|
|
29
29
|
"名",
|
|
30
30
|
"倍",
|
|
31
31
|
"個",
|
|
32
|
+
"つ",
|
|
32
33
|
"%",
|
|
33
34
|
"%",
|
|
34
35
|
];
|
|
@@ -64,6 +65,18 @@ const runEnd = (morphs: readonly Morph[], index: number): number => {
|
|
|
64
65
|
/** 「第3条」の 3 は番号で、数量ではない。 */
|
|
65
66
|
const isOrdinal = (morph: Morph | undefined): boolean => morph?.surface === "第";
|
|
66
67
|
|
|
68
|
+
/** 助数詞の後ろの「目」「め」(3つ目、2回目、1年目、一つめ)も順番で、数量ではない。「目標」は一語なので当たらない。 */
|
|
69
|
+
const ORDINAL_SUFFIXES: ReadonlySet<string> = new Set(["目", "め"]);
|
|
70
|
+
|
|
71
|
+
const isOrdinalSuffix = (morph: Morph | undefined): boolean => morph?.pos === "名詞" && ORDINAL_SUFFIXES.has(morph.surface);
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* 解析器の無いときに順番と読む単位の後ろ。漢字の続かない「目」(「2回目標」「5人目線」の目は次の語の頭)と、
|
|
75
|
+
* 「めど」でない「め」(「10日めどに」は期限)。「つ」の後ろの漢字は見ない。「3つ選ぶ」のように数えた後ろに漢字が来るほうが、
|
|
76
|
+
* 「三つ巴」のような一語より多い。
|
|
77
|
+
*/
|
|
78
|
+
const ORDINAL_BY_TABLE = /^(?:目(?!\p{Script=Han})|め(?!ど))/u;
|
|
79
|
+
|
|
67
80
|
/** 数と単位のあいだに置かれうる空白 1 文字。構造の型(structure.ts の SPACE)と同じく、全角空白とタブも含む。 */
|
|
68
81
|
const GAP = new Set([" ", "\t", "\u3000"]);
|
|
69
82
|
|
|
@@ -71,23 +84,32 @@ const isSpace = (morph: Morph | undefined): morph is Morph => morph !== undefine
|
|
|
71
84
|
|
|
72
85
|
const SPACES = /[ \t\u3000]/gu;
|
|
73
86
|
|
|
87
|
+
/** 数量の後ろを読むのに要る文字数。「万人が」「億円の」まで見えれば足りる。 */
|
|
88
|
+
const LOOKAHEAD = 8;
|
|
89
|
+
|
|
74
90
|
/**
|
|
75
91
|
* 「1.5 倍」の「倍」は、数とのあいだに空白があると解析器が普通の名詞と読む。
|
|
76
92
|
* 空白を詰めて読み直し、数の直後に来る語が助数詞ならその語を返す。
|
|
77
93
|
*/
|
|
94
|
+
const rereadAfterSpace = (number: string, rest: string): Morph[] => morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
|
|
95
|
+
|
|
78
96
|
const unitAfterSpace = (number: string, rest: string): string | undefined => {
|
|
79
|
-
const next =
|
|
97
|
+
const next = rereadAfterSpace(number, rest).find((morph) => morph.start >= number.length);
|
|
80
98
|
return next !== undefined && isCounter(next) ? next.surface : undefined;
|
|
81
99
|
};
|
|
82
100
|
|
|
83
|
-
|
|
101
|
+
/** ordinal: 単位のすぐ後ろが順番の「目」「め」。空白を詰めて読み直したときは、読み直した語で見る(「3 つめ」の「つめ」は詰めると つ + め)。 */
|
|
102
|
+
type Unit = { readonly unit: string; readonly end: number; readonly ordinal: boolean };
|
|
84
103
|
|
|
85
104
|
const counterAt = (text: string, morphs: readonly Morph[], number: string, end: number): Unit | undefined => {
|
|
86
105
|
const next = morphs[end];
|
|
87
|
-
if (next !== undefined && isCounter(next)) return { unit: next.surface, end: next.end };
|
|
106
|
+
if (next !== undefined && isCounter(next)) return { unit: next.surface, end: next.end, ordinal: isOrdinalSuffix(morphs[end + 1]) };
|
|
88
107
|
if (!isSpace(next)) return undefined;
|
|
89
|
-
const
|
|
90
|
-
|
|
108
|
+
const rest = text.slice(next.end);
|
|
109
|
+
const unit = unitAfterSpace(number, rest);
|
|
110
|
+
if (unit === undefined) return undefined;
|
|
111
|
+
const ordinal = isOrdinalSuffix(rereadAfterSpace(number, rest).find((morph) => morph.start === number.length + unit.length));
|
|
112
|
+
return { unit, end: next.end + unit.length, ordinal };
|
|
91
113
|
};
|
|
92
114
|
|
|
93
115
|
const countedByMorphemes = (text: string, morphs: readonly Morph[]): Counted[] => {
|
|
@@ -100,7 +122,7 @@ const countedByMorphemes = (text: string, morphs: readonly Morph[]): Counted[] =
|
|
|
100
122
|
const number = text.slice(first.start, morphs[end - 1]?.end ?? first.end).replace(SPACES, "");
|
|
101
123
|
const value = parseJapaneseNumber(number);
|
|
102
124
|
const counter = counterAt(text, morphs, number, end);
|
|
103
|
-
if (value !== undefined && counter !== undefined) found.push({ start: first.start, end: counter.end, value, unit: counter.unit });
|
|
125
|
+
if (value !== undefined && counter !== undefined && !counter.ordinal) found.push({ start: first.start, end: counter.end, value, unit: counter.unit });
|
|
104
126
|
}
|
|
105
127
|
index = Math.max(end, index + 1);
|
|
106
128
|
}
|
|
@@ -116,7 +138,7 @@ export const countedByTable = (text: string): Counted[] =>
|
|
|
116
138
|
const after = match.index + match[0].length + (GAP.has(text[match.index + match[0].length] ?? "") ? 1 : 0);
|
|
117
139
|
const unit = UNITS.find((candidate) => text.startsWith(candidate, after));
|
|
118
140
|
const value = parseJapaneseNumber(toHalfWidth(match[0].replace(SPACES, "")));
|
|
119
|
-
const ordinal = text[match.index - 1] === "第";
|
|
141
|
+
const ordinal = text[match.index - 1] === "第" || (unit !== undefined && ORDINAL_BY_TABLE.test(text.slice(after + unit.length)));
|
|
120
142
|
return unit === undefined || value === undefined || ordinal ? [] : [{ start: match.index, end: after + unit.length, value, unit }];
|
|
121
143
|
});
|
|
122
144
|
|
|
@@ -227,11 +249,41 @@ export const dates = (text: string): Mention[] =>
|
|
|
227
249
|
return weekday === undefined ? date : { ...date, attrs: { ...date.attrs, weekday } };
|
|
228
250
|
});
|
|
229
251
|
|
|
252
|
+
/** 数がそこで閉じる。助数詞(「万人」)か、名詞でない語(「万を」「万。」)か、行の終わり。 */
|
|
253
|
+
const closesNumber = (morph: Morph | undefined): boolean => morph === undefined || isCounter(morph) || morph.pos !== "名詞";
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* 桁の語が一語として続く。「1.5 万を超える」のように単位が無くても数の一部。
|
|
257
|
+
* 「万葉集」は一語で、「万葉の」「万一」は桁の語の後ろに名詞や数が続くので、どれも数の続きではない。
|
|
258
|
+
*/
|
|
259
|
+
export const continuesWithMultiplier = (morphs: readonly Morph[], index: number): boolean =>
|
|
260
|
+
MULTIPLIERS.has(morphs[index]?.surface ?? "") && isNumeral(morphs[index]) && closesNumber(morphs[index + 1]);
|
|
261
|
+
|
|
262
|
+
const multiplierAfter = (number: string, rest: string): boolean => {
|
|
263
|
+
const morphs = morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
|
|
264
|
+
return continuesWithMultiplier(
|
|
265
|
+
morphs,
|
|
266
|
+
morphs.findIndex((morph) => morph.start >= number.length),
|
|
267
|
+
);
|
|
268
|
+
};
|
|
269
|
+
|
|
270
|
+
const startsWithUnit = (text: string): boolean => UNITS.some((unit) => text.startsWith(unit));
|
|
271
|
+
|
|
272
|
+
/** 解析器の無いときの、数がそこで閉じる字。行の終わり、表の単位、仮名の助詞や句読点(「万を」「万。」)。漢字(「万葉」「万全」)は閉じない。 */
|
|
273
|
+
const CLOSES_BY_TABLE = /^(?:$|[\p{Script=Hiragana}\p{P}\s])/u;
|
|
274
|
+
|
|
275
|
+
/** 解析器の無いときの読み方。桁の語が先にあれば、その後ろで数が閉じるかを見る。 */
|
|
276
|
+
const countedByTableAfter = (rest: string): boolean => {
|
|
277
|
+
if (!MULTIPLIERS.has(rest[0] ?? "")) return startsWithUnit(rest);
|
|
278
|
+
const after = rest.slice(1);
|
|
279
|
+
return startsWithUnit(after) || CLOSES_BY_TABLE.test(after);
|
|
280
|
+
};
|
|
281
|
+
|
|
230
282
|
/**
|
|
231
|
-
* 「1.5
|
|
283
|
+
* 「1.5 倍になった。」「1.5 万人が参加した。」の 1.5 は章番号ではない。番号と続く語のあいだの空白を詰めて読み直し、
|
|
232
284
|
* 続く語が助数詞なら数量の一部と分かる。空白のあるままだと、解析器は「倍」を普通の名詞と読む。
|
|
233
285
|
*/
|
|
234
286
|
export const countedAfter = (number: string, rest: string): boolean => {
|
|
235
|
-
if (!isReady()) return
|
|
236
|
-
return unitAfterSpace(number, rest) !== undefined;
|
|
287
|
+
if (!isReady()) return countedByTableAfter(rest);
|
|
288
|
+
return unitAfterSpace(number, rest) !== undefined || multiplierAfter(number, rest);
|
|
237
289
|
};
|
package/src/reduplication.ts
CHANGED
|
@@ -26,7 +26,27 @@ const isParticleAfter = (next: Token | undefined, second: Token, particles: Read
|
|
|
26
26
|
|
|
27
27
|
const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
const isDistributive = (tokens: readonly Token[], index: number, vocabulary: Distributive): boolean => {
|
|
30
|
+
const token = tokens[index];
|
|
31
|
+
return token !== undefined && isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles);
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** 文字列を解析器が一語の副詞と読むか。解析器を持つ index.ts が渡す。 */
|
|
35
|
+
export type ReadsAsAdverb = (text: string) => boolean;
|
|
36
|
+
|
|
37
|
+
const CONTENT_WORD = new Set(["NOUN", "VERB", "ADJ"]);
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* 重ね言葉の副詞(がんがん)は、前の語に引かれると同じ名詞二つに切られる(労働者ががんがん → が・がん・がん)。
|
|
41
|
+
* 二語を続けて読み直して一語の副詞なら、重ね言葉。機能語の重なりは読み直さない。書き損じの「行ったたので」の「たた」も一語の副詞に読める。
|
|
42
|
+
*/
|
|
43
|
+
const isSplitAdverb = (first: Token | undefined, second: Token, readsAsAdverb: ReadsAsAdverb): boolean =>
|
|
44
|
+
first !== undefined &&
|
|
45
|
+
first.surface === second.surface &&
|
|
46
|
+
first.pos === second.pos &&
|
|
47
|
+
CONTENT_WORD.has(second.pos) &&
|
|
48
|
+
first.span.end === second.span.start &&
|
|
49
|
+
readsAsAdverb(`${first.surface}${second.surface}`);
|
|
50
|
+
|
|
51
|
+
export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive, readsAsAdverb: ReadsAsAdverb): Token[] =>
|
|
52
|
+
tokens.map((token, index) => (isDistributive(tokens, index, vocabulary) || isSplitAdverb(tokens[index - 1], token, readsAsAdverb) ? echoed(token) : token));
|
package/src/structure.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { citationVocabulary, citedDocument } from "./citation.ts";
|
|
|
4
4
|
import { loadLexicons } from "./lexicons.ts";
|
|
5
5
|
import { countedAfter, dates, quantities } from "./quantities.ts";
|
|
6
6
|
import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
|
|
7
|
+
import { startsWithParticle } from "./particle-after.ts";
|
|
7
8
|
|
|
8
9
|
// 契約書・規程・法令の番号の書き方。core は番号の書き方を知らず、ここで読んだものを入れ子にする。
|
|
9
10
|
|
|
@@ -202,6 +203,9 @@ const definitions = (text: string): Mention[] =>
|
|
|
202
203
|
}),
|
|
203
204
|
);
|
|
204
205
|
|
|
206
|
+
/** 条を数える語。見出しに「1 目的」と番号を振っただけの文書には、この語で数えた条が無い。 */
|
|
207
|
+
const ARTICLE_UNIT = "条";
|
|
208
|
+
|
|
205
209
|
const REFERENCE = new RegExp(`第(?<a>${NUMBER})条(?:の(?<s>${NUMBER}))?(?:第(?<p>${NUMBER})項)?(?:第(?<i>${NUMBER})号)?`, "gu");
|
|
206
210
|
|
|
207
211
|
/**
|
|
@@ -266,6 +270,10 @@ const addressOfReference = (
|
|
|
266
270
|
const LEXICONS = loadLexicons();
|
|
267
271
|
const CITATION = citationVocabulary(LEXICONS);
|
|
268
272
|
const SECTIONS = sectionVocabulary(LEXICONS);
|
|
273
|
+
const NOT_MAGNITUDE = (LEXICONS["not-magnitude"] ?? []).map((entry) => entry.pattern);
|
|
274
|
+
|
|
275
|
+
/** 辞書が桁の語と助数詞に切る一語(「1.5 万葉の世界」)は数量の続きではないので、行頭の番号は節のまま。 */
|
|
276
|
+
const countedAfterNumber = (number: string, rest: string): boolean => !NOT_MAGNITUDE.some((word) => rest.startsWith(word)) && countedAfter(number, rest);
|
|
269
277
|
|
|
270
278
|
const sectionsOf = (text: string): Mention[] => sectionReferences(text, SECTIONS);
|
|
271
279
|
|
|
@@ -274,7 +282,7 @@ const articlesOf = (text: string): Mention[] =>
|
|
|
274
282
|
const address = addressOfReference(match.groups ?? {});
|
|
275
283
|
if (address === undefined) return [];
|
|
276
284
|
const { target, fallback } = address;
|
|
277
|
-
const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }) };
|
|
285
|
+
const attrs = { target, label: match[0], unitWord: ARTICLE_UNIT, ...(fallback === undefined ? {} : { fallback }) };
|
|
278
286
|
return [{ start: match.index, end: match.index + match[0].length, attrs }];
|
|
279
287
|
});
|
|
280
288
|
|
|
@@ -355,6 +363,7 @@ export const structure: StructurePatterns = {
|
|
|
355
363
|
obligations,
|
|
356
364
|
quantities: quantitiesOutsideSections,
|
|
357
365
|
dates,
|
|
358
|
-
countedAfter,
|
|
366
|
+
countedAfter: countedAfterNumber,
|
|
367
|
+
continuesSentence: startsWithParticle,
|
|
359
368
|
number,
|
|
360
369
|
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Span, Token } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
/** start が from 以上になる最初の位置。tokens は start の昇順。 */
|
|
4
|
+
const firstFrom = (tokens: readonly Token[], from: number): number => {
|
|
5
|
+
let low = 0;
|
|
6
|
+
let high = tokens.length;
|
|
7
|
+
while (low < high) {
|
|
8
|
+
const middle = (low + high) >> 1;
|
|
9
|
+
if ((tokens[middle]?.span.start ?? Number.POSITIVE_INFINITY) < from) low = middle + 1;
|
|
10
|
+
else high = middle;
|
|
11
|
+
}
|
|
12
|
+
return low;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* span に収まる token。tokens は start の昇順(解析器が本文の頭から出した順)。
|
|
17
|
+
* 文ごとに全 token をなめると、空行の無い何千行もの .txt で文の数 × token の数になる。
|
|
18
|
+
*/
|
|
19
|
+
export const tokensWithin = (tokens: readonly Token[], span: Span): Token[] => {
|
|
20
|
+
const found: Token[] = [];
|
|
21
|
+
for (let at = firstFrom(tokens, span.start); at < tokens.length; at += 1) {
|
|
22
|
+
const token = tokens[at];
|
|
23
|
+
if (token === undefined || token.span.start > span.end) break;
|
|
24
|
+
if (token.span.end <= span.end) found.push(token);
|
|
25
|
+
}
|
|
26
|
+
return found;
|
|
27
|
+
};
|