@chaffjs/lang-ja 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/closing-quote.d.ts +6 -0
- package/dist/closing-quote.d.ts.map +1 -0
- package/dist/closing-quote.js +50 -0
- package/dist/closing-quote.js.map +1 -0
- package/dist/english-run.d.ts +8 -0
- package/dist/english-run.d.ts.map +1 -0
- package/dist/english-run.js +24 -0
- package/dist/english-run.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +56 -21
- package/dist/index.js.map +1 -1
- package/dist/inside-brackets.d.ts +5 -0
- package/dist/inside-brackets.d.ts.map +1 -0
- package/dist/inside-brackets.js +44 -0
- package/dist/inside-brackets.js.map +1 -0
- package/dist/label-stop.d.ts +13 -0
- package/dist/label-stop.d.ts.map +1 -0
- package/dist/label-stop.js +14 -0
- package/dist/label-stop.js.map +1 -0
- package/dist/long-vowel-form.d.ts +12 -0
- package/dist/long-vowel-form.d.ts.map +1 -0
- package/dist/long-vowel-form.js +28 -0
- package/dist/long-vowel-form.js.map +1 -0
- package/dist/number-stop.d.ts +3 -0
- package/dist/number-stop.d.ts.map +1 -0
- package/dist/number-stop.js +25 -0
- package/dist/number-stop.js.map +1 -0
- package/dist/quantities.d.ts.map +1 -1
- package/dist/quantities.js +11 -1
- package/dist/quantities.js.map +1 -1
- package/dist/quoted-stop.d.ts +13 -0
- package/dist/quoted-stop.d.ts.map +1 -0
- package/dist/quoted-stop.js +45 -0
- package/dist/quoted-stop.js.map +1 -0
- package/dist/regexp.d.ts +3 -0
- package/dist/regexp.d.ts.map +1 -0
- package/dist/regexp.js +4 -0
- package/dist/regexp.js.map +1 -0
- package/dist/sentence-close.d.ts +2 -0
- package/dist/sentence-close.d.ts.map +1 -0
- package/dist/sentence-close.js +10 -0
- package/dist/sentence-close.js.map +1 -0
- package/dist/sentence-split.d.ts +2 -0
- package/dist/sentence-split.d.ts.map +1 -1
- package/dist/sentence-split.js +3 -1
- package/dist/sentence-split.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +23 -4
- package/dist/structure.js.map +1 -1
- package/lexicons/abbreviated-label.yaml +16 -0
- package/lexicons/ai-tell.yaml +69 -0
- package/lexicons/announcing-opener.yaml +11 -0
- package/lexicons/assistant-residue.yaml +37 -0
- package/lexicons/calendar-era.yaml +10 -0
- package/lexicons/closing-cliche.yaml +2 -0
- package/lexicons/contrast-frame.yaml +9 -0
- package/lexicons/contrast-lead.yaml +10 -0
- package/lexicons/contrast-turn.yaml +9 -0
- package/lexicons/count-anchor.yaml +10 -0
- package/lexicons/count-counter.yaml +17 -0
- package/lexicons/count-hedge.yaml +61 -0
- package/lexicons/count-number.yaml +16 -0
- package/lexicons/email-attachment-note.yaml +8 -0
- package/lexicons/email-attribution.yaml +10 -0
- package/lexicons/email-header-field.yaml +37 -0
- package/lexicons/email-written-field.yaml +11 -0
- package/lexicons/figure-label.yaml +11 -0
- package/lexicons/measure-unit.yaml +78 -0
- package/lexicons/neutral-ending.yaml +10 -0
- package/lexicons/numbered-label.yaml +50 -0
- package/lexicons/percent-unit.yaml +7 -0
- package/lexicons/placeholder-word.yaml +21 -0
- package/lexicons/range-closer.yaml +5 -0
- package/lexicons/range-connector.yaml +11 -0
- package/lexicons/range-opener.yaml +7 -0
- package/lexicons/share-exception.yaml +7 -0
- package/lexicons/share-label.yaml +11 -0
- package/lexicons/stock-transition.yaml +11 -0
- package/package.json +1 -1
- package/src/closing-quote.ts +53 -0
- package/src/english-run.ts +25 -0
- package/src/index.ts +54 -13
- package/src/inside-brackets.ts +45 -0
- package/src/label-stop.ts +26 -0
- package/src/long-vowel-form.ts +38 -0
- package/src/number-stop.ts +30 -0
- package/src/quantities.ts +12 -1
- package/src/quoted-stop.ts +53 -0
- package/src/regexp.ts +3 -0
- package/src/sentence-close.ts +10 -0
- package/src/sentence-split.ts +3 -1
- package/src/structure.ts +24 -4
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lang-en/src/number-stop.ts の写し。和文の中の英文も同じ分割器で切るので同じ手当てが要る。アダプタは互いに依存しないので写しを置く。
|
|
3
|
+
*
|
|
4
|
+
* sentence-splitter は空白の後の「数字だけ + ピリオド」を、どこにあっても箇条書きの番号と読み、そこで文を切らない。
|
|
5
|
+
* 「in the spring of 2026. We shipped」が 1 文になる。
|
|
6
|
+
*
|
|
7
|
+
* 行の途中にあり、次の語が大文字で始まる番号は文末である。その数字だけを同じ長さの英字に替えて分割器に渡し、
|
|
8
|
+
* 普通の語として読ませる。長さが変わらないので、返る span は元の文字列にそのまま当てはまる。
|
|
9
|
+
* 行頭の番号(「1. First」)と、次が大文字でない番号は替えない。
|
|
10
|
+
*/
|
|
11
|
+
const NUMBER_WORD = /(?<=\s)\d+(?=\.\s)/g;
|
|
12
|
+
// 次の語の頭。開き括弧・引用符の後の大文字も文頭である。
|
|
13
|
+
const CAPITAL_AFTER_SPACE = /\s+[\p{Ps}\p{Pi}"']*\p{Lu}/uy;
|
|
14
|
+
// 分割器の略語表に n だけでできた語は無い。数字の代わりに置いても、略語として読まれない。
|
|
15
|
+
const PLAIN_LETTER = "n";
|
|
16
|
+
|
|
17
|
+
const lineStartOf = (text: string, index: number): number => Math.max(text.lastIndexOf("\n", index - 1), text.lastIndexOf("\r", index - 1)) + 1;
|
|
18
|
+
|
|
19
|
+
const isFirstOnLine = (text: string, index: number): boolean => text.slice(lineStartOf(text, index), index).trim() === "";
|
|
20
|
+
|
|
21
|
+
const capitalFollows = (text: string, dot: number): boolean => {
|
|
22
|
+
CAPITAL_AFTER_SPACE.lastIndex = dot + 1;
|
|
23
|
+
return CAPITAL_AFTER_SPACE.test(text);
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const isSentenceEnd = (text: string, index: number, digits: string): boolean => !isFirstOnLine(text, index) && capitalFollows(text, index + digits.length);
|
|
27
|
+
|
|
28
|
+
/** 文末の番号の数字を英字に替えた文字列。長さは変わらない。 */
|
|
29
|
+
export const unmarkNumberStops = (text: string): string =>
|
|
30
|
+
text.replace(NUMBER_WORD, (digits: string, index: number) => (isSentenceEnd(text, index, digits) ? PLAIN_LETTER.repeat(digits.length) : digits));
|
package/src/quantities.ts
CHANGED
|
@@ -93,6 +93,17 @@ const LOOKAHEAD = 8;
|
|
|
93
93
|
*/
|
|
94
94
|
const rereadAfterSpace = (number: string, rest: string): Morph[] => morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
|
|
95
95
|
|
|
96
|
+
/**
|
|
97
|
+
* 空白の後ろだけを読むと、名詞につく接頭詞と名詞で始まる(「1.4 本利用ルール」「1.2 本規約」の「本」)。詰めて読み直すと
|
|
98
|
+
* 助数詞に見えるが、行頭の番号の後ろなら、書き手が空けたのは番号と題のあいだ。「3 本の鉛筆」の「本」は単独で名詞、
|
|
99
|
+
* 「0.5 枚の紙」の「枚」は数につく接頭詞と読まれるので、どちらも数量のまま。
|
|
100
|
+
* 文中の数量には使わない。「10 両編成」の「両」も単独では名詞につく接頭詞と読まれる。
|
|
101
|
+
*/
|
|
102
|
+
const opensWithPrefix = (rest: string): boolean => {
|
|
103
|
+
const [first, second] = morphemes(toHalfWidth(rest.slice(0, LOOKAHEAD))) ?? [];
|
|
104
|
+
return first?.pos === "接頭詞" && first.detail1 === "名詞接続" && second?.pos === "名詞";
|
|
105
|
+
};
|
|
106
|
+
|
|
96
107
|
const unitAfterSpace = (number: string, rest: string): string | undefined => {
|
|
97
108
|
const next = rereadAfterSpace(number, rest).find((morph) => morph.start >= number.length);
|
|
98
109
|
return next !== undefined && isCounter(next) ? next.surface : undefined;
|
|
@@ -285,5 +296,5 @@ const countedByTableAfter = (rest: string): boolean => {
|
|
|
285
296
|
*/
|
|
286
297
|
export const countedAfter = (number: string, rest: string): boolean => {
|
|
287
298
|
if (!isReady()) return countedByTableAfter(rest);
|
|
288
|
-
return unitAfterSpace(number, rest) !== undefined || multiplierAfter(number, rest);
|
|
299
|
+
return (unitAfterSpace(number, rest) !== undefined && !opensWithPrefix(rest)) || multiplierAfter(number, rest);
|
|
289
300
|
};
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// lang-en/src/quoted-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
|
|
2
|
+
import type { Span } from "chaffjs/plugin";
|
|
3
|
+
import { isAbbreviation } from "./sentence-split.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* 米国式の引用は、文末のピリオドを閉じ引用符の内側に置く(…called it "a fair trial." Plainly, …)。
|
|
7
|
+
* sentence-splitter は引用符の中の句点で文を切らず、引用符が閉じた後の空白でも切らないので、次の文が前の文につながる。
|
|
8
|
+
*
|
|
9
|
+
* 分割器が返した一つの文を、文末の記号 + 閉じ引用符 + 空白 + 次の文の頭(大文字。開き括弧・引用符の後の大文字も)で切る。
|
|
10
|
+
* 引用符の中の語が略語・頭文字("U.S." "Dr." "J.")のときと、括弧が開いたままのときは切らない。
|
|
11
|
+
*/
|
|
12
|
+
/** 次の文の頭。空白の後の大文字。開き括弧・引用符の後の大文字も文頭である。 */
|
|
13
|
+
export const NEXT_SENTENCE_HEAD = String.raw`\s+[\p{Ps}\p{Pi}"']*\p{Lu}`;
|
|
14
|
+
const QUOTED_STOP = new RegExp(String.raw`[.?!]["'”’]+(?=${NEXT_SENTENCE_HEAD})`, "gu");
|
|
15
|
+
// ピリオドの前の語は、空白・開き引用符・開き括弧の後から数える。
|
|
16
|
+
const WORD_START = /[\s\p{Ps}\p{Pi}"']/u;
|
|
17
|
+
// 一字ずつピリオドを打った語(J. / U.S. / e.g.)は、略語の一覧に無くても略語。
|
|
18
|
+
const INITIALS = /^(?:\p{L}\.)+$/u;
|
|
19
|
+
// 分割器が一組として読む括弧(( [ { ( [ { 【 《 「 『 …)は、Unicode の開き・閉じ括弧の分類にすべて入る。
|
|
20
|
+
const OPENER = /\p{Ps}/u;
|
|
21
|
+
const CLOSER = /\p{Pe}/u;
|
|
22
|
+
|
|
23
|
+
const endsWithAbbreviation = (sentence: string, stop: number): boolean => {
|
|
24
|
+
if (sentence[stop] !== ".") return false;
|
|
25
|
+
const word = `${sentence.slice(0, stop).split(WORD_START).at(-1) ?? ""}.`;
|
|
26
|
+
return isAbbreviation(word) || INITIALS.test(word);
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// 対の無い閉じ括弧(「1) 最初の項目」の番号)は、後から開く括弧を打ち消さない。
|
|
30
|
+
const depthAfter = (depth: number, char: string): number => {
|
|
31
|
+
if (OPENER.test(char)) return depth + 1;
|
|
32
|
+
return CLOSER.test(char) ? Math.max(0, depth - 1) : depth;
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const bracketOpenAt = (sentence: string, index: number): boolean => Array.from(sentence.slice(0, index)).reduce(depthAfter, 0) > 0;
|
|
36
|
+
|
|
37
|
+
type Cut = { readonly end: number; readonly next: number };
|
|
38
|
+
|
|
39
|
+
const cutsIn = (sentence: string): Cut[] =>
|
|
40
|
+
[...sentence.matchAll(QUOTED_STOP)]
|
|
41
|
+
.filter((match) => !endsWithAbbreviation(sentence, match.index) && !bracketOpenAt(sentence, match.index))
|
|
42
|
+
.map((match) => {
|
|
43
|
+
const end = match.index + match[0].length;
|
|
44
|
+
return { end, next: end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0) };
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
/** span の文を、閉じ引用符の内側で閉じた文ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
|
|
48
|
+
export const splitAtQuotedStops = (text: string, span: Span): Span[] => {
|
|
49
|
+
const cuts = cutsIn(text.slice(span.start, span.end));
|
|
50
|
+
const starts = [0, ...cuts.map((cut) => cut.next)];
|
|
51
|
+
const ends = [...cuts.map((cut) => cut.end), span.end - span.start];
|
|
52
|
+
return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
|
|
53
|
+
};
|
package/src/regexp.ts
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
// lang-en/src/regexp.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
|
|
2
|
+
/** A word from a lexicon, matched as written: a sign in it is that character, not a pattern. */
|
|
3
|
+
export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`);
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 文が閉じているか。句点の後に閉じ括弧と空白が続いてよい。
|
|
3
|
+
*
|
|
4
|
+
* 「.」は、論文などが「,.」で書くときの句点。仮名・漢字か閉じ括弧の後だけ句点と読む。
|
|
5
|
+
* 数字の後の「.」は番号か小数点なので(1.はじめに、3.5%)、文を閉じない。
|
|
6
|
+
*/
|
|
7
|
+
const CLOSED = /[。!?!?]["))」』]*\s*$/u;
|
|
8
|
+
const FULL_STOP_AFTER_WORD = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}ー))」』].["))」』]*\s*$/u;
|
|
9
|
+
|
|
10
|
+
export const closesSentence = (text: string): boolean => CLOSED.test(text) || FULL_STOP_AFTER_WORD.test(text);
|
package/src/sentence-split.ts
CHANGED
|
@@ -45,6 +45,8 @@ const { language } = DefaultAbbrMarkerOptions;
|
|
|
45
45
|
const ABBREVIATIONS = new Set(
|
|
46
46
|
[...language.ABBREVIATIONS, ...language.PREPOSITIVE_ABBREVIATIONS, ...language.EXCLAMATION_WORDS].map((word) => word.toLowerCase()),
|
|
47
47
|
);
|
|
48
|
+
/** 分割器が略語と読む語(ピリオドまで含めて渡す)。「Dr.」「U.S.」「Jan.」。 */
|
|
49
|
+
export const isAbbreviation = (word: string): boolean => ABBREVIATIONS.has(word.toLowerCase());
|
|
48
50
|
// 「J. Smith」のような語は前の語を見て略語か決まる。前の語が切れ目の向こうにあると判定が変わる。
|
|
49
51
|
const CAPITAL_DOT = /\p{Lu}\./gu;
|
|
50
52
|
|
|
@@ -76,7 +78,7 @@ const indexesOf = (text: string, pattern: RegExp): number[] => [...text.matchAll
|
|
|
76
78
|
/** 句点の並びの直後。和文は直前が仮名・漢字の句点、英文は略語でない語のピリオド。 */
|
|
77
79
|
const stopEnds = (text: string): number[] => {
|
|
78
80
|
const japanese = [...text.matchAll(JAPANESE_STOPS)].filter((match) => isCJK(text[match.index - 1]));
|
|
79
|
-
const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !
|
|
81
|
+
const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !isAbbreviation(match[0]));
|
|
80
82
|
return [...japanese, ...english].map((match) => match.index + match[0].length).toSorted((a, b) => a - b);
|
|
81
83
|
};
|
|
82
84
|
|
package/src/structure.ts
CHANGED
|
@@ -254,10 +254,22 @@ const depthsOf = (text: string): Int32Array => {
|
|
|
254
254
|
* 「民法第709条」のように他の文書の名前が前にあれば、その名前を document に入れる。この文書の木では引かない。
|
|
255
255
|
*/
|
|
256
256
|
/** at より前で開いたまま閉じていない括弧の数。 */
|
|
257
|
-
/**
|
|
257
|
+
/**
|
|
258
|
+
* 間に挟まった括弧書き(中に参照があっても)を外す。入れ子は内側から外れる。閉じない「(」と開かない「)」は残す。
|
|
259
|
+
* 内側を外すのを繰り返すと深さの数だけ読み直すので、組ごと 1 度で外す。
|
|
260
|
+
*/
|
|
258
261
|
const withoutClosedParentheses = (gap: string): string => {
|
|
259
|
-
const
|
|
260
|
-
|
|
262
|
+
const kept: string[] = [];
|
|
263
|
+
const opens: number[] = [];
|
|
264
|
+
for (const char of gap) {
|
|
265
|
+
const opened = char === ")" ? opens.pop() : undefined;
|
|
266
|
+
if (opened !== undefined) kept.splice(opened);
|
|
267
|
+
else {
|
|
268
|
+
if (char === "(") opens.push(kept.length);
|
|
269
|
+
kept.push(char);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return kept.join("");
|
|
261
273
|
};
|
|
262
274
|
|
|
263
275
|
/**
|
|
@@ -283,9 +295,17 @@ const LEXICONS = loadLexicons();
|
|
|
283
295
|
const CITATION = citationVocabulary(LEXICONS);
|
|
284
296
|
const SECTIONS = sectionVocabulary(LEXICONS);
|
|
285
297
|
const NOT_MAGNITUDE = (LEXICONS["not-magnitude"] ?? []).map((entry) => entry.pattern);
|
|
298
|
+
const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
|
|
299
|
+
|
|
300
|
+
/** 記号のすぐ後ろにラテン文字・数字・ハイフンが続けば、単位ではなく語の頭(「2.1 mmap」)。 */
|
|
301
|
+
const CONTINUES_WORD = /^[\p{Script=Latin}\p{Nd}_-]/u;
|
|
302
|
+
|
|
303
|
+
/** 「1.5 mM の塩化マグネシウム」「37.5 ℃で」: 解析器が助数詞と読まない単位の記号。 */
|
|
304
|
+
const startsWithMeasureUnit = (rest: string): boolean => MEASURE_UNITS.some((unit) => rest.startsWith(unit) && !CONTINUES_WORD.test(rest.slice(unit.length)));
|
|
286
305
|
|
|
287
306
|
/** 辞書が桁の語と助数詞に切る一語(「1.5 万葉の世界」)は数量の続きではないので、行頭の番号は節のまま。 */
|
|
288
|
-
const countedAfterNumber = (number: string, rest: string): boolean =>
|
|
307
|
+
const countedAfterNumber = (number: string, rest: string): boolean =>
|
|
308
|
+
startsWithMeasureUnit(rest) || (!NOT_MAGNITUDE.some((word) => rest.startsWith(word)) && countedAfter(number, rest));
|
|
289
309
|
|
|
290
310
|
const sectionsOf = (text: string): Mention[] => sectionReferences(text, SECTIONS);
|
|
291
311
|
|