@chaffjs/lang-ja 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/closing-quote.d.ts +6 -0
- package/dist/closing-quote.d.ts.map +1 -0
- package/dist/closing-quote.js +50 -0
- package/dist/closing-quote.js.map +1 -0
- package/dist/english-run.d.ts +8 -0
- package/dist/english-run.d.ts.map +1 -0
- package/dist/english-run.js +24 -0
- package/dist/english-run.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +56 -21
- package/dist/index.js.map +1 -1
- package/dist/inside-brackets.d.ts +5 -0
- package/dist/inside-brackets.d.ts.map +1 -0
- package/dist/inside-brackets.js +44 -0
- package/dist/inside-brackets.js.map +1 -0
- package/dist/label-stop.d.ts +13 -0
- package/dist/label-stop.d.ts.map +1 -0
- package/dist/label-stop.js +14 -0
- package/dist/label-stop.js.map +1 -0
- package/dist/long-vowel-form.d.ts +12 -0
- package/dist/long-vowel-form.d.ts.map +1 -0
- package/dist/long-vowel-form.js +28 -0
- package/dist/long-vowel-form.js.map +1 -0
- package/dist/number-stop.d.ts +3 -0
- package/dist/number-stop.d.ts.map +1 -0
- package/dist/number-stop.js +25 -0
- package/dist/number-stop.js.map +1 -0
- package/dist/quantities.d.ts.map +1 -1
- package/dist/quantities.js +11 -1
- package/dist/quantities.js.map +1 -1
- package/dist/quoted-stop.d.ts +13 -0
- package/dist/quoted-stop.d.ts.map +1 -0
- package/dist/quoted-stop.js +45 -0
- package/dist/quoted-stop.js.map +1 -0
- package/dist/regexp.d.ts +3 -0
- package/dist/regexp.d.ts.map +1 -0
- package/dist/regexp.js +4 -0
- package/dist/regexp.js.map +1 -0
- package/dist/sentence-close.d.ts +2 -0
- package/dist/sentence-close.d.ts.map +1 -0
- package/dist/sentence-close.js +10 -0
- package/dist/sentence-close.js.map +1 -0
- package/dist/sentence-split.d.ts +2 -0
- package/dist/sentence-split.d.ts.map +1 -1
- package/dist/sentence-split.js +3 -1
- package/dist/sentence-split.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +23 -4
- package/dist/structure.js.map +1 -1
- package/lexicons/abbreviated-label.yaml +16 -0
- package/lexicons/ai-tell.yaml +69 -0
- package/lexicons/announcing-opener.yaml +11 -0
- package/lexicons/assistant-residue.yaml +37 -0
- package/lexicons/calendar-era.yaml +10 -0
- package/lexicons/closing-cliche.yaml +2 -0
- package/lexicons/contrast-frame.yaml +9 -0
- package/lexicons/contrast-lead.yaml +10 -0
- package/lexicons/contrast-turn.yaml +9 -0
- package/lexicons/count-anchor.yaml +10 -0
- package/lexicons/count-counter.yaml +17 -0
- package/lexicons/count-hedge.yaml +61 -0
- package/lexicons/count-number.yaml +16 -0
- package/lexicons/email-attachment-note.yaml +8 -0
- package/lexicons/email-attribution.yaml +10 -0
- package/lexicons/email-header-field.yaml +37 -0
- package/lexicons/email-written-field.yaml +11 -0
- package/lexicons/figure-label.yaml +11 -0
- package/lexicons/measure-unit.yaml +78 -0
- package/lexicons/neutral-ending.yaml +10 -0
- package/lexicons/numbered-label.yaml +50 -0
- package/lexicons/percent-unit.yaml +7 -0
- package/lexicons/placeholder-word.yaml +21 -0
- package/lexicons/range-closer.yaml +5 -0
- package/lexicons/range-connector.yaml +11 -0
- package/lexicons/range-opener.yaml +7 -0
- package/lexicons/share-exception.yaml +7 -0
- package/lexicons/share-label.yaml +11 -0
- package/lexicons/stock-transition.yaml +11 -0
- package/package.json +1 -1
- package/src/closing-quote.ts +53 -0
- package/src/english-run.ts +25 -0
- package/src/index.ts +54 -13
- package/src/inside-brackets.ts +45 -0
- package/src/label-stop.ts +26 -0
- package/src/long-vowel-form.ts +38 -0
- package/src/number-stop.ts +30 -0
- package/src/quantities.ts +12 -1
- package/src/quoted-stop.ts +53 -0
- package/src/regexp.ts +3 -0
- package/src/sentence-close.ts +10 -0
- package/src/sentence-split.ts +3 -1
- package/src/structure.ts +24 -4
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# メールのヘッダーの項目名。メッセージの頭か区切りの行の後で、これを名乗る「項目: 値」の行の塊は、書いた人の文章ではなくヘッダー。
|
|
2
|
+
# 日本語の画面でも英語の項目名が出るので、英語の名前も入れる。大文字小文字は区別しない。
|
|
3
|
+
id: email-header-field
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: 件名
|
|
7
|
+
- pattern: 差出人
|
|
8
|
+
- pattern: 送信者
|
|
9
|
+
- pattern: 宛先
|
|
10
|
+
- pattern: 宛て先
|
|
11
|
+
- pattern: 送信日時
|
|
12
|
+
- pattern: 受信日時
|
|
13
|
+
- pattern: 日付
|
|
14
|
+
- pattern: 返信先
|
|
15
|
+
- pattern: From
|
|
16
|
+
- pattern: To
|
|
17
|
+
- pattern: Cc
|
|
18
|
+
- pattern: Bcc
|
|
19
|
+
- pattern: Subject
|
|
20
|
+
- pattern: Date
|
|
21
|
+
- pattern: Sent
|
|
22
|
+
- pattern: Reply-To
|
|
23
|
+
- pattern: Message-ID
|
|
24
|
+
- pattern: In-Reply-To
|
|
25
|
+
- pattern: References
|
|
26
|
+
- pattern: Received
|
|
27
|
+
- pattern: Return-Path
|
|
28
|
+
- pattern: Delivered-To
|
|
29
|
+
- pattern: Sender
|
|
30
|
+
- pattern: Organization
|
|
31
|
+
- pattern: Importance
|
|
32
|
+
- pattern: MIME-Version
|
|
33
|
+
- pattern: Content-Type
|
|
34
|
+
- pattern: Content-Transfer-Encoding
|
|
35
|
+
- pattern: User-Agent
|
|
36
|
+
- pattern: X-Mailer
|
|
37
|
+
- pattern: List-Id
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# 測った数の後ろに書く単位の記号(1.5 mM、37 ℃、2.5 mg/kg)。行頭の小数の後ろがこれなら数量で、節の番号ではない。
|
|
2
|
+
# 助数詞(倍・本・日)は解析器が読むので、ここには解析器が助数詞と読まない記号を置く。書かれたとおりに当て、すぐ後ろに
|
|
3
|
+
# ラテン文字・数字・ハイフンが続くときは当てない(「2.1 mmap の使い方」は節の題)。大文字で始まる記号(Da・Pa・GB)は、
|
|
4
|
+
# 英語の表と同じく入れない。
|
|
5
|
+
id: measure-unit
|
|
6
|
+
language: ja
|
|
7
|
+
entries:
|
|
8
|
+
- pattern: "mM"
|
|
9
|
+
- pattern: "µM"
|
|
10
|
+
- pattern: "μM"
|
|
11
|
+
- pattern: "nM"
|
|
12
|
+
- pattern: "pM"
|
|
13
|
+
- pattern: "mol"
|
|
14
|
+
- pattern: "mmol"
|
|
15
|
+
- pattern: "µmol"
|
|
16
|
+
- pattern: "μmol"
|
|
17
|
+
- pattern: "nmol"
|
|
18
|
+
- pattern: "pmol"
|
|
19
|
+
- pattern: "kg"
|
|
20
|
+
- pattern: "mg"
|
|
21
|
+
- pattern: "µg"
|
|
22
|
+
- pattern: "μg"
|
|
23
|
+
- pattern: "ng"
|
|
24
|
+
- pattern: "pg"
|
|
25
|
+
- pattern: "kDa"
|
|
26
|
+
- pattern: "mL"
|
|
27
|
+
- pattern: "ml"
|
|
28
|
+
- pattern: "µL"
|
|
29
|
+
- pattern: "μL"
|
|
30
|
+
- pattern: "µl"
|
|
31
|
+
- pattern: "μl"
|
|
32
|
+
- pattern: "dL"
|
|
33
|
+
- pattern: "nL"
|
|
34
|
+
- pattern: "km"
|
|
35
|
+
- pattern: "cm"
|
|
36
|
+
- pattern: "mm"
|
|
37
|
+
- pattern: "µm"
|
|
38
|
+
- pattern: "μm"
|
|
39
|
+
- pattern: "nm"
|
|
40
|
+
- pattern: "ms"
|
|
41
|
+
- pattern: "µs"
|
|
42
|
+
- pattern: "μs"
|
|
43
|
+
- pattern: "ns"
|
|
44
|
+
- pattern: "min"
|
|
45
|
+
- pattern: "sec"
|
|
46
|
+
- pattern: "hr"
|
|
47
|
+
- pattern: "hrs"
|
|
48
|
+
- pattern: "kHz"
|
|
49
|
+
- pattern: "mA"
|
|
50
|
+
- pattern: "mV"
|
|
51
|
+
- pattern: "kV"
|
|
52
|
+
- pattern: "kW"
|
|
53
|
+
- pattern: "kWh"
|
|
54
|
+
- pattern: "mAh"
|
|
55
|
+
- pattern: "kPa"
|
|
56
|
+
- pattern: "mmHg"
|
|
57
|
+
- pattern: "atm"
|
|
58
|
+
- pattern: "psi"
|
|
59
|
+
- pattern: "kB"
|
|
60
|
+
- pattern: "bp"
|
|
61
|
+
- pattern: "kb"
|
|
62
|
+
- pattern: "kbp"
|
|
63
|
+
- pattern: "ppm"
|
|
64
|
+
- pattern: "ppb"
|
|
65
|
+
- pattern: "rpm"
|
|
66
|
+
- pattern: "dB"
|
|
67
|
+
- pattern: "mCi"
|
|
68
|
+
- pattern: "µCi"
|
|
69
|
+
- pattern: "μCi"
|
|
70
|
+
- pattern: "mGy"
|
|
71
|
+
- pattern: "mSv"
|
|
72
|
+
- pattern: "°C"
|
|
73
|
+
- pattern: "°F"
|
|
74
|
+
- pattern: "℃"
|
|
75
|
+
- pattern: "℉"
|
|
76
|
+
- pattern: "°"
|
|
77
|
+
- pattern: "%"
|
|
78
|
+
- pattern: "‰"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# 番号と組んで、題や設問に名前を付ける語。position は番号のどちら側に書くか。
|
|
2
|
+
# before は番号のすぐ前(問3、図2、例 3:)。見出しの頭にあれば札で、見出しと文の重なり(heading-echo)には数えない。語として照らす(「質問3」の「問」は照らさない)。後ろの語が助数詞や助詞なら数量として数える(図3枚、問3の)。
|
|
3
|
+
# after は「第」と番号の後ろ(第1節、第2章)。その後ろの空白は題との区切りで、空け方の好みではない。字面どおりに照らす。
|
|
4
|
+
id: numbered-label
|
|
5
|
+
language: ja
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: 問
|
|
8
|
+
position: before
|
|
9
|
+
- pattern: 設問
|
|
10
|
+
position: before
|
|
11
|
+
- pattern: 項目
|
|
12
|
+
position: before
|
|
13
|
+
- pattern: 図
|
|
14
|
+
position: before
|
|
15
|
+
- pattern: 表
|
|
16
|
+
position: before
|
|
17
|
+
- pattern: 別表
|
|
18
|
+
position: before
|
|
19
|
+
- pattern: 別紙
|
|
20
|
+
position: before
|
|
21
|
+
- pattern: 様式
|
|
22
|
+
position: before
|
|
23
|
+
- pattern: 例
|
|
24
|
+
position: before
|
|
25
|
+
- pattern: 手順
|
|
26
|
+
position: before
|
|
27
|
+
- pattern: ステップ
|
|
28
|
+
position: before
|
|
29
|
+
- pattern: Step
|
|
30
|
+
position: before
|
|
31
|
+
- pattern: 編
|
|
32
|
+
position: after
|
|
33
|
+
- pattern: 部
|
|
34
|
+
position: after
|
|
35
|
+
- pattern: 章
|
|
36
|
+
position: after
|
|
37
|
+
- pattern: 節
|
|
38
|
+
position: after
|
|
39
|
+
- pattern: 款
|
|
40
|
+
position: after
|
|
41
|
+
- pattern: 目
|
|
42
|
+
position: after
|
|
43
|
+
- pattern: 条
|
|
44
|
+
position: after
|
|
45
|
+
- pattern: 項
|
|
46
|
+
position: after
|
|
47
|
+
- pattern: 号
|
|
48
|
+
position: after
|
|
49
|
+
- pattern: 問
|
|
50
|
+
position: after
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# 雛形の空欄の括弧に書く語。【会社名】【氏名】。括弧の中がこの語そのものか、この語で始まって空白が続くときに数える。
|
|
2
|
+
# 例として置く「○○」は正しい書き方なので入れない(官公庁の文書は記入例に使う)。
|
|
3
|
+
id: placeholder-word
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: 氏名
|
|
7
|
+
- pattern: 名前
|
|
8
|
+
- pattern: 会社名
|
|
9
|
+
- pattern: 企業名
|
|
10
|
+
- pattern: 貴社名
|
|
11
|
+
- pattern: 担当者名
|
|
12
|
+
- pattern: 部署名
|
|
13
|
+
- pattern: 商品名
|
|
14
|
+
- pattern: 製品名
|
|
15
|
+
- pattern: サービス名
|
|
16
|
+
- pattern: 日付
|
|
17
|
+
- pattern: 金額
|
|
18
|
+
- pattern: 住所
|
|
19
|
+
- pattern: 電話番号
|
|
20
|
+
- pattern: メールアドレス
|
|
21
|
+
- pattern: ここに
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# 一つの全体を分けた割合(足すと 100% になるもの)を示す語。表の見出しか、箇条書きや表のすぐ上の文・見出しにこの語があれば、その百分率を足して 100% と比べる。
|
|
2
|
+
# 「割合」「比率」は行ごとの率(利用率の割合)にも使うので入れない。
|
|
3
|
+
id: share-label
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: 構成比
|
|
7
|
+
- pattern: 構成割合
|
|
8
|
+
- pattern: 内訳
|
|
9
|
+
- pattern: シェア
|
|
10
|
+
- pattern: 占有率
|
|
11
|
+
- pattern: 配分
|
package/package.json
CHANGED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// lang-en/src/closing-quote.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
|
|
2
|
+
import type { Span } from "chaffjs/plugin";
|
|
3
|
+
import { NEXT_SENTENCE_HEAD } from "./quoted-stop.ts";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* sentence-splitter は疑問符・感嘆符の後の曲がった閉じ引用符(“Is it done?” Nobody …)と直線の一重引用符の手前で文を切る。
|
|
7
|
+
* 次の文が「”」で始まり、前の文は引用符が開いたまま終わる。
|
|
8
|
+
*
|
|
9
|
+
* 前の文が句点で終わり、次の文がそこから間を置かずに閉じ引用符(閉じ括弧も)で始まるとき、それを前の文の末尾へ戻す。
|
|
10
|
+
* 戻した後に空白と大文字が続けば文は二つのまま、何も続かなければ前の文だけ、
|
|
11
|
+
* 小文字・数字・句読点が続けば引用は文の途中にあるので一つの文につなぐ(直線の二重引用符で分割器がそうするのと同じ)。
|
|
12
|
+
* 開き引用符(“ ‘)は閉じの分類に入らない。語頭のアポストロフィ(’Tis ’90s)は後ろに字が続くので閉じと読まない。
|
|
13
|
+
*/
|
|
14
|
+
const CLOSING_RUN = /^[\p{Pf}\p{Pe}"']+/u;
|
|
15
|
+
const WORD_CHARACTER = /^[\p{L}\p{N}]/u;
|
|
16
|
+
const STOP_AT_END = /[.?!]$/u;
|
|
17
|
+
// 語頭のアポストロフィの後の大文字(’Tis)も次の文の頭。
|
|
18
|
+
const NEXT_SENTENCE = new RegExp(String.raw`^(?:${NEXT_SENTENCE_HEAD}|\s+’\p{Lu})`, "u");
|
|
19
|
+
const LEADING_SPACE = /^\s*/u;
|
|
20
|
+
|
|
21
|
+
/** 文頭の閉じ引用符の並びの長さ。後ろに字が続けば開き引用符かアポストロフィなので 0。 */
|
|
22
|
+
export const closingRunLength = (sentence: string): number => {
|
|
23
|
+
const run = CLOSING_RUN.exec(sentence)?.[0].length ?? 0;
|
|
24
|
+
return run > 0 && !WORD_CHARACTER.test(sentence.slice(run)) ? run : 0;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const follows = (text: string, previous: Span, span: Span): boolean =>
|
|
28
|
+
previous.end === span.start && STOP_AT_END.test(text.slice(previous.start, previous.end));
|
|
29
|
+
|
|
30
|
+
/** previous と span を置き換える文。閉じ引用符が戻らなければ undefined。 */
|
|
31
|
+
const reattached = (text: string, previous: Span, span: Span): Span[] | undefined => {
|
|
32
|
+
const run = follows(text, previous, span) ? closingRunLength(text.slice(span.start, span.end)) : 0;
|
|
33
|
+
if (run === 0) return undefined;
|
|
34
|
+
const quoteEnd = span.start + run;
|
|
35
|
+
const rest = text.slice(quoteEnd, span.end);
|
|
36
|
+
if (rest.trim() === "") return [{ start: previous.start, end: quoteEnd }];
|
|
37
|
+
if (!NEXT_SENTENCE.test(rest)) return [{ start: previous.start, end: span.end }];
|
|
38
|
+
const next = quoteEnd + (LEADING_SPACE.exec(rest)?.[0].length ?? 0);
|
|
39
|
+
return [
|
|
40
|
+
{ start: previous.start, end: quoteEnd },
|
|
41
|
+
{ start: next, end: span.end },
|
|
42
|
+
];
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/** 文の span の並び(text 上の位置)で、文頭に取り残された閉じ引用符を前の文へ戻したもの。 */
|
|
46
|
+
export const reattachClosingQuotes = (text: string, spans: readonly Span[]): Span[] =>
|
|
47
|
+
spans.reduce<Span[]>((sentences, span) => {
|
|
48
|
+
const previous = sentences.at(-1);
|
|
49
|
+
const replaced = previous === undefined ? undefined : reattached(text, previous, span);
|
|
50
|
+
if (replaced === undefined) sentences.push(span);
|
|
51
|
+
else sentences.splice(-1, 1, ...replaced);
|
|
52
|
+
return sentences;
|
|
53
|
+
}, []);
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 和文の中の英文(英語の要旨、英語の文献)。仮名を含まず、文字(数字と記号は数えない)の半分以上が英字。
|
|
3
|
+
* 数字を数えると「In 1997.」のような年の入った短い英文が英文でなくなる。
|
|
4
|
+
* 漢字だけなら含んでよい(英文の中の「東京」)。仮名があれば和文の一部と読む。
|
|
5
|
+
*/
|
|
6
|
+
const KANA = /[\p{Script=Hiragana}\p{Script=Katakana}]/u;
|
|
7
|
+
const LATIN = /\p{Script=Latin}/gu;
|
|
8
|
+
const LETTER = /\p{L}/gu;
|
|
9
|
+
// 英文の文末。閉じ括弧・引用符が続いてよい。断片の末尾だけを見る。
|
|
10
|
+
const ENGLISH_STOP_AT_END = /[.?!][)\]"'”’]{0,3}$/u;
|
|
11
|
+
const TAIL_LENGTH = 4;
|
|
12
|
+
|
|
13
|
+
export const isEnglishRun = (text: string): boolean => {
|
|
14
|
+
if (KANA.test(text)) return false;
|
|
15
|
+
const latin = [...text.matchAll(LATIN)].length;
|
|
16
|
+
return latin > 0 && latin * 2 >= [...text.matchAll(LETTER)].length;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* 分割器が切った二つの断片の間が、英文と英文の切れ目か。
|
|
21
|
+
* 後ろが和文なら切れ目と読まない。「Version 2.0. を使う」「See Fig. 1. これは図」の英字の後のピリオドと見分けられないため。
|
|
22
|
+
* before は前の断片をつないだもので長くなりうるので、短い after を先に見る。
|
|
23
|
+
*/
|
|
24
|
+
export const isEnglishBoundary = (before: string, after: string): boolean =>
|
|
25
|
+
ENGLISH_STOP_AT_END.test(before.trimEnd().slice(-TAIL_LENGTH)) && isEnglishRun(after) && isEnglishRun(before);
|
package/src/index.ts
CHANGED
|
@@ -1,16 +1,27 @@
|
|
|
1
1
|
import { loadLexicons } from "./lexicons.ts";
|
|
2
|
+
import { closesSentence } from "./sentence-close.ts";
|
|
2
3
|
import { sentenceSpans } from "./sentence-split.ts";
|
|
4
|
+
import { isEnglishBoundary, isEnglishRun } from "./english-run.ts";
|
|
5
|
+
import { unmarkNumberStops } from "./number-stop.ts";
|
|
6
|
+
import { labelStops, unmarkLabelStops } from "./label-stop.ts";
|
|
7
|
+
import { splitAtQuotedStops } from "./quoted-stop.ts";
|
|
8
|
+
import { reattachClosingQuotes } from "./closing-quote.ts";
|
|
9
|
+
import { insideBrackets, startsWithCloser } from "./inside-brackets.ts";
|
|
3
10
|
import { structure } from "./structure.ts";
|
|
4
11
|
import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.ts";
|
|
5
12
|
import { markSpacedCounters } from "./spaced-counter.ts";
|
|
6
13
|
import { tokensWithin } from "./tokens-within.ts";
|
|
7
14
|
import { distributiveVocabulary, iterationMarkReading, markReduplication } from "./reduplication.ts";
|
|
8
|
-
import
|
|
15
|
+
import { knownWordReading, markDroppedLongVowels, remembered } from "./long-vowel-form.ts";
|
|
16
|
+
import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
|
|
9
17
|
|
|
10
18
|
// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
|
|
11
19
|
|
|
20
|
+
const JAPANESE = /[-ゟ゠-ヿ一-鿿]/gu;
|
|
21
|
+
const COUNTABLE = /\S/gu;
|
|
22
|
+
|
|
12
23
|
/**
|
|
13
|
-
*
|
|
24
|
+
* 日本語の文末は「。!?」と、語の後の「.」に限られる(closesSentence)。
|
|
14
25
|
*
|
|
15
26
|
* sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
|
|
16
27
|
* AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
|
|
@@ -19,30 +30,42 @@ import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from
|
|
|
19
30
|
* 文長は sentence-rhythm と max-sentence-length の入力なので、
|
|
20
31
|
* 誤分割はそのまま指標を動かす。ここで閉じる。
|
|
21
32
|
*/
|
|
22
|
-
const
|
|
33
|
+
const isOpen = (text: string): boolean => text.trim().length > 0 && !closesSentence(text);
|
|
23
34
|
|
|
24
|
-
|
|
25
|
-
|
|
35
|
+
/**
|
|
36
|
+
* 和文の句点で閉じていない断片、括弧・二重引用符の中で切れた断片、閉じ括弧で始まる断片の前は次へ続く。
|
|
37
|
+
* 閉じ括弧は閉じる文に付き、後ろは分割器が組にした括弧(「…。」と言った)と同じく同じ文に続く。
|
|
38
|
+
* ただし英文どうしは、英語のアダプタと同じく英語の文末で切ったところで切る。
|
|
39
|
+
*/
|
|
40
|
+
const continues = (before: string, after: string, bracketOpen: boolean): boolean =>
|
|
41
|
+
(isOpen(before) || bracketOpen || startsWithCloser(after)) && !isEnglishBoundary(before, after);
|
|
42
|
+
|
|
43
|
+
const LINE_BREAK = /[\n\r]/u;
|
|
26
44
|
|
|
27
|
-
const
|
|
45
|
+
const ENGLISH: EmbeddedLanguage = { id: "en", lengthUnit: "word" };
|
|
46
|
+
|
|
47
|
+
const withLanguage = (text: string, span: Span): Sentence => (isEnglishRun(text) ? { span, text, embeddedLanguage: ENGLISH } : { span, text });
|
|
28
48
|
|
|
29
49
|
/**
|
|
30
50
|
* 断片を繋ぐときは raw の連結ではなくオフセットを使う。
|
|
31
51
|
* sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
|
|
32
52
|
*/
|
|
33
|
-
const merge = (source: string, spans: readonly Span[]): Sentence[] =>
|
|
34
|
-
|
|
53
|
+
const merge = (source: string, spans: readonly Span[]): Sentence[] => {
|
|
54
|
+
const open = insideBrackets(source);
|
|
55
|
+
return spans
|
|
35
56
|
.reduce<Span[]>((acc, span) => {
|
|
36
57
|
const last = acc.at(-1);
|
|
37
58
|
// 断片の「間」に改行があれば繋がない。結合は「Dr. 田中」のような行の途中の
|
|
38
59
|
// 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
|
|
39
60
|
// 英文と訳文のように別の行のものまで 1 文に繋がる。
|
|
40
|
-
const acrossLines = last !== undefined && source.slice(last.end, span.start)
|
|
41
|
-
if (last !== undefined && !acrossLines &&
|
|
61
|
+
const acrossLines = last !== undefined && LINE_BREAK.test(source.slice(last.end, span.start));
|
|
62
|
+
if (last !== undefined && !acrossLines && continues(source.slice(last.start, last.end), source.slice(span.start, span.end), open[last.end] === true))
|
|
63
|
+
acc[acc.length - 1] = { start: last.start, end: span.end };
|
|
42
64
|
else acc.push(span);
|
|
43
65
|
return acc;
|
|
44
66
|
}, [])
|
|
45
|
-
.map((span) => (
|
|
67
|
+
.map((span) => withLanguage(source.slice(span.start, span.end), span));
|
|
68
|
+
};
|
|
46
69
|
|
|
47
70
|
/**
|
|
48
71
|
* token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
|
|
@@ -51,11 +74,15 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
|
|
|
51
74
|
const LEXICONS = loadLexicons();
|
|
52
75
|
const DISTRIBUTIVE = distributiveVocabulary(LEXICONS);
|
|
53
76
|
const TAKES_ITERATION_MARK = iterationMarkReading(LEXICONS, readsAsOneWord);
|
|
77
|
+
const KNOWS_WITH_LONG_VOWEL = remembered(knownWordReading(tokenize));
|
|
78
|
+
/** Whether a running rule reads LongVowelEnding. Asking the dictionary about every katakana word costs, so only then. */
|
|
79
|
+
const askForLongVowels = { value: false };
|
|
54
80
|
|
|
55
81
|
const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
|
|
56
82
|
const read = tokenize(source);
|
|
57
83
|
if (read === undefined) return [...sentences];
|
|
58
|
-
const
|
|
84
|
+
const counted = markSpacedCounters(read, readsAsCounter);
|
|
85
|
+
const tokens = askForLongVowels.value ? markDroppedLongVowels(counted, KNOWS_WITH_LONG_VOWEL) : counted;
|
|
59
86
|
return sentences.map((sentence) => ({
|
|
60
87
|
...sentence,
|
|
61
88
|
// 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
|
|
@@ -63,6 +90,18 @@ const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[]
|
|
|
63
90
|
}));
|
|
64
91
|
};
|
|
65
92
|
|
|
93
|
+
const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* 英語のアダプタと同じ手当てを英文にも当てる。番号の前の略した名前(FIG. 1)の点で切らず、
|
|
97
|
+
* 英文の閉じ引用符の内側の句点で切り、文頭に取り残された閉じ引用符を前の文へ戻す。
|
|
98
|
+
*/
|
|
99
|
+
const spansOf = (text: string): Span[] => {
|
|
100
|
+
const spans = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS)));
|
|
101
|
+
const quotedStops = spans.flatMap((span) => (isEnglishRun(text.slice(span.start, span.end)) ? splitAtQuotedStops(text, span) : [span]));
|
|
102
|
+
return reattachClosingQuotes(text, quotedStops);
|
|
103
|
+
};
|
|
104
|
+
|
|
66
105
|
export const adapter: LanguageAdapter = {
|
|
67
106
|
kind: "language",
|
|
68
107
|
id: "ja",
|
|
@@ -76,6 +115,8 @@ export const adapter: LanguageAdapter = {
|
|
|
76
115
|
lengthUnit: "char",
|
|
77
116
|
},
|
|
78
117
|
prepare: async (need: AdapterNeeds): Promise<void> => {
|
|
118
|
+
// Once asked, keep marking: files are prepared in parallel, and a later prepare must not unmark an earlier file's run.
|
|
119
|
+
askForLongVowels.value ||= need.features?.includes("LongVowelEnding") ?? false;
|
|
79
120
|
if (need.pos) await prepare();
|
|
80
121
|
},
|
|
81
122
|
detect: (source: string): number => {
|
|
@@ -86,7 +127,7 @@ export const adapter: LanguageAdapter = {
|
|
|
86
127
|
lexicons: loadLexicons(),
|
|
87
128
|
structure,
|
|
88
129
|
segment: (text: string): Segmentation => {
|
|
89
|
-
const sentences = merge(text,
|
|
130
|
+
const sentences = merge(text, spansOf(text));
|
|
90
131
|
return { sentences: isReady() ? withTokens(text, sentences) : sentences };
|
|
91
132
|
},
|
|
92
133
|
};
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 括弧・二重引用符の中の句点は文を閉じない。sentence-splitter は括弧を種類ごとに一つしか数えないので、
|
|
3
|
+
* 同じ種類が入れ子になると内側の閉じ括弧で外側まで閉じたと読み(「(…(…)…。以下同じ。)に」)、曲がった二重引用符(“…”)は数えない。
|
|
4
|
+
* ここでは開きと閉じを入れ子のまま組にする。一重引用符(‘ ’)はアポストロフィと見分けられず、直線の引用符は分割器が数えるので、どちらも数えない。
|
|
5
|
+
*/
|
|
6
|
+
const OPENER = /[\p{Ps}“«]/u;
|
|
7
|
+
const CLOSER = /[\p{Pe}”»]/u;
|
|
8
|
+
const LINE_BREAK = /[\n\r]/u;
|
|
9
|
+
|
|
10
|
+
type Pair = { readonly open: number; readonly close: number };
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* 同じ行の中で組になった開きと閉じの位置。組にならないもの(「1)」の番号、同上の印の「“」、閉じない括弧)は含めない。
|
|
14
|
+
* 閉じない括弧を数えると、行の終わりまでの文が全部つながる。
|
|
15
|
+
*/
|
|
16
|
+
const pairsOf = (text: string): Pair[] => {
|
|
17
|
+
const pairs: Pair[] = [];
|
|
18
|
+
const opened: number[] = [];
|
|
19
|
+
text.split("").forEach((unit, index) => {
|
|
20
|
+
if (LINE_BREAK.test(unit)) opened.length = 0;
|
|
21
|
+
else if (OPENER.test(unit)) opened.push(index);
|
|
22
|
+
else if (CLOSER.test(unit)) {
|
|
23
|
+
const open = opened.pop();
|
|
24
|
+
if (open !== undefined) pairs.push({ open, close: index });
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
return pairs;
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/** 閉じ括弧・閉じ引用符で始まるか。前の行で開いた括弧や数え損ねた括弧の閉じが、句点の後で文頭に残ったもの。 */
|
|
31
|
+
export const startsWithCloser = (text: string): boolean => CLOSER.test(text.trimStart().charAt(0));
|
|
32
|
+
|
|
33
|
+
/** 各位置(0 から text.length まで)の手前に、同じ行で後から閉じる括弧・二重引用符が開いているか。 */
|
|
34
|
+
export const insideBrackets = (text: string): boolean[] => {
|
|
35
|
+
const change = Array.from({ length: text.length + 2 }, () => 0);
|
|
36
|
+
pairsOf(text).forEach((pair) => {
|
|
37
|
+
change[pair.open + 1] = (change[pair.open + 1] ?? 0) + 1;
|
|
38
|
+
change[pair.close + 1] = (change[pair.close + 1] ?? 0) - 1;
|
|
39
|
+
});
|
|
40
|
+
const depth = { value: 0 };
|
|
41
|
+
return change.slice(0, text.length + 1).map((step) => {
|
|
42
|
+
depth.value += step;
|
|
43
|
+
return depth.value > 0;
|
|
44
|
+
});
|
|
45
|
+
};
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// lang-en/src/label-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
|
|
2
|
+
import { escapeRegExp } from "./regexp.ts";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
|
|
6
|
+
* sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
|
|
7
|
+
* before splitting. The length does not change, so the spans fit the original text.
|
|
8
|
+
* A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** Built once from the lexicon; undefined when no label ends in a full stop. */
|
|
12
|
+
export type LabelStops = RegExp | undefined;
|
|
13
|
+
|
|
14
|
+
const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
|
|
15
|
+
const PLAIN_LETTER = "n";
|
|
16
|
+
|
|
17
|
+
/** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
|
|
18
|
+
export const labelStops = (labels: readonly string[]): LabelStops => {
|
|
19
|
+
const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
|
|
20
|
+
if (stems.length === 0) return undefined;
|
|
21
|
+
return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
/** The text with the full stop of every label before a number replaced by a letter. */
|
|
25
|
+
export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
|
|
26
|
+
stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { Token } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
const KATAKANA_WORD = /^[ァ-ヺー]+$/u;
|
|
4
|
+
const LONG_VOWEL = "ー";
|
|
5
|
+
|
|
6
|
+
/** A katakana noun written without a final ー, which could be a loanword whose final long vowel was dropped (コンピュータ). */
|
|
7
|
+
const mayHaveDropped = (token: Token): boolean =>
|
|
8
|
+
token.pos === "NOUN" && KATAKANA_WORD.test(token.surface) && !token.surface.endsWith(LONG_VOWEL) && Array.from(token.surface).length > 1;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Marks a katakana noun whose form with a final ー is a word the dictionary knows (コンピュータ → コンピューター): LongVowelEnding=Dropped.
|
|
12
|
+
* Only the dictionary can tell a dropped long vowel (メモリ) from a word that never had one (データ, ハードウェア).
|
|
13
|
+
* knowsWithLongVowel is given by index.ts, which holds the analyser; each surface is asked once.
|
|
14
|
+
*/
|
|
15
|
+
export const markDroppedLongVowels = (tokens: readonly Token[], knowsWithLongVowel: (surface: string) => boolean): Token[] =>
|
|
16
|
+
tokens.map((token) =>
|
|
17
|
+
mayHaveDropped(token) && knowsWithLongVowel(token.surface) ? { ...token, features: { ...token.features, LongVowelEnding: "Dropped" } } : token,
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
/** Whether text reads as one word the dictionary knows (a lemma, not an unknown run of katakana). */
|
|
21
|
+
export const knownWordReading =
|
|
22
|
+
(tokenize: (text: string) => readonly Token[] | undefined) =>
|
|
23
|
+
(surface: string): boolean => {
|
|
24
|
+
const read = tokenize(`${surface}${LONG_VOWEL}`);
|
|
25
|
+
return read?.length === 1 && read[0]?.lemma !== undefined && read[0].pos === "NOUN";
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/** Remembers each answer: a document repeats its words, and the analyser is the cost. */
|
|
29
|
+
export const remembered = (ask: (surface: string) => boolean): ((surface: string) => boolean) => {
|
|
30
|
+
const answers = new Map<string, boolean>();
|
|
31
|
+
return (surface) => {
|
|
32
|
+
const known = answers.get(surface);
|
|
33
|
+
if (known !== undefined) return known;
|
|
34
|
+
const answer = ask(surface);
|
|
35
|
+
answers.set(surface, answer);
|
|
36
|
+
return answer;
|
|
37
|
+
};
|
|
38
|
+
};
|