@chaffjs/lang-ja 0.36.0 → 0.38.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (125) hide show
  1. package/dist/english-run.d.ts.map +1 -1
  2. package/dist/english-run.js +2 -1
  3. package/dist/english-run.js.map +1 -1
  4. package/dist/index.d.ts.map +1 -1
  5. package/dist/index.js +5 -2
  6. package/dist/index.js.map +1 -1
  7. package/dist/initial-stop.d.ts +23 -0
  8. package/dist/initial-stop.d.ts.map +1 -0
  9. package/dist/initial-stop.js +33 -0
  10. package/dist/initial-stop.js.map +1 -0
  11. package/dist/list-line.d.ts.map +1 -1
  12. package/dist/list-line.js +55 -7
  13. package/dist/list-line.js.map +1 -1
  14. package/dist/pos.d.ts.map +1 -1
  15. package/dist/pos.js +7 -1
  16. package/dist/pos.js.map +1 -1
  17. package/dist/quoted-line-stop.d.ts +4 -0
  18. package/dist/quoted-line-stop.d.ts.map +1 -0
  19. package/dist/quoted-line-stop.js +18 -0
  20. package/dist/quoted-line-stop.js.map +1 -0
  21. package/dist/structure.d.ts.map +1 -1
  22. package/dist/structure.js +5 -2
  23. package/dist/structure.js.map +1 -1
  24. package/lexicons/adversative-ga-causal.yaml +11 -0
  25. package/lexicons/ai-tell.yaml +41 -20
  26. package/lexicons/alternative-clause-label.yaml +7 -0
  27. package/lexicons/announcing-opener.yaml +4 -0
  28. package/lexicons/asker-heading.yaml +14 -0
  29. package/lexicons/bracket-annotation.yaml +8 -0
  30. package/lexicons/bracket-width.yaml +6 -0
  31. package/lexicons/case-party-joiner.yaml +7 -0
  32. package/lexicons/closing-cliche.yaml +22 -1
  33. package/lexicons/closing-exhortation.yaml +23 -0
  34. package/lexicons/cold-open.yaml +30 -0
  35. package/lexicons/common-acronym.yaml +17 -2
  36. package/lexicons/comparative-baseline.yaml +16 -1
  37. package/lexicons/condition-heading.yaml +14 -0
  38. package/lexicons/contact-line-opener.yaml +16 -0
  39. package/lexicons/contrast-amount.yaml +14 -0
  40. package/lexicons/contrast-lead.yaml +5 -0
  41. package/lexicons/contrast-turn.yaml +5 -0
  42. package/lexicons/count-hedge.yaml +5 -0
  43. package/lexicons/declaration-particle.yaml +14 -0
  44. package/lexicons/defined-term-group-suffix.yaml +6 -0
  45. package/lexicons/demonstrative-not-opener.yaml +14 -0
  46. package/lexicons/email-closing.yaml +5 -3
  47. package/lexicons/email-greeting.yaml +10 -3
  48. package/lexicons/empty-intensifier.yaml +2 -0
  49. package/lexicons/everyday-name.yaml +29 -0
  50. package/lexicons/final-period.yaml +27 -0
  51. package/lexicons/fixed-name-ending.yaml +25 -0
  52. package/lexicons/headnote-heading.yaml +9 -0
  53. package/lexicons/honorific-group.yaml +9 -0
  54. package/lexicons/humble-own-act.yaml +15 -0
  55. package/lexicons/intent-ending.yaml +33 -0
  56. package/lexicons/interviewer-mark.yaml +15 -0
  57. package/lexicons/kanji-known-word.yaml +14 -0
  58. package/lexicons/licence-word.yaml +7 -0
  59. package/lexicons/link-label-ending.yaml +11 -0
  60. package/lexicons/list-conjunction.yaml +13 -0
  61. package/lexicons/list-continues.yaml +7 -0
  62. package/lexicons/list-note-lead.yaml +2 -0
  63. package/lexicons/list-suffix.yaml +6 -0
  64. package/lexicons/name-capitals-prefix.yaml +15 -0
  65. package/lexicons/name-copula.yaml +17 -0
  66. package/lexicons/name-direction-word.yaml +8 -0
  67. package/lexicons/name-title-head.yaml +23 -0
  68. package/lexicons/navigation-line.yaml +14 -0
  69. package/lexicons/necessity-purpose.yaml +13 -0
  70. package/lexicons/neutral-request.yaml +9 -0
  71. package/lexicons/neutral-wish.yaml +8 -0
  72. package/lexicons/newsletter-issue.yaml +10 -0
  73. package/lexicons/ngram-fixed-ending.yaml +22 -0
  74. package/lexicons/organisation-unit.yaml +8 -0
  75. package/lexicons/orthographic-variant-conjunction.yaml +4 -0
  76. package/lexicons/orthographic-variant-distinct.yaml +5 -0
  77. package/lexicons/orthographic-variant-idiom.yaml +3 -0
  78. package/lexicons/orthographic-variant-judging.yaml +4 -1
  79. package/lexicons/orthographic-variant-katakana-sense.yaml +8 -0
  80. package/lexicons/orthographic-variant-law-number.yaml +7 -0
  81. package/lexicons/orthographic-variant-light-verb.yaml +8 -0
  82. package/lexicons/orthographic-variant-means.yaml +9 -0
  83. package/lexicons/orthographic-variant-name-ending.yaml +10 -0
  84. package/lexicons/orthographic-variant-two-readings.yaml +14 -0
  85. package/lexicons/orthographic-variant-voice.yaml +8 -0
  86. package/lexicons/orthographic-variant-whole-word.yaml +12 -0
  87. package/lexicons/padded-intro.yaml +11 -0
  88. package/lexicons/placeholder-division.yaml +8 -0
  89. package/lexicons/placeholder-word.yaml +63 -1
  90. package/lexicons/prepositive-abbreviation.yaml +9 -0
  91. package/lexicons/product-name.yaml +9 -0
  92. package/lexicons/quote-label.yaml +16 -0
  93. package/lexicons/quoted-wording-cue.yaml +14 -0
  94. package/lexicons/reference-prefix.yaml +12 -0
  95. package/lexicons/rejected-wording.yaml +18 -0
  96. package/lexicons/relative-start.yaml +8 -0
  97. package/lexicons/request-deadline-section.yaml +12 -0
  98. package/lexicons/request-prohibition.yaml +9 -0
  99. package/lexicons/requirement-open-end.yaml +8 -0
  100. package/lexicons/sentence-start-word.yaml +59 -0
  101. package/lexicons/set-polite-phrase.yaml +13 -0
  102. package/lexicons/spelling-mention.yaml +9 -0
  103. package/lexicons/statute-article-opener.yaml +6 -0
  104. package/lexicons/superlative-goal.yaml +16 -0
  105. package/lexicons/superlative-owner.yaml +6 -0
  106. package/lexicons/superlative-phrase.yaml +18 -0
  107. package/lexicons/superlative-realm.yaml +14 -0
  108. package/lexicons/superlative-viewpoint.yaml +6 -0
  109. package/lexicons/takeup-opener.yaml +13 -0
  110. package/lexicons/telling-cue.yaml +12 -0
  111. package/lexicons/template-heading.yaml +32 -0
  112. package/lexicons/term-source-marker.yaml +11 -0
  113. package/lexicons/text-message-verb.yaml +11 -0
  114. package/lexicons/title-ending.yaml +18 -1
  115. package/lexicons/topic-clause-close.yaml +11 -0
  116. package/lexicons/unnumbered-heading.yaml +3 -0
  117. package/lexicons/unprompted-denial.yaml +99 -0
  118. package/package.json +1 -1
  119. package/src/english-run.ts +2 -1
  120. package/src/index.ts +8 -2
  121. package/src/initial-stop.ts +58 -0
  122. package/src/list-line.ts +66 -8
  123. package/src/pos.ts +14 -1
  124. package/src/quoted-line-stop.ts +20 -0
  125. package/src/structure.ts +5 -2
package/src/list-line.ts CHANGED
@@ -9,6 +9,7 @@ import { escapeRegExp } from "./regexp.ts";
9
9
  * 印の無い行の折り返しは切らない。
10
10
  *
11
11
  * 番号だけの札の行(「第1」「第3条」)と、行頭の括弧で囲んだ札(「(別添)…」)も、印で始まる行と同じに切る。札の語は辞書から取る。
12
+ * URL だけの行、欄の行(電話:…、〒…)、括弧の語だけの行も、文の途中に立っていなければ前後で切る(メールマガジンや連絡先の欄)。
12
13
  */
13
14
 
14
15
  const SPACE = "[ \\t\\u3000]";
@@ -87,18 +88,75 @@ export const lineVocabulary = (counters: readonly string[], tags: readonly strin
87
88
  const isListLine = (line: string, vocabulary: LineVocabulary): boolean =>
88
89
  startsWithListMarker(line) || vocabulary.label.test(line) || (vocabulary.tag?.test(line) ?? false);
89
90
 
90
- /** 前の行と後ろの行の間で文が改まるか: どちらかが項目の行か、注記の行が続き、前の行が次へ続いていない。 */
91
- const isCut = (previous: Line, next: Line, vocabulary: LineVocabulary): boolean =>
92
- next.text.trim().length > 0 &&
93
- !continuesToNextLine(previous.text) &&
94
- (isListLine(next.text, vocabulary) || isListLine(previous.text, vocabulary) || (NOTE.test(previous.text) && NOTE.test(next.text)));
91
+ /** URL だけの行(▼こちら の次の行のアドレス)。 */
92
+ const URL_LINE = new RegExp(`^${SPACE}*https?://\\S+${SPACE}*$`, "u");
93
+ /**
94
+ * 欄の行: 短い札と全角か半角のコロンで始まる行(電話:03-…、E-mail:…)と、郵便番号の印〒で始まる行。連絡先の欄は 1 行 1 項目。
95
+ * 札が数字で終われば時刻(10:00、午前10:00)か比(1:2)で、札ではない。
96
+ */
97
+ const FIELD_LINE = new RegExp(`^${SPACE}*(?:〒|[^\\s、。,::「」『』()()【】]{0,11}[^\\s、。,::「」『』()()【】0-90-9][::])`, "u");
98
+ /** 括弧でくくった語だけの行((お問い合わせ先)、【新着】(文部科学省)、「報告書は実践の宝庫」)。札か題で、文の途中ではない。 */
99
+ const BRACKET_GROUP = "[((【〔[「『][^((【〔[「『))】〕]」』\\n]*[))】〕]」』]";
100
+ const BRACKET_LINE = new RegExp(`^(?:${SPACE}*${BRACKET_GROUP})+${SPACE}*$`, "u");
101
+ /** 空白の後の括弧の注記で終わる行(題 (○○小学校))。題の行の終わりで、文は折り返しても括弧の前に空白を置かない。 */
102
+ const NOTE_IN_BRACKETS_END = new RegExp(`\\S${SPACE}+[((][^(())\\n]*[))]${SPACE}*$`, "u");
103
+
104
+ /** 行が助詞で始まる: 前の行の語を受けて文が続いている(「…」\nと述べた)。 */
105
+ const PARTICLE_START = new RegExp(`^${SPACE}*(?:の|に|を|が|は|と|で|へ|も|から|まで|より|及び|又は|並びに|若しくは)`, "u");
106
+ /** 行が助詞で終わる: 文が次の行へ続いている(担当者は\n「…」)。 */
107
+ const PARTICLE_END = new RegExp(`(?:の|に|を|が|は|と|で|へ|も|から|まで|より)${SPACE}*$`, "u");
108
+
109
+ /** 1 行で一つの単位になりうる行: URL、欄、括弧の語だけ。 */
110
+ const standsAlone = (line: string): boolean => URL_LINE.test(line) || FIELD_LINE.test(line) || BRACKET_LINE.test(line);
111
+
112
+ /** 空白だけの行。段落の中では、伏せた URL やコードが 1 行を占めていた所。 */
113
+ const isMaskedLine = (line: string): boolean => line.trim() === "";
114
+
115
+ /**
116
+ * 行が文の途中に立っていない: 前の行が読点・開いた括弧・助詞で終わらず、次の行が助詞で始まらない。
117
+ * 折り返した文の途中に括弧の語だけの行が来ることがある(〜始まり、\n(休憩を挟んで)\n夕方まで続く。)。
118
+ */
119
+ const standsBetween = (before: Line | undefined, after: Line | undefined): boolean =>
120
+ (before === undefined || (!continuesToNextLine(before.text) && !PARTICLE_END.test(before.text))) && (after === undefined || !PARTICLE_START.test(after.text));
121
+
122
+ /** 1 行で一つの単位の行。その前でも後ろでも文が改まる。 */
123
+ const isUnitLine = (lines: readonly Line[], index: number): boolean =>
124
+ standsAlone(lines[index]?.text ?? "") && standsBetween(lines[index - 1], lines[index + 1]);
125
+
126
+ /**
127
+ * index の行の後ろで文が改まる: 1 行で一つの単位の行、題の注記で終わる行、伏せた行の後ろ。
128
+ * 次の行が注記(※)なら、その行に添える注記なので続ける。
129
+ */
130
+ const endsUnitAt = (lines: readonly Line[], index: number): boolean => {
131
+ const line = lines[index]?.text ?? "";
132
+ const ends = isUnitLine(lines, index) || ((NOTE_IN_BRACKETS_END.test(line) || isMaskedLine(line)) && standsBetween(lines[index - 1], lines[index + 1]));
133
+ return ends && !NOTE.test(lines[index + 1]?.text ?? "");
134
+ };
135
+
136
+ /**
137
+ * index - 1 の行と index の行の間で文が改まるか: どちらかが項目の行か、1 行で一つの単位の行か、注記の行が続く。
138
+ * 前の行が次へ続いている(読点、開いた括弧)なら改まらない。
139
+ */
140
+ const isCut = (lines: readonly Line[], index: number, vocabulary: LineVocabulary): boolean => {
141
+ const previous = lines[index - 1]?.text ?? "";
142
+ const next = lines[index]?.text ?? "";
143
+ return (
144
+ next.trim().length > 0 &&
145
+ !continuesToNextLine(previous) &&
146
+ (isListLine(next, vocabulary) ||
147
+ isListLine(previous, vocabulary) ||
148
+ isUnitLine(lines, index) ||
149
+ endsUnitAt(lines, index - 1) ||
150
+ (NOTE.test(previous) && NOTE.test(next)))
151
+ );
152
+ };
95
153
 
96
154
  /** 文を改める改行の位置: 前の行の終わり(end)と、次の行の最初の文字(start)。 */
97
155
  export const listLineBreaks = (text: string, vocabulary: LineVocabulary): Span[] => {
98
156
  const lines = linesOf(text);
99
- return lines.slice(1).flatMap((next, index) => {
100
- const previous = lines[index];
101
- if (previous === undefined || !isCut(previous, next, vocabulary)) return [];
157
+ return lines.slice(1).flatMap((next, offset) => {
158
+ const previous = lines[offset];
159
+ if (previous === undefined || !isCut(lines, offset + 1, vocabulary)) return [];
102
160
  return [{ start: previous.end, end: next.start + (next.text.length - next.text.trimStart().length) }];
103
161
  });
104
162
  };
package/src/pos.ts CHANGED
@@ -161,7 +161,10 @@ const toToken = (morpheme: Morpheme, start: number, { passive, echo, light, raDr
161
161
  pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
162
162
  ...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
163
163
  ...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
164
- ...withRaDropped(withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo), raDropped),
164
+ ...withAdverbial(
165
+ withRaDropped(withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo), raDropped),
166
+ isAdverbialNoun(morpheme),
167
+ ),
165
168
  });
166
169
 
167
170
  /**
@@ -195,6 +198,16 @@ const withEcho = (found: { features?: Readonly<Record<string, string>> }, echo:
195
198
  const withRaDropped = (found: { features?: Readonly<Record<string, string>> }, raDropped: boolean): { features?: Readonly<Record<string, string>> } =>
196
199
  raDropped ? { features: { ...found.features, PotentialRa: "Dropped" } } : found;
197
200
 
201
+ /**
202
+ * 副詞としても使える名詞(IPADIC の「副詞可能」: 今夜・当日・午前、その日の「日」、午前中の「中」)。今夜は来る、のように助詞なしで
203
+ * 時や量を言える。日本語に固有の印で、UD の FEATS の言語別拡張。非自立名詞や接尾の印はそのまま残す。
204
+ */
205
+ const isAdverbialNoun = (morpheme: Morpheme): boolean =>
206
+ morpheme.pos === "名詞" && (morpheme.pos_detail_1 === "副詞可能" || morpheme.pos_detail_2 === "副詞可能");
207
+
208
+ const withAdverbial = (found: { features?: Readonly<Record<string, string>> }, adverbial: boolean): { features?: Readonly<Record<string, string>> } =>
209
+ adverbial ? { features: { ...found.features, Adverbial: "Yes" } } : found;
210
+
198
211
  /**
199
212
  * 数(名詞,数)。UPOS では名詞に寄せるので、数であることは UD の NumType=Card で渡す。
200
213
  * 決まった言い回し(二人三脚、三日坊主、一人ひとり、十分)は辞書が一語として持つので、数にはならない。
@@ -0,0 +1,20 @@
1
+ import type { Span } from "chaffjs/plugin";
2
+
3
+ /**
4
+ * 句点で閉じた引用が行の終わりにあるとき(「…こたえる。」の後で改行)、分割器は括弧の組の後ろで文を切らず、次の行の文まで
5
+ * 一つの文にする。演説や宣誓の引用を一行ずつ置いた文書では、次の行は別の文である。
6
+ * 文末の記号 + 閉じ括弧 + 改行で切る。次の行が「と」や読点で始まるもの(「…。」\nと述べた)は引用が文の途中にあるので切らない。
7
+ */
8
+ const QUOTE_AT_LINE_END = /[。.!?!?][」』))】〉》]\]]+[ \t\u3000]*(?=\r?\n)/gu;
9
+ const CONTINUES = /^\s*[と、,。」』))]/u;
10
+
11
+ /** span の文を、行の終わりで閉じた引用ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
12
+ export const splitAtQuotedLineEnds = (text: string, span: Span): Span[] => {
13
+ const sentence = text.slice(span.start, span.end);
14
+ const cuts = [...sentence.matchAll(QUOTE_AT_LINE_END)]
15
+ .map((match) => match.index + match[0].length)
16
+ .filter((end) => !CONTINUES.test(sentence.slice(end)) && sentence.slice(end).trim() !== "");
17
+ const starts = [0, ...cuts.map((end) => end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0))];
18
+ const ends = [...cuts, sentence.length];
19
+ return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
20
+ };
package/src/structure.ts CHANGED
@@ -12,8 +12,11 @@ import { deniesSaying } from "./denied-saying.ts";
12
12
  const NUMBER = "[0-90-9〇一二三四五六七八九十百千]{1,6}";
13
13
  const SPACE = "[ \\t\\u3000]";
14
14
 
15
- /** 見出しとして読めるのは、番号の直後が空白・括弧・行末のとき。「第3条に定める」は本文。 */
16
- const ARTICLE = new RegExp(`^${SPACE}*第(?<n>${NUMBER})条(?:の(?<sub>${NUMBER}))?(?<rest>(?:${SPACE}|(|\\().*|)$`, "u");
15
+ /**
16
+ * 見出しとして読めるのは、番号の直後が空白・括弧・行末のとき。「第3条に定める」は本文。
17
+ * ひな形が選択肢ごとに項を見出しにした「第1条1項(オプション1:…)」は第1条の見出し。項の後ろは括弧に限る(「第3条第2項中」は改める文)。
18
+ */
19
+ const ARTICLE = new RegExp(`^${SPACE}*第(?<n>${NUMBER})条(?:の(?<sub>${NUMBER}))?(?:第?${NUMBER}項(?=[((]))?(?<rest>(?:${SPACE}|(|\\().*|)$`, "u");
17
20
  /**
18
21
  * 見出しの行では、題を詰めて書いた「第7条委託」「第2章概要」も読む。後ろが平仮名(「第3条に定める」「第2章では」)、
19
22
  * 次の番号(「第2章第1節」)、並べる語(「第4条及び第5条」「第2条若しくは第3条」「第2条乃至第5条」)なら、番号は題の札でないので読まない。