@chaffjs/lang-ja 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/README.md +1 -1
  2. package/dist/citation.d.ts +6 -3
  3. package/dist/citation.d.ts.map +1 -1
  4. package/dist/citation.js +31 -9
  5. package/dist/citation.js.map +1 -1
  6. package/dist/closing-quote.d.ts +6 -0
  7. package/dist/closing-quote.d.ts.map +1 -0
  8. package/dist/closing-quote.js +50 -0
  9. package/dist/closing-quote.js.map +1 -0
  10. package/dist/english-run.d.ts +8 -0
  11. package/dist/english-run.d.ts.map +1 -0
  12. package/dist/english-run.js +24 -0
  13. package/dist/english-run.js.map +1 -0
  14. package/dist/index.d.ts.map +1 -1
  15. package/dist/index.js +56 -21
  16. package/dist/index.js.map +1 -1
  17. package/dist/inside-brackets.d.ts +5 -0
  18. package/dist/inside-brackets.d.ts.map +1 -0
  19. package/dist/inside-brackets.js +44 -0
  20. package/dist/inside-brackets.js.map +1 -0
  21. package/dist/label-stop.d.ts +13 -0
  22. package/dist/label-stop.d.ts.map +1 -0
  23. package/dist/label-stop.js +14 -0
  24. package/dist/label-stop.js.map +1 -0
  25. package/dist/lexicons.d.ts.map +1 -1
  26. package/dist/lexicons.js +4 -0
  27. package/dist/lexicons.js.map +1 -1
  28. package/dist/long-vowel-form.d.ts +12 -0
  29. package/dist/long-vowel-form.d.ts.map +1 -0
  30. package/dist/long-vowel-form.js +28 -0
  31. package/dist/long-vowel-form.js.map +1 -0
  32. package/dist/number-stop.d.ts +3 -0
  33. package/dist/number-stop.d.ts.map +1 -0
  34. package/dist/number-stop.js +25 -0
  35. package/dist/number-stop.js.map +1 -0
  36. package/dist/pos.d.ts +17 -1
  37. package/dist/pos.d.ts.map +1 -1
  38. package/dist/pos.js +69 -36
  39. package/dist/pos.js.map +1 -1
  40. package/dist/predicate-frame.d.ts +14 -0
  41. package/dist/predicate-frame.d.ts.map +1 -0
  42. package/dist/predicate-frame.js +22 -0
  43. package/dist/predicate-frame.js.map +1 -0
  44. package/dist/quantities.d.ts.map +1 -1
  45. package/dist/quantities.js +11 -1
  46. package/dist/quantities.js.map +1 -1
  47. package/dist/quoted-stop.d.ts +13 -0
  48. package/dist/quoted-stop.d.ts.map +1 -0
  49. package/dist/quoted-stop.js +45 -0
  50. package/dist/quoted-stop.js.map +1 -0
  51. package/dist/ra-dropped.d.ts +12 -0
  52. package/dist/ra-dropped.d.ts.map +1 -0
  53. package/dist/ra-dropped.js +18 -0
  54. package/dist/ra-dropped.js.map +1 -0
  55. package/dist/reduplication.d.ts +17 -5
  56. package/dist/reduplication.d.ts.map +1 -1
  57. package/dist/reduplication.js +85 -6
  58. package/dist/reduplication.js.map +1 -1
  59. package/dist/regexp.d.ts +3 -0
  60. package/dist/regexp.d.ts.map +1 -0
  61. package/dist/regexp.js +4 -0
  62. package/dist/regexp.js.map +1 -0
  63. package/dist/sentence-close.d.ts +2 -0
  64. package/dist/sentence-close.d.ts.map +1 -0
  65. package/dist/sentence-close.js +10 -0
  66. package/dist/sentence-close.js.map +1 -0
  67. package/dist/sentence-split.d.ts +2 -0
  68. package/dist/sentence-split.d.ts.map +1 -1
  69. package/dist/sentence-split.js +3 -1
  70. package/dist/sentence-split.js.map +1 -1
  71. package/dist/structure.d.ts.map +1 -1
  72. package/dist/structure.js +60 -21
  73. package/dist/structure.js.map +1 -1
  74. package/dist/well-formed.d.ts +4 -1
  75. package/dist/well-formed.d.ts.map +1 -1
  76. package/dist/well-formed.js +5 -2
  77. package/dist/well-formed.js.map +1 -1
  78. package/lexicons/abbreviated-label.yaml +16 -0
  79. package/lexicons/ai-tell.yaml +105 -0
  80. package/lexicons/announcing-opener.yaml +11 -0
  81. package/lexicons/assistant-residue.yaml +37 -0
  82. package/lexicons/calendar-era.yaml +10 -0
  83. package/lexicons/closing-cliche.yaml +2 -0
  84. package/lexicons/comparison-marker.yaml +7 -0
  85. package/lexicons/contrast-frame.yaml +9 -0
  86. package/lexicons/contrast-lead.yaml +10 -0
  87. package/lexicons/contrast-turn.yaml +9 -0
  88. package/lexicons/count-anchor.yaml +10 -0
  89. package/lexicons/count-counter.yaml +17 -0
  90. package/lexicons/count-hedge.yaml +61 -0
  91. package/lexicons/count-number.yaml +16 -0
  92. package/lexicons/distributive-noun.yaml +14 -2
  93. package/lexicons/double-negative.yaml +32 -0
  94. package/lexicons/doubled-word.yaml +12 -0
  95. package/lexicons/email-attachment-note.yaml +8 -0
  96. package/lexicons/email-attribution.yaml +10 -0
  97. package/lexicons/email-header-field.yaml +37 -0
  98. package/lexicons/email-written-field.yaml +11 -0
  99. package/lexicons/enumeration-frame.yaml +32 -0
  100. package/lexicons/enumeration-joiner.yaml +15 -0
  101. package/lexicons/enumeration-negation.yaml +10 -0
  102. package/lexicons/figure-label.yaml +11 -0
  103. package/lexicons/item-number.yaml +38 -0
  104. package/lexicons/long-vowel-distinct.yaml +25 -0
  105. package/lexicons/long-vowel-drop-after.yaml +25 -0
  106. package/lexicons/measure-unit.yaml +78 -0
  107. package/lexicons/name-note.yaml +5 -1
  108. package/lexicons/neutral-ending.yaml +10 -0
  109. package/lexicons/nominalization-event-noun.yaml +13 -0
  110. package/lexicons/nominalization-light-verb.yaml +10 -0
  111. package/lexicons/numbered-label.yaml +50 -0
  112. package/lexicons/passive-predicate-frame.yaml +13 -0
  113. package/lexicons/percent-unit.yaml +7 -0
  114. package/lexicons/placeholder-word.yaml +21 -0
  115. package/lexicons/ra-dropped-verb.yaml +7 -0
  116. package/lexicons/range-closer.yaml +5 -0
  117. package/lexicons/range-connector.yaml +11 -0
  118. package/lexicons/range-opener.yaml +7 -0
  119. package/lexicons/requirement-either.yaml +13 -0
  120. package/lexicons/requirement-loophole.yaml +13 -0
  121. package/lexicons/requirement-marker.yaml +15 -0
  122. package/lexicons/requirement-modal-jis-z8301-2019.yaml +14 -0
  123. package/lexicons/requirement-not-marker.yaml +8 -0
  124. package/lexicons/requirement-open-end.yaml +6 -0
  125. package/lexicons/sasete-itadaku.yaml +11 -0
  126. package/lexicons/share-exception.yaml +7 -0
  127. package/lexicons/share-label.yaml +11 -0
  128. package/lexicons/stock-transition.yaml +11 -0
  129. package/lexicons/vague-clause-pointer.yaml +14 -0
  130. package/lexicons/vague-figure-pointer.yaml +23 -0
  131. package/package.json +1 -1
  132. package/src/citation.ts +35 -10
  133. package/src/closing-quote.ts +53 -0
  134. package/src/english-run.ts +25 -0
  135. package/src/index.ts +54 -13
  136. package/src/inside-brackets.ts +45 -0
  137. package/src/label-stop.ts +26 -0
  138. package/src/lexicons.ts +4 -0
  139. package/src/long-vowel-form.ts +38 -0
  140. package/src/number-stop.ts +30 -0
  141. package/src/pos.ts +78 -29
  142. package/src/predicate-frame.ts +28 -0
  143. package/src/quantities.ts +12 -1
  144. package/src/quoted-stop.ts +53 -0
  145. package/src/ra-dropped.ts +29 -0
  146. package/src/reduplication.ts +119 -8
  147. package/src/regexp.ts +3 -0
  148. package/src/sentence-close.ts +10 -0
  149. package/src/sentence-split.ts +3 -1
  150. package/src/structure.ts +62 -19
  151. package/src/well-formed.ts +5 -2
package/src/index.ts CHANGED
@@ -1,16 +1,27 @@
1
1
  import { loadLexicons } from "./lexicons.ts";
2
+ import { closesSentence } from "./sentence-close.ts";
2
3
  import { sentenceSpans } from "./sentence-split.ts";
4
+ import { isEnglishBoundary, isEnglishRun } from "./english-run.ts";
5
+ import { unmarkNumberStops } from "./number-stop.ts";
6
+ import { labelStops, unmarkLabelStops } from "./label-stop.ts";
7
+ import { splitAtQuotedStops } from "./quoted-stop.ts";
8
+ import { reattachClosingQuotes } from "./closing-quote.ts";
9
+ import { insideBrackets, startsWithCloser } from "./inside-brackets.ts";
3
10
  import { structure } from "./structure.ts";
4
11
  import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.ts";
5
12
  import { markSpacedCounters } from "./spaced-counter.ts";
6
13
  import { tokensWithin } from "./tokens-within.ts";
7
14
  import { distributiveVocabulary, iterationMarkReading, markReduplication } from "./reduplication.ts";
8
- import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
15
+ import { knownWordReading, markDroppedLongVowels, remembered } from "./long-vowel-form.ts";
16
+ import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
9
17
 
10
18
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
11
19
 
20
+ const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
21
+ const COUNTABLE = /\S/gu;
22
+
12
23
  /**
13
- * 日本語の文末は「。!?」に限られる。
24
+ * 日本語の文末は「。!?」と、語の後の「.」に限られる(closesSentence)。
14
25
  *
15
26
  * sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
16
27
  * AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
@@ -19,30 +30,42 @@ import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from
19
30
  * 文長は sentence-rhythm と max-sentence-length の入力なので、
20
31
  * 誤分割はそのまま指標を動かす。ここで閉じる。
21
32
  */
22
- const CLOSED = /[。!?!?]["))」』]*\s*$/u;
33
+ const isOpen = (text: string): boolean => text.trim().length > 0 && !closesSentence(text);
23
34
 
24
- const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
25
- const COUNTABLE = /\S/gu;
35
+ /**
36
+ * 和文の句点で閉じていない断片、括弧・二重引用符の中で切れた断片、閉じ括弧で始まる断片の前は次へ続く。
37
+ * 閉じ括弧は閉じる文に付き、後ろは分割器が組にした括弧(「…。」と言った)と同じく同じ文に続く。
38
+ * ただし英文どうしは、英語のアダプタと同じく英語の文末で切ったところで切る。
39
+ */
40
+ const continues = (before: string, after: string, bracketOpen: boolean): boolean =>
41
+ (isOpen(before) || bracketOpen || startsWithCloser(after)) && !isEnglishBoundary(before, after);
42
+
43
+ const LINE_BREAK = /[\n\r]/u;
26
44
 
27
- const isOpen = (text: string): boolean => text.trim().length > 0 && !CLOSED.test(text);
45
+ const ENGLISH: EmbeddedLanguage = { id: "en", lengthUnit: "word" };
46
+
47
+ const withLanguage = (text: string, span: Span): Sentence => (isEnglishRun(text) ? { span, text, embeddedLanguage: ENGLISH } : { span, text });
28
48
 
29
49
  /**
30
50
  * 断片を繋ぐときは raw の連結ではなくオフセットを使う。
31
51
  * sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
32
52
  */
33
- const merge = (source: string, spans: readonly Span[]): Sentence[] =>
34
- spans
53
+ const merge = (source: string, spans: readonly Span[]): Sentence[] => {
54
+ const open = insideBrackets(source);
55
+ return spans
35
56
  .reduce<Span[]>((acc, span) => {
36
57
  const last = acc.at(-1);
37
58
  // 断片の「間」に改行があれば繋がない。結合は「Dr. 田中」のような行の途中の
38
59
  // 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
39
60
  // 英文と訳文のように別の行のものまで 1 文に繋がる。
40
- const acrossLines = last !== undefined && source.slice(last.end, span.start).includes("\n");
41
- if (last !== undefined && !acrossLines && isOpen(source.slice(last.start, last.end))) acc[acc.length - 1] = { start: last.start, end: span.end };
61
+ const acrossLines = last !== undefined && LINE_BREAK.test(source.slice(last.end, span.start));
62
+ if (last !== undefined && !acrossLines && continues(source.slice(last.start, last.end), source.slice(span.start, span.end), open[last.end] === true))
63
+ acc[acc.length - 1] = { start: last.start, end: span.end };
42
64
  else acc.push(span);
43
65
  return acc;
44
66
  }, [])
45
- .map((span) => ({ span, text: source.slice(span.start, span.end) }));
67
+ .map((span) => withLanguage(source.slice(span.start, span.end), span));
68
+ };
46
69
 
47
70
  /**
48
71
  * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
@@ -51,11 +74,15 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
51
74
  const LEXICONS = loadLexicons();
52
75
  const DISTRIBUTIVE = distributiveVocabulary(LEXICONS);
53
76
  const TAKES_ITERATION_MARK = iterationMarkReading(LEXICONS, readsAsOneWord);
77
+ const KNOWS_WITH_LONG_VOWEL = remembered(knownWordReading(tokenize));
78
+ /** Whether a running rule reads LongVowelEnding. Asking the dictionary about every katakana word costs, so only then. */
79
+ const askForLongVowels = { value: false };
54
80
 
55
81
  const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
56
82
  const read = tokenize(source);
57
83
  if (read === undefined) return [...sentences];
58
- const tokens = markSpacedCounters(read, readsAsCounter);
84
+ const counted = markSpacedCounters(read, readsAsCounter);
85
+ const tokens = askForLongVowels.value ? markDroppedLongVowels(counted, KNOWS_WITH_LONG_VOWEL) : counted;
59
86
  return sentences.map((sentence) => ({
60
87
  ...sentence,
61
88
  // 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
@@ -63,6 +90,18 @@ const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[]
63
90
  }));
64
91
  };
65
92
 
93
+ const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
94
+
95
+ /**
96
+ * 英語のアダプタと同じ手当てを英文にも当てる。番号の前の略した名前(FIG. 1)の点で切らず、
97
+ * 英文の閉じ引用符の内側の句点で切り、文頭に取り残された閉じ引用符を前の文へ戻す。
98
+ */
99
+ const spansOf = (text: string): Span[] => {
100
+ const spans = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS)));
101
+ const quotedStops = spans.flatMap((span) => (isEnglishRun(text.slice(span.start, span.end)) ? splitAtQuotedStops(text, span) : [span]));
102
+ return reattachClosingQuotes(text, quotedStops);
103
+ };
104
+
66
105
  export const adapter: LanguageAdapter = {
67
106
  kind: "language",
68
107
  id: "ja",
@@ -76,6 +115,8 @@ export const adapter: LanguageAdapter = {
76
115
  lengthUnit: "char",
77
116
  },
78
117
  prepare: async (need: AdapterNeeds): Promise<void> => {
118
+ // Once asked, keep marking: files are prepared in parallel, and a later prepare must not unmark an earlier file's run.
119
+ askForLongVowels.value ||= need.features?.includes("LongVowelEnding") ?? false;
79
120
  if (need.pos) await prepare();
80
121
  },
81
122
  detect: (source: string): number => {
@@ -86,7 +127,7 @@ export const adapter: LanguageAdapter = {
86
127
  lexicons: loadLexicons(),
87
128
  structure,
88
129
  segment: (text: string): Segmentation => {
89
- const sentences = merge(text, sentenceSpans(text));
130
+ const sentences = merge(text, spansOf(text));
90
131
  return { sentences: isReady() ? withTokens(text, sentences) : sentences };
91
132
  },
92
133
  };
@@ -0,0 +1,45 @@
1
+ /**
2
+ * 括弧・二重引用符の中の句点は文を閉じない。sentence-splitter は括弧を種類ごとに一つしか数えないので、
3
+ * 同じ種類が入れ子になると内側の閉じ括弧で外側まで閉じたと読み(「(…(…)…。以下同じ。)に」)、曲がった二重引用符(“…”)は数えない。
4
+ * ここでは開きと閉じを入れ子のまま組にする。一重引用符(‘ ’)はアポストロフィと見分けられず、直線の引用符は分割器が数えるので、どちらも数えない。
5
+ */
6
+ const OPENER = /[\p{Ps}“«]/u;
7
+ const CLOSER = /[\p{Pe}”»]/u;
8
+ const LINE_BREAK = /[\n\r]/u;
9
+
10
+ type Pair = { readonly open: number; readonly close: number };
11
+
12
+ /**
13
+ * 同じ行の中で組になった開きと閉じの位置。組にならないもの(「1)」の番号、同上の印の「“」、閉じない括弧)は含めない。
14
+ * 閉じない括弧を数えると、行の終わりまでの文が全部つながる。
15
+ */
16
+ const pairsOf = (text: string): Pair[] => {
17
+ const pairs: Pair[] = [];
18
+ const opened: number[] = [];
19
+ text.split("").forEach((unit, index) => {
20
+ if (LINE_BREAK.test(unit)) opened.length = 0;
21
+ else if (OPENER.test(unit)) opened.push(index);
22
+ else if (CLOSER.test(unit)) {
23
+ const open = opened.pop();
24
+ if (open !== undefined) pairs.push({ open, close: index });
25
+ }
26
+ });
27
+ return pairs;
28
+ };
29
+
30
+ /** 閉じ括弧・閉じ引用符で始まるか。前の行で開いた括弧や数え損ねた括弧の閉じが、句点の後で文頭に残ったもの。 */
31
+ export const startsWithCloser = (text: string): boolean => CLOSER.test(text.trimStart().charAt(0));
32
+
33
+ /** 各位置(0 から text.length まで)の手前に、同じ行で後から閉じる括弧・二重引用符が開いているか。 */
34
+ export const insideBrackets = (text: string): boolean[] => {
35
+ const change = Array.from({ length: text.length + 2 }, () => 0);
36
+ pairsOf(text).forEach((pair) => {
37
+ change[pair.open + 1] = (change[pair.open + 1] ?? 0) + 1;
38
+ change[pair.close + 1] = (change[pair.close + 1] ?? 0) - 1;
39
+ });
40
+ const depth = { value: 0 };
41
+ return change.slice(0, text.length + 1).map((step) => {
42
+ depth.value += step;
43
+ return depth.value > 0;
44
+ });
45
+ };
@@ -0,0 +1,26 @@
1
+ // lang-en/src/label-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ import { escapeRegExp } from "./regexp.ts";
3
+
4
+ /**
5
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
6
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
7
+ * before splitting. The length does not change, so the spans fit the original text.
8
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
9
+ */
10
+
11
+ /** Built once from the lexicon; undefined when no label ends in a full stop. */
12
+ export type LabelStops = RegExp | undefined;
13
+
14
+ const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
15
+ const PLAIN_LETTER = "n";
16
+
17
+ /** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
18
+ export const labelStops = (labels: readonly string[]): LabelStops => {
19
+ const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
20
+ if (stems.length === 0) return undefined;
21
+ return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
22
+ };
23
+
24
+ /** The text with the full stop of every label before a number replaced by a letter. */
25
+ export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
26
+ stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);
package/src/lexicons.ts CHANGED
@@ -14,11 +14,15 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
14
14
  if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
15
15
  const weight = raw["weight"];
16
16
  const instead = raw["instead_of"];
17
+ const rewrite = raw["rewrite"];
18
+ const group = raw["group"];
17
19
  return {
18
20
  pattern: raw["pattern"],
19
21
  weight: typeof weight === "number" ? weight : undefined,
20
22
  instead_of: typeof instead === "string" ? instead : undefined,
21
23
  position: POSITIONS.find((position) => position === raw["position"]),
24
+ rewrite: typeof rewrite === "string" ? rewrite : undefined,
25
+ ...(typeof group === "string" ? { group } : {}),
22
26
  };
23
27
  };
24
28
 
@@ -0,0 +1,38 @@
1
+ import type { Token } from "chaffjs/plugin";
2
+
3
+ const KATAKANA_WORD = /^[ァ-ヺー]+$/u;
4
+ const LONG_VOWEL = "ー";
5
+
6
+ /** A katakana noun written without a final ー, which could be a loanword whose final long vowel was dropped (コンピュータ). */
7
+ const mayHaveDropped = (token: Token): boolean =>
8
+ token.pos === "NOUN" && KATAKANA_WORD.test(token.surface) && !token.surface.endsWith(LONG_VOWEL) && Array.from(token.surface).length > 1;
9
+
10
+ /**
11
+ * Marks a katakana noun whose form with a final ー is a word the dictionary knows (コンピュータ → コンピューター): LongVowelEnding=Dropped.
12
+ * Only the dictionary can tell a dropped long vowel (メモリ) from a word that never had one (データ, ハードウェア).
13
+ * knowsWithLongVowel is given by index.ts, which holds the analyser; each surface is asked once.
14
+ */
15
+ export const markDroppedLongVowels = (tokens: readonly Token[], knowsWithLongVowel: (surface: string) => boolean): Token[] =>
16
+ tokens.map((token) =>
17
+ mayHaveDropped(token) && knowsWithLongVowel(token.surface) ? { ...token, features: { ...token.features, LongVowelEnding: "Dropped" } } : token,
18
+ );
19
+
20
+ /** Whether text reads as one word the dictionary knows (a lemma, not an unknown run of katakana). */
21
+ export const knownWordReading =
22
+ (tokenize: (text: string) => readonly Token[] | undefined) =>
23
+ (surface: string): boolean => {
24
+ const read = tokenize(`${surface}${LONG_VOWEL}`);
25
+ return read?.length === 1 && read[0]?.lemma !== undefined && read[0].pos === "NOUN";
26
+ };
27
+
28
+ /** Remembers each answer: a document repeats its words, and the analyser is the cost. */
29
+ export const remembered = (ask: (surface: string) => boolean): ((surface: string) => boolean) => {
30
+ const answers = new Map<string, boolean>();
31
+ return (surface) => {
32
+ const known = answers.get(surface);
33
+ if (known !== undefined) return known;
34
+ const answer = ask(surface);
35
+ answers.set(surface, answer);
36
+ return answer;
37
+ };
38
+ };
@@ -0,0 +1,30 @@
1
+ /**
2
+ * lang-en/src/number-stop.ts の写し。和文の中の英文も同じ分割器で切るので同じ手当てが要る。アダプタは互いに依存しないので写しを置く。
3
+ *
4
+ * sentence-splitter は空白の後の「数字だけ + ピリオド」を、どこにあっても箇条書きの番号と読み、そこで文を切らない。
5
+ * 「in the spring of 2026. We shipped」が 1 文になる。
6
+ *
7
+ * 行の途中にあり、次の語が大文字で始まる番号は文末である。その数字だけを同じ長さの英字に替えて分割器に渡し、
8
+ * 普通の語として読ませる。長さが変わらないので、返る span は元の文字列にそのまま当てはまる。
9
+ * 行頭の番号(「1. First」)と、次が大文字でない番号は替えない。
10
+ */
11
+ const NUMBER_WORD = /(?<=\s)\d+(?=\.\s)/g;
12
+ // 次の語の頭。開き括弧・引用符の後の大文字も文頭である。
13
+ const CAPITAL_AFTER_SPACE = /\s+[\p{Ps}\p{Pi}"']*\p{Lu}/uy;
14
+ // 分割器の略語表に n だけでできた語は無い。数字の代わりに置いても、略語として読まれない。
15
+ const PLAIN_LETTER = "n";
16
+
17
+ const lineStartOf = (text: string, index: number): number => Math.max(text.lastIndexOf("\n", index - 1), text.lastIndexOf("\r", index - 1)) + 1;
18
+
19
+ const isFirstOnLine = (text: string, index: number): boolean => text.slice(lineStartOf(text, index), index).trim() === "";
20
+
21
+ const capitalFollows = (text: string, dot: number): boolean => {
22
+ CAPITAL_AFTER_SPACE.lastIndex = dot + 1;
23
+ return CAPITAL_AFTER_SPACE.test(text);
24
+ };
25
+
26
+ const isSentenceEnd = (text: string, index: number, digits: string): boolean => !isFirstOnLine(text, index) && capitalFollows(text, index + digits.length);
27
+
28
+ /** 文末の番号の数字を英字に替えた文字列。長さは変わらない。 */
29
+ export const unmarkNumberStops = (text: string): string =>
30
+ text.replace(NUMBER_WORD, (digits: string, index: number) => (isSentenceEnd(text, index, digits) ? PLAIN_LETTER.repeat(digits.length) : digits));
package/src/pos.ts CHANGED
@@ -4,7 +4,9 @@ import { analyserPieces } from "./analyser-pieces.ts";
4
4
  import { readCounterTsu, type Morpheme } from "./counter-tsu.ts";
5
5
  import { outsideTheReport, isPassiveForm, passiveVocabulary, readsAsPassive } from "./passive-reading.ts";
6
6
  import { loadLexicons } from "./lexicons.ts";
7
- import { isEchoAt, type Inflection } from "./reduplication.ts";
7
+ import { predicateFrameAfter } from "./predicate-frame.ts";
8
+ import { distributiveVocabulary, isEchoAt, type Inflection } from "./reduplication.ts";
9
+ import { isRaDroppedAt, raDroppedVocabulary } from "./ra-dropped.ts";
8
10
  import { createRequire } from "node:module";
9
11
  import { dirname, join } from "node:path";
10
12
  import type { Token } from "chaffjs/plugin";
@@ -113,7 +115,15 @@ export const upos = (pos: string, detail: string): string => BY_DETAIL[detail] ?
113
115
  * (自発や状態を言う動詞、名付けの「と呼ばれる」、尊敬の形と決まり文句、ら抜きと可能)と、文が報告する動作の外にあるもの
114
116
  * (仮定の節、「〜されやすい」)は passive-reading.ts が外す。残りは「受動の形」として印を付ける。
115
117
  */
116
- const PASSIVE_VOCABULARY = passiveVocabulary(loadLexicons());
118
+ const LEXICONS = loadLexicons();
119
+
120
+ const PASSIVE_VOCABULARY = passiveVocabulary(LEXICONS);
121
+
122
+ const RA_DROPPED_VOCABULARY = raDroppedVocabulary(LEXICONS);
123
+
124
+ const PREDICATE_FRAMES = (LEXICONS["passive-predicate-frame"] ?? []).map((entry) => entry.pattern);
125
+
126
+ const REDUPLICATING_NOUNS = distributiveVocabulary(LEXICONS).nouns;
117
127
 
118
128
  /**
119
129
  * 非自立名詞(の・こと・もの・ため・はず)。品詞は名詞だが、単独では何も指さない。
@@ -146,14 +156,17 @@ export const isReady = (): boolean => state.ready !== undefined;
146
156
  * span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
147
157
  * 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
148
158
  */
149
- const toToken = (morpheme: Morpheme, start: number, passive: boolean, echo: boolean, light: boolean): Token => ({
159
+ /** 形態素を語にするときに、前後の形態素から決まる印。 */
160
+ type Marks = { readonly passive: boolean; readonly echo: boolean; readonly light: boolean; readonly raDropped: boolean };
161
+
162
+ const toToken = (morpheme: Morpheme, start: number, { passive, echo, light, raDropped }: Marks): Token => ({
150
163
  span: { start, end: start + morpheme.surface_form.length },
151
164
  surface: morpheme.surface_form,
152
165
  // UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
153
166
  pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
154
167
  ...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
155
168
  ...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
156
- ...withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo),
169
+ ...withRaDropped(withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo), raDropped),
157
170
  });
158
171
 
159
172
  /**
@@ -183,6 +196,10 @@ const withWordStatus = (
183
196
  const withEcho = (found: { features?: Readonly<Record<string, string>> }, echo: boolean): { features?: Readonly<Record<string, string>> } =>
184
197
  echo ? { features: { ...found.features, Echo: "Rdp" } } : found;
185
198
 
199
+ /** ら抜き言葉の一部(「食べれる」の 食べ と れる、「見れる」)。日本語に固有の印で、UD の FEATS の言語別拡張。 */
200
+ const withRaDropped = (found: { features?: Readonly<Record<string, string>> }, raDropped: boolean): { features?: Readonly<Record<string, string>> } =>
201
+ raDropped ? { features: { ...found.features, PotentialRa: "Dropped" } } : found;
202
+
186
203
  /**
187
204
  * 数(名詞,数)。UPOS では名詞に寄せるので、数であることは UD の NumType=Card で渡す。
188
205
  * 決まった言い回し(二人三脚、三日坊主、一人ひとり、十分)は辞書が一語として持つので、数にはならない。
@@ -222,9 +239,17 @@ const featuresOf = (morpheme: Morpheme, passive: boolean): { features?: Readonly
222
239
  const name = placeType(morpheme) ?? personOrOrganisation(morpheme);
223
240
  if (name !== undefined) return { features: { NameType: name } };
224
241
  if (isCounter(morpheme)) return { features: { NounType: "Class" } };
242
+ if (isVerbalNoun(morpheme)) return { features: { VerbForm: "Vnoun" } };
225
243
  return {};
226
244
  };
227
245
 
246
+ /**
247
+ * サ変名詞(調査・確認)。「する」を付ければ動詞になる名詞。UD の VerbForm=Vnoun。
248
+ * 辞書は知らない記号(「(VM)を」の「)」)もサ変接続に入れるので、字(漢字・かな・英字)を含む語だけ。
249
+ */
250
+ const LETTER = /\p{L}/u;
251
+ const isVerbalNoun = (morpheme: Morpheme): boolean => morpheme.pos === "名詞" && morpheme.pos_detail_1 === "サ変接続" && LETTER.test(morpheme.surface_form);
252
+
228
253
  /**
229
254
  * 名詞を修飾しているだけの受動から印を外す。「使用されるフレームワーク」
230
255
  * 「開催される BootCamp」は動作主を隠しているのではなく、名前の付けかたで、
@@ -244,13 +269,27 @@ const BREAK = new Set(["、", ",", ",", "。", ".", "."]);
244
269
  const clauseEnd = (tokens: readonly Token[], from: number): number =>
245
270
  tokens.find((token) => token.span.start >= from && BREAK.has(token.surface))?.span.start ?? Number.MAX_SAFE_INTEGER;
246
271
 
272
+ const unmarked = (token: Token): Token => ({
273
+ span: token.span,
274
+ surface: token.surface,
275
+ pos: token.pos,
276
+ ...(token.lemma === undefined ? {} : { lemma: token.lemma }),
277
+ });
278
+
279
+ const isDependentToken = (token: Token): boolean => token.features?.["NounType"] === "Dependent";
280
+
281
+ /** 受動の後ろ(from から節の終わりまで)に、受動が修飾する名詞があるか。型があれば、型の「こと」と後ろの「ため」のような非自立名詞は数えない。 */
282
+ const nounModifiedAfter = (tokens: readonly Token[], from: number, framed: boolean): boolean => {
283
+ const end = clauseEnd(tokens, from);
284
+ return tokens.some((other) => other.span.start >= from && other.span.end <= end && NOMINAL.has(other.pos) && !(framed && isDependentToken(other)));
285
+ };
286
+
247
287
  export const predicateOnly = (tokens: readonly Token[]): Token[] =>
248
- tokens.map((token) => {
288
+ tokens.map((token, index) => {
249
289
  if (token.features?.["Voice"] !== "Pass") return token;
250
- const end = clauseEnd(tokens, token.span.end);
251
- const modifiesNoun = tokens.some((other) => other.span.start >= token.span.end && other.span.end <= end && NOMINAL.has(other.pos));
252
- if (!modifiesNoun) return token;
253
- return { span: token.span, surface: token.surface, pos: token.pos, ...(token.lemma === undefined ? {} : { lemma: token.lemma }) };
290
+ const frame = predicateFrameAfter(tokens, index + 1, PREDICATE_FRAMES);
291
+ if (frame === undefined) return nounModifiedAfter(tokens, token.span.end, false) ? unmarked(token) : token;
292
+ return frame.conditional || nounModifiedAfter(tokens, token.span.end, true) ? unmarked(token) : token;
254
293
  });
255
294
 
256
295
  const inflectionOf = (morpheme: Morpheme, start: number): Inflection => ({
@@ -262,20 +301,24 @@ const inflectionOf = (morpheme: Morpheme, start: number): Inflection => ({
262
301
  start,
263
302
  });
264
303
 
304
+ /** 解析器を読み込んでいないか、読めなかったら undefined。 */
305
+ const readReady = (text: string): Placed[] | undefined => (state.ready === undefined ? undefined : readWith(state.ready, text));
306
+
265
307
  export const tokenize = (text: string): Token[] | undefined => {
266
- const tokenizer = state.ready;
267
- if (tokenizer === undefined) return undefined;
268
- const read = readAll(tokenizer, text);
308
+ const read = readReady(text);
309
+ return read === undefined ? undefined : tokensOf(read);
310
+ };
311
+
312
+ const tokensOf = (read: readonly Placed[]): Token[] => {
269
313
  const sequence = read.map(({ morpheme }) => morpheme);
270
314
  const inflections = read.map(({ morpheme, start }) => inflectionOf(morpheme, start));
271
315
  return read.map(({ morpheme, start }, index) =>
272
- toToken(
273
- morpheme,
274
- start,
275
- readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index),
276
- isEchoAt(inflections, index),
277
- isLightVerbAt(sequence, index),
278
- ),
316
+ toToken(morpheme, start, {
317
+ passive: readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index),
318
+ echo: isEchoAt(inflections, index, REDUPLICATING_NOUNS),
319
+ light: isLightVerbAt(sequence, index),
320
+ raDropped: isRaDroppedAt(sequence, index, RA_DROPPED_VOCABULARY),
321
+ }),
279
322
  );
280
323
  };
281
324
 
@@ -285,14 +328,23 @@ const PIECE_LIMIT = 1000;
285
328
  const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] =>
286
329
  readCounterTsu(analyserPieces(text, PIECE_LIMIT).flatMap((piece) => toArray(callMethod(tokenizer, "tokenize", [piece])).flatMap(toMorpheme)));
287
330
 
288
- /** 解析器は片割れのサロゲートで例外を投げる。数量の後ろを数文字だけ読み直すと、絵文字を半分に切ることがある。 */
289
- const readAll = (tokenizer: Tokenizer, text: string): { readonly morpheme: Morpheme; readonly start: number }[] => {
331
+ type Placed = { readonly morpheme: Morpheme; readonly start: number };
332
+
333
+ /**
334
+ * 読めない文字は先に置き換える(wellFormed)。それでも解析器が投げたら undefined を返し、品詞の無い文として渡す。
335
+ * 一つの文書のために run 全体を止めない。品詞が要る rule は、core が「読めなかった」として動かさない。
336
+ */
337
+ export const readWith = (tokenizer: Tokenizer, text: string): Placed[] | undefined => {
290
338
  const readable = wellFormed(text);
291
- return placed(readable, analyse(tokenizer, readable));
339
+ try {
340
+ return placed(readable, analyse(tokenizer, readable));
341
+ } catch {
342
+ return undefined;
343
+ }
292
344
  };
293
345
 
294
346
  /** 形態素と、本文の中での始まり。本文に見つからないものは落とす。 */
295
- const placed = (text: string, raws: readonly Morpheme[]): { readonly morpheme: Morpheme; readonly start: number }[] => {
347
+ const placed = (text: string, raws: readonly Morpheme[]): Placed[] => {
296
348
  const starts = surfaceStarts(
297
349
  text,
298
350
  raws.map((raw) => raw.surface_form),
@@ -316,11 +368,9 @@ export type Morph = {
316
368
  readonly detail2: string;
317
369
  };
318
370
 
319
- /** 解析器を読み込んでいなければ undefined。呼ぶ側は、形態素なしの読み方に戻る。 */
320
- export const morphemes = (text: string): Morph[] | undefined => {
321
- const tokenizer = state.ready;
322
- if (tokenizer === undefined) return undefined;
323
- return readAll(tokenizer, text).map(({ morpheme: raw, start }) => ({
371
+ /** 解析器を読み込んでいないか、読めなければ undefined。呼ぶ側は、形態素なしの読み方に戻る。 */
372
+ export const morphemes = (text: string): Morph[] | undefined =>
373
+ readReady(text)?.map(({ morpheme: raw, start }) => ({
324
374
  start,
325
375
  end: start + raw.surface_form.length,
326
376
  surface: raw.surface_form,
@@ -328,7 +378,6 @@ export const morphemes = (text: string): Morph[] | undefined => {
328
378
  detail1: raw.pos_detail_1,
329
379
  detail2: raw.pos_detail_2,
330
380
  }));
331
- };
332
381
 
333
382
  const isCounterMorph = (morph: Morph): boolean => morph.pos === "名詞" && morph.detail1 === "接尾" && morph.detail2 === "助数詞";
334
383
 
@@ -0,0 +1,28 @@
1
+ import type { Token } from "chaffjs/plugin";
2
+
3
+ const lemmaOf = (token: Token): string => token.lemma ?? token.surface;
4
+
5
+ /** at から読んだ語の原形をつないで、frame とちょうど同じになる語の数。ならなければ 0。語は 1 字以上なので、frame の字数より多くは読まない。 */
6
+ const lengthOf = (tokens: readonly Token[], at: number, frame: string): number => {
7
+ const lemmas = tokens.slice(at, at + frame.length).map(lemmaOf);
8
+ return lemmas.map((_lemma, index) => lemmas.slice(0, index + 1).join("")).indexOf(frame) + 1;
9
+ };
10
+
11
+ /**
12
+ * at から始まる述語の型(「ことになる」「こととなる」)の語の数。型の名詞は修飾される名詞ではなく、述語の一部。
13
+ * 「検討されることとなった」の「こと」は、受動を名詞の修飾にしない。原形で照らすので、活用した形(なりました)にも当たる。
14
+ */
15
+ export const predicateFrameAt = (tokens: readonly Token[], at: number, frames: readonly string[]): number =>
16
+ Math.max(0, ...frames.map((frame) => lengthOf(tokens, at, frame)));
17
+
18
+ /** 型のすぐ後ろで仮定の節を作る語(「検討されることとなれば」「〜ことになると」)。起きたことではなく、その前提を言う。 */
19
+ const CONDITIONAL = new Set(["ば", "と"]);
20
+
21
+ /**
22
+ * at から始まる述語の型と、その後ろが仮定の節かどうか。型が無ければ undefined。
23
+ * 型の名詞(こと・事)は非自立名詞なので、型の後ろの名詞を読む側は非自立名詞を数えなければ型を読み飛ばせる。
24
+ */
25
+ export const predicateFrameAfter = (tokens: readonly Token[], at: number, frames: readonly string[]): { readonly conditional: boolean } | undefined => {
26
+ const length = predicateFrameAt(tokens, at, frames);
27
+ return length === 0 ? undefined : { conditional: CONDITIONAL.has(tokens[at + length]?.surface ?? "") };
28
+ };
package/src/quantities.ts CHANGED
@@ -93,6 +93,17 @@ const LOOKAHEAD = 8;
93
93
  */
94
94
  const rereadAfterSpace = (number: string, rest: string): Morph[] => morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
95
95
 
96
+ /**
97
+ * 空白の後ろだけを読むと、名詞につく接頭詞と名詞で始まる(「1.4 本利用ルール」「1.2 本規約」の「本」)。詰めて読み直すと
98
+ * 助数詞に見えるが、行頭の番号の後ろなら、書き手が空けたのは番号と題のあいだ。「3 本の鉛筆」の「本」は単独で名詞、
99
+ * 「0.5 枚の紙」の「枚」は数につく接頭詞と読まれるので、どちらも数量のまま。
100
+ * 文中の数量には使わない。「10 両編成」の「両」も単独では名詞につく接頭詞と読まれる。
101
+ */
102
+ const opensWithPrefix = (rest: string): boolean => {
103
+ const [first, second] = morphemes(toHalfWidth(rest.slice(0, LOOKAHEAD))) ?? [];
104
+ return first?.pos === "接頭詞" && first.detail1 === "名詞接続" && second?.pos === "名詞";
105
+ };
106
+
96
107
  const unitAfterSpace = (number: string, rest: string): string | undefined => {
97
108
  const next = rereadAfterSpace(number, rest).find((morph) => morph.start >= number.length);
98
109
  return next !== undefined && isCounter(next) ? next.surface : undefined;
@@ -285,5 +296,5 @@ const countedByTableAfter = (rest: string): boolean => {
285
296
  */
286
297
  export const countedAfter = (number: string, rest: string): boolean => {
287
298
  if (!isReady()) return countedByTableAfter(rest);
288
- return unitAfterSpace(number, rest) !== undefined || multiplierAfter(number, rest);
299
+ return (unitAfterSpace(number, rest) !== undefined && !opensWithPrefix(rest)) || multiplierAfter(number, rest);
289
300
  };
@@ -0,0 +1,53 @@
1
+ // lang-en/src/quoted-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ import type { Span } from "chaffjs/plugin";
3
+ import { isAbbreviation } from "./sentence-split.ts";
4
+
5
+ /**
6
+ * 米国式の引用は、文末のピリオドを閉じ引用符の内側に置く(…called it "a fair trial." Plainly, …)。
7
+ * sentence-splitter は引用符の中の句点で文を切らず、引用符が閉じた後の空白でも切らないので、次の文が前の文につながる。
8
+ *
9
+ * 分割器が返した一つの文を、文末の記号 + 閉じ引用符 + 空白 + 次の文の頭(大文字。開き括弧・引用符の後の大文字も)で切る。
10
+ * 引用符の中の語が略語・頭文字("U.S." "Dr." "J.")のときと、括弧が開いたままのときは切らない。
11
+ */
12
+ /** 次の文の頭。空白の後の大文字。開き括弧・引用符の後の大文字も文頭である。 */
13
+ export const NEXT_SENTENCE_HEAD = String.raw`\s+[\p{Ps}\p{Pi}"']*\p{Lu}`;
14
+ const QUOTED_STOP = new RegExp(String.raw`[.?!]["'”’]+(?=${NEXT_SENTENCE_HEAD})`, "gu");
15
+ // ピリオドの前の語は、空白・開き引用符・開き括弧の後から数える。
16
+ const WORD_START = /[\s\p{Ps}\p{Pi}"']/u;
17
+ // 一字ずつピリオドを打った語(J. / U.S. / e.g.)は、略語の一覧に無くても略語。
18
+ const INITIALS = /^(?:\p{L}\.)+$/u;
19
+ // 分割器が一組として読む括弧(( [ { ( [ { 【 《 「 『 …)は、Unicode の開き・閉じ括弧の分類にすべて入る。
20
+ const OPENER = /\p{Ps}/u;
21
+ const CLOSER = /\p{Pe}/u;
22
+
23
+ const endsWithAbbreviation = (sentence: string, stop: number): boolean => {
24
+ if (sentence[stop] !== ".") return false;
25
+ const word = `${sentence.slice(0, stop).split(WORD_START).at(-1) ?? ""}.`;
26
+ return isAbbreviation(word) || INITIALS.test(word);
27
+ };
28
+
29
+ // 対の無い閉じ括弧(「1) 最初の項目」の番号)は、後から開く括弧を打ち消さない。
30
+ const depthAfter = (depth: number, char: string): number => {
31
+ if (OPENER.test(char)) return depth + 1;
32
+ return CLOSER.test(char) ? Math.max(0, depth - 1) : depth;
33
+ };
34
+
35
+ const bracketOpenAt = (sentence: string, index: number): boolean => Array.from(sentence.slice(0, index)).reduce(depthAfter, 0) > 0;
36
+
37
+ type Cut = { readonly end: number; readonly next: number };
38
+
39
+ const cutsIn = (sentence: string): Cut[] =>
40
+ [...sentence.matchAll(QUOTED_STOP)]
41
+ .filter((match) => !endsWithAbbreviation(sentence, match.index) && !bracketOpenAt(sentence, match.index))
42
+ .map((match) => {
43
+ const end = match.index + match[0].length;
44
+ return { end, next: end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0) };
45
+ });
46
+
47
+ /** span の文を、閉じ引用符の内側で閉じた文ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
48
+ export const splitAtQuotedStops = (text: string, span: Span): Span[] => {
49
+ const cuts = cutsIn(text.slice(span.start, span.end));
50
+ const starts = [0, ...cuts.map((cut) => cut.next)];
51
+ const ends = [...cuts.map((cut) => cut.end), span.end - span.start];
52
+ return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
53
+ };
@@ -0,0 +1,29 @@
1
+ import type { Lexicon } from "chaffjs/plugin";
2
+ import type { Morpheme } from "./counter-tsu.ts";
3
+
4
+ /**
5
+ * ら抜き言葉(「見れる」「食べれる」「来れる」)。一段動詞とカ変動詞の可能は「られる」で作る(食べられる・来られる)。
6
+ * 一段動詞・カ変動詞の未然形に「れる」が直に付けば「ら」が抜けている(食べ+れる、こ+れる)。五段動詞の可能動詞(走れる・帰れる)は解析器が一語の一段動詞と読むので当たらない。
7
+ * 解析器が一語として持つら抜きの形(見れる・来れる)は、語彙表 ra-dropped-verb に並べる。
8
+ */
9
+ export type RaDroppedVocabulary = ReadonlySet<string>;
10
+
11
+ export const raDroppedVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): RaDroppedVocabulary =>
12
+ new Set((lexicons["ra-dropped-verb"] ?? []).map((entry) => entry.pattern));
13
+
14
+ /** 可能を「られる」で作る動詞(一段動詞とカ変動詞)。「れる」はその未然形にしか付かない。 */
15
+ const isRareruStem = (morpheme: Morpheme | undefined): boolean =>
16
+ morpheme?.pos === "動詞" && (morpheme.conjugated_type === "一段" || (morpheme.conjugated_type?.startsWith("カ変") ?? false));
17
+
18
+ /** 動詞に付いた「れる」(IPADIC の動詞,接尾)。 */
19
+ const isBareReru = (morpheme: Morpheme | undefined): boolean => morpheme?.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && morpheme.basic_form === "れる";
20
+
21
+ /** sequence[index] が、ら抜き言葉の一部(一段・カ変動詞の未然形とそれに付いた「れる」、または一語で持つら抜きの形)か。 */
22
+ export const isRaDroppedAt = (sequence: readonly Morpheme[], index: number, vocabulary: RaDroppedVocabulary): boolean => {
23
+ const morpheme = sequence[index];
24
+ if (morpheme === undefined) return false;
25
+ if (morpheme.pos === "動詞" && vocabulary.has(morpheme.basic_form)) return true;
26
+ // 「れ」も一段の活用をするので(こ・れ・ない の れ は一段の未然形)、先に「れる」かを見る。
27
+ if (isBareReru(morpheme)) return isRareruStem(sequence[index - 1]);
28
+ return isRareruStem(morpheme) && isBareReru(sequence[index + 1]);
29
+ };