@chaffjs/lang-ja 0.15.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/README.md +1 -1
  2. package/dist/closing-quote.d.ts +6 -0
  3. package/dist/closing-quote.d.ts.map +1 -0
  4. package/dist/closing-quote.js +50 -0
  5. package/dist/closing-quote.js.map +1 -0
  6. package/dist/english-run.d.ts +8 -0
  7. package/dist/english-run.d.ts.map +1 -0
  8. package/dist/english-run.js +24 -0
  9. package/dist/english-run.js.map +1 -0
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +56 -21
  12. package/dist/index.js.map +1 -1
  13. package/dist/inside-brackets.d.ts +5 -0
  14. package/dist/inside-brackets.d.ts.map +1 -0
  15. package/dist/inside-brackets.js +44 -0
  16. package/dist/inside-brackets.js.map +1 -0
  17. package/dist/label-stop.d.ts +13 -0
  18. package/dist/label-stop.d.ts.map +1 -0
  19. package/dist/label-stop.js +14 -0
  20. package/dist/label-stop.js.map +1 -0
  21. package/dist/long-vowel-form.d.ts +12 -0
  22. package/dist/long-vowel-form.d.ts.map +1 -0
  23. package/dist/long-vowel-form.js +28 -0
  24. package/dist/long-vowel-form.js.map +1 -0
  25. package/dist/number-stop.d.ts +3 -0
  26. package/dist/number-stop.d.ts.map +1 -0
  27. package/dist/number-stop.js +25 -0
  28. package/dist/number-stop.js.map +1 -0
  29. package/dist/pos.d.ts.map +1 -1
  30. package/dist/pos.js +19 -3
  31. package/dist/pos.js.map +1 -1
  32. package/dist/quantities.d.ts.map +1 -1
  33. package/dist/quantities.js +11 -1
  34. package/dist/quantities.js.map +1 -1
  35. package/dist/quoted-stop.d.ts +13 -0
  36. package/dist/quoted-stop.d.ts.map +1 -0
  37. package/dist/quoted-stop.js +45 -0
  38. package/dist/quoted-stop.js.map +1 -0
  39. package/dist/regexp.d.ts +3 -0
  40. package/dist/regexp.d.ts.map +1 -0
  41. package/dist/regexp.js +4 -0
  42. package/dist/regexp.js.map +1 -0
  43. package/dist/sentence-close.d.ts +2 -0
  44. package/dist/sentence-close.d.ts.map +1 -0
  45. package/dist/sentence-close.js +10 -0
  46. package/dist/sentence-close.js.map +1 -0
  47. package/dist/sentence-split.d.ts +2 -0
  48. package/dist/sentence-split.d.ts.map +1 -1
  49. package/dist/sentence-split.js +3 -1
  50. package/dist/sentence-split.js.map +1 -1
  51. package/dist/spaced-counter.d.ts.map +1 -1
  52. package/dist/spaced-counter.js +2 -1
  53. package/dist/spaced-counter.js.map +1 -1
  54. package/dist/structure.d.ts.map +1 -1
  55. package/dist/structure.js +26 -5
  56. package/dist/structure.js.map +1 -1
  57. package/lexicons/abbreviated-label.yaml +16 -0
  58. package/lexicons/ai-tell.yaml +69 -0
  59. package/lexicons/announcing-opener.yaml +11 -0
  60. package/lexicons/assistant-residue.yaml +37 -0
  61. package/lexicons/calendar-era.yaml +10 -0
  62. package/lexicons/closing-cliche.yaml +2 -0
  63. package/lexicons/contrast-frame.yaml +9 -0
  64. package/lexicons/contrast-lead.yaml +10 -0
  65. package/lexicons/contrast-turn.yaml +9 -0
  66. package/lexicons/count-anchor.yaml +10 -0
  67. package/lexicons/count-counter.yaml +17 -0
  68. package/lexicons/count-hedge.yaml +61 -0
  69. package/lexicons/count-number.yaml +16 -0
  70. package/lexicons/document-kind.yaml +1 -0
  71. package/lexicons/email-attachment-note.yaml +8 -0
  72. package/lexicons/email-attribution.yaml +10 -0
  73. package/lexicons/email-header-field.yaml +37 -0
  74. package/lexicons/email-written-field.yaml +11 -0
  75. package/lexicons/emphasis-word.yaml +2 -1
  76. package/lexicons/figure-label.yaml +11 -0
  77. package/lexicons/measure-unit.yaml +78 -0
  78. package/lexicons/name-title.yaml +20 -0
  79. package/lexicons/neutral-ending.yaml +10 -0
  80. package/lexicons/numbered-label.yaml +50 -0
  81. package/lexicons/percent-unit.yaml +7 -0
  82. package/lexicons/placeholder-word.yaml +21 -0
  83. package/lexicons/range-closer.yaml +5 -0
  84. package/lexicons/range-connector.yaml +11 -0
  85. package/lexicons/range-opener.yaml +7 -0
  86. package/lexicons/share-exception.yaml +7 -0
  87. package/lexicons/share-label.yaml +11 -0
  88. package/lexicons/stock-transition.yaml +11 -0
  89. package/package.json +1 -1
  90. package/src/closing-quote.ts +53 -0
  91. package/src/english-run.ts +25 -0
  92. package/src/index.ts +54 -13
  93. package/src/inside-brackets.ts +45 -0
  94. package/src/label-stop.ts +26 -0
  95. package/src/long-vowel-form.ts +38 -0
  96. package/src/number-stop.ts +30 -0
  97. package/src/pos.ts +32 -3
  98. package/src/quantities.ts +12 -1
  99. package/src/quoted-stop.ts +53 -0
  100. package/src/regexp.ts +3 -0
  101. package/src/sentence-close.ts +10 -0
  102. package/src/sentence-split.ts +3 -1
  103. package/src/spaced-counter.ts +2 -1
  104. package/src/structure.ts +27 -5
package/src/index.ts CHANGED
@@ -1,16 +1,27 @@
1
1
  import { loadLexicons } from "./lexicons.ts";
2
+ import { closesSentence } from "./sentence-close.ts";
2
3
  import { sentenceSpans } from "./sentence-split.ts";
4
+ import { isEnglishBoundary, isEnglishRun } from "./english-run.ts";
5
+ import { unmarkNumberStops } from "./number-stop.ts";
6
+ import { labelStops, unmarkLabelStops } from "./label-stop.ts";
7
+ import { splitAtQuotedStops } from "./quoted-stop.ts";
8
+ import { reattachClosingQuotes } from "./closing-quote.ts";
9
+ import { insideBrackets, startsWithCloser } from "./inside-brackets.ts";
3
10
  import { structure } from "./structure.ts";
4
11
  import { isReady, predicateOnly, prepare, readsAsCounter, readsAsOneAdverb, readsAsOneWord, tokenize } from "./pos.ts";
5
12
  import { markSpacedCounters } from "./spaced-counter.ts";
6
13
  import { tokensWithin } from "./tokens-within.ts";
7
14
  import { distributiveVocabulary, iterationMarkReading, markReduplication } from "./reduplication.ts";
8
- import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
15
+ import { knownWordReading, markDroppedLongVowels, remembered } from "./long-vowel-form.ts";
16
+ import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
9
17
 
10
18
  // chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
11
19
 
20
+ const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
21
+ const COUNTABLE = /\S/gu;
22
+
12
23
  /**
13
- * 日本語の文末は「。!?」に限られる。
24
+ * 日本語の文末は「。!?」と、語の後の「.」に限られる(closesSentence)。
14
25
  *
15
26
  * sentence-splitter は "." も文末と見なすため、「、Dr. 田中は」で誤分割する。
16
27
  * AbbrMarker の language を差し替えても直らない。原因は略語の保護ではなく、
@@ -19,30 +30,42 @@ import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from
19
30
  * 文長は sentence-rhythm と max-sentence-length の入力なので、
20
31
  * 誤分割はそのまま指標を動かす。ここで閉じる。
21
32
  */
22
- const CLOSED = /[。!?!?]["))」』]*\s*$/u;
33
+ const isOpen = (text: string): boolean => text.trim().length > 0 && !closesSentence(text);
23
34
 
24
- const JAPANESE = /[぀-ゟ゠-ヿ一-鿿]/gu;
25
- const COUNTABLE = /\S/gu;
35
+ /**
36
+ * 和文の句点で閉じていない断片、括弧・二重引用符の中で切れた断片、閉じ括弧で始まる断片の前は次へ続く。
37
+ * 閉じ括弧は閉じる文に付き、後ろは分割器が組にした括弧(「…。」と言った)と同じく同じ文に続く。
38
+ * ただし英文どうしは、英語のアダプタと同じく英語の文末で切ったところで切る。
39
+ */
40
+ const continues = (before: string, after: string, bracketOpen: boolean): boolean =>
41
+ (isOpen(before) || bracketOpen || startsWithCloser(after)) && !isEnglishBoundary(before, after);
42
+
43
+ const LINE_BREAK = /[\n\r]/u;
26
44
 
27
- const isOpen = (text: string): boolean => text.trim().length > 0 && !CLOSED.test(text);
45
+ const ENGLISH: EmbeddedLanguage = { id: "en", lengthUnit: "word" };
46
+
47
+ const withLanguage = (text: string, span: Span): Sentence => (isEnglishRun(text) ? { span, text, embeddedLanguage: ENGLISH } : { span, text });
28
48
 
29
49
  /**
30
50
  * 断片を繋ぐときは raw の連結ではなくオフセットを使う。
31
51
  * sentence-splitter は空白を別ノードに分けるため、raw を繋ぐと空白が落ちて文長が縮む。
32
52
  */
33
- const merge = (source: string, spans: readonly Span[]): Sentence[] =>
34
- spans
53
+ const merge = (source: string, spans: readonly Span[]): Sentence[] => {
54
+ const open = insideBrackets(source);
55
+ return spans
35
56
  .reduce<Span[]>((acc, span) => {
36
57
  const last = acc.at(-1);
37
58
  // 断片の「間」に改行があれば繋がない。結合は「Dr. 田中」のような行の途中の
38
59
  // 誤分割を閉じるためのもので、行またぎは要らない。またぐと、引用ブロックの
39
60
  // 英文と訳文のように別の行のものまで 1 文に繋がる。
40
- const acrossLines = last !== undefined && source.slice(last.end, span.start).includes("\n");
41
- if (last !== undefined && !acrossLines && isOpen(source.slice(last.start, last.end))) acc[acc.length - 1] = { start: last.start, end: span.end };
61
+ const acrossLines = last !== undefined && LINE_BREAK.test(source.slice(last.end, span.start));
62
+ if (last !== undefined && !acrossLines && continues(source.slice(last.start, last.end), source.slice(span.start, span.end), open[last.end] === true))
63
+ acc[acc.length - 1] = { start: last.start, end: span.end };
42
64
  else acc.push(span);
43
65
  return acc;
44
66
  }, [])
45
- .map((span) => ({ span, text: source.slice(span.start, span.end) }));
67
+ .map((span) => withLanguage(source.slice(span.start, span.end), span));
68
+ };
46
69
 
47
70
  /**
48
71
  * token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
@@ -51,11 +74,15 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
51
74
  const LEXICONS = loadLexicons();
52
75
  const DISTRIBUTIVE = distributiveVocabulary(LEXICONS);
53
76
  const TAKES_ITERATION_MARK = iterationMarkReading(LEXICONS, readsAsOneWord);
77
+ const KNOWS_WITH_LONG_VOWEL = remembered(knownWordReading(tokenize));
78
+ /** Whether a running rule reads LongVowelEnding. Asking the dictionary about every katakana word costs, so only then. */
79
+ const askForLongVowels = { value: false };
54
80
 
55
81
  const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
56
82
  const read = tokenize(source);
57
83
  if (read === undefined) return [...sentences];
58
- const tokens = markSpacedCounters(read, readsAsCounter);
84
+ const counted = markSpacedCounters(read, readsAsCounter);
85
+ const tokens = askForLongVowels.value ? markDroppedLongVowels(counted, KNOWS_WITH_LONG_VOWEL) : counted;
59
86
  return sentences.map((sentence) => ({
60
87
  ...sentence,
61
88
  // 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
@@ -63,6 +90,18 @@ const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[]
63
90
  }));
64
91
  };
65
92
 
93
+ const LABEL_STOPS = labelStops((LEXICONS["abbreviated-label"] ?? []).map((entry) => entry.pattern));
94
+
95
+ /**
96
+ * 英語のアダプタと同じ手当てを英文にも当てる。番号の前の略した名前(FIG. 1)の点で切らず、
97
+ * 英文の閉じ引用符の内側の句点で切り、文頭に取り残された閉じ引用符を前の文へ戻す。
98
+ */
99
+ const spansOf = (text: string): Span[] => {
100
+ const spans = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS)));
101
+ const quotedStops = spans.flatMap((span) => (isEnglishRun(text.slice(span.start, span.end)) ? splitAtQuotedStops(text, span) : [span]));
102
+ return reattachClosingQuotes(text, quotedStops);
103
+ };
104
+
66
105
  export const adapter: LanguageAdapter = {
67
106
  kind: "language",
68
107
  id: "ja",
@@ -76,6 +115,8 @@ export const adapter: LanguageAdapter = {
76
115
  lengthUnit: "char",
77
116
  },
78
117
  prepare: async (need: AdapterNeeds): Promise<void> => {
118
+ // Once asked, keep marking: files are prepared in parallel, and a later prepare must not unmark an earlier file's run.
119
+ askForLongVowels.value ||= need.features?.includes("LongVowelEnding") ?? false;
79
120
  if (need.pos) await prepare();
80
121
  },
81
122
  detect: (source: string): number => {
@@ -86,7 +127,7 @@ export const adapter: LanguageAdapter = {
86
127
  lexicons: loadLexicons(),
87
128
  structure,
88
129
  segment: (text: string): Segmentation => {
89
- const sentences = merge(text, sentenceSpans(text));
130
+ const sentences = merge(text, spansOf(text));
90
131
  return { sentences: isReady() ? withTokens(text, sentences) : sentences };
91
132
  },
92
133
  };
@@ -0,0 +1,45 @@
1
+ /**
2
+ * 括弧・二重引用符の中の句点は文を閉じない。sentence-splitter は括弧を種類ごとに一つしか数えないので、
3
+ * 同じ種類が入れ子になると内側の閉じ括弧で外側まで閉じたと読み(「(…(…)…。以下同じ。)に」)、曲がった二重引用符(“…”)は数えない。
4
+ * ここでは開きと閉じを入れ子のまま組にする。一重引用符(‘ ’)はアポストロフィと見分けられず、直線の引用符は分割器が数えるので、どちらも数えない。
5
+ */
6
+ const OPENER = /[\p{Ps}“«]/u;
7
+ const CLOSER = /[\p{Pe}”»]/u;
8
+ const LINE_BREAK = /[\n\r]/u;
9
+
10
+ type Pair = { readonly open: number; readonly close: number };
11
+
12
+ /**
13
+ * 同じ行の中で組になった開きと閉じの位置。組にならないもの(「1)」の番号、同上の印の「“」、閉じない括弧)は含めない。
14
+ * 閉じない括弧を数えると、行の終わりまでの文が全部つながる。
15
+ */
16
+ const pairsOf = (text: string): Pair[] => {
17
+ const pairs: Pair[] = [];
18
+ const opened: number[] = [];
19
+ text.split("").forEach((unit, index) => {
20
+ if (LINE_BREAK.test(unit)) opened.length = 0;
21
+ else if (OPENER.test(unit)) opened.push(index);
22
+ else if (CLOSER.test(unit)) {
23
+ const open = opened.pop();
24
+ if (open !== undefined) pairs.push({ open, close: index });
25
+ }
26
+ });
27
+ return pairs;
28
+ };
29
+
30
+ /** 閉じ括弧・閉じ引用符で始まるか。前の行で開いた括弧や数え損ねた括弧の閉じが、句点の後で文頭に残ったもの。 */
31
+ export const startsWithCloser = (text: string): boolean => CLOSER.test(text.trimStart().charAt(0));
32
+
33
+ /** 各位置(0 から text.length まで)の手前に、同じ行で後から閉じる括弧・二重引用符が開いているか。 */
34
+ export const insideBrackets = (text: string): boolean[] => {
35
+ const change = Array.from({ length: text.length + 2 }, () => 0);
36
+ pairsOf(text).forEach((pair) => {
37
+ change[pair.open + 1] = (change[pair.open + 1] ?? 0) + 1;
38
+ change[pair.close + 1] = (change[pair.close + 1] ?? 0) - 1;
39
+ });
40
+ const depth = { value: 0 };
41
+ return change.slice(0, text.length + 1).map((step) => {
42
+ depth.value += step;
43
+ return depth.value > 0;
44
+ });
45
+ };
@@ -0,0 +1,26 @@
1
+ // lang-en/src/label-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ import { escapeRegExp } from "./regexp.ts";
3
+
4
+ /**
5
+ * sentence-splitter ends a sentence at the full stop of a label before its number: "FIG." and "1 illustrates …" were two
6
+ * sentences, and so were "Vol." and "XLIII (1979)". Where a number follows, that full stop is replaced with a letter
7
+ * before splitting. The length does not change, so the spans fit the original text.
8
+ * A lone "I" after a label is the pronoun ("He said No. I left."), and a word ("the last Fig. The tree") is not a number.
9
+ */
10
+
11
+ /** Built once from the lexicon; undefined when no label ends in a full stop. */
12
+ export type LabelStops = RegExp | undefined;
13
+
14
+ const NUMBER_AFTER = String.raw`(?=\s+(?:\d|(?:[IVXLCDM]{2,}|[VXLCDM])(?![\p{L}\p{N}_])))`;
15
+ const PLAIN_LETTER = "n";
16
+
17
+ /** The labels as written and in capitals, without their full stop ("Fig." → Fig, FIG). */
18
+ export const labelStops = (labels: readonly string[]): LabelStops => {
19
+ const stems = labels.filter((label) => label.endsWith(".")).flatMap((label) => [label.slice(0, -1), label.slice(0, -1).toUpperCase()]);
20
+ if (stems.length === 0) return undefined;
21
+ return new RegExp(String.raw`(?<![\p{L}\p{N}_.])(?:${[...new Set(stems)].map(escapeRegExp).join("|")})\.${NUMBER_AFTER}`, "gu");
22
+ };
23
+
24
+ /** The text with the full stop of every label before a number replaced by a letter. */
25
+ export const unmarkLabelStops = (text: string, stops: LabelStops): string =>
26
+ stops === undefined ? text : text.replace(stops, (label: string) => `${label.slice(0, -1)}${PLAIN_LETTER}`);
@@ -0,0 +1,38 @@
1
+ import type { Token } from "chaffjs/plugin";
2
+
3
+ const KATAKANA_WORD = /^[ァ-ヺー]+$/u;
4
+ const LONG_VOWEL = "ー";
5
+
6
+ /** A katakana noun written without a final ー, which could be a loanword whose final long vowel was dropped (コンピュータ). */
7
+ const mayHaveDropped = (token: Token): boolean =>
8
+ token.pos === "NOUN" && KATAKANA_WORD.test(token.surface) && !token.surface.endsWith(LONG_VOWEL) && Array.from(token.surface).length > 1;
9
+
10
+ /**
11
+ * Marks a katakana noun whose form with a final ー is a word the dictionary knows (コンピュータ → コンピューター): LongVowelEnding=Dropped.
12
+ * Only the dictionary can tell a dropped long vowel (メモリ) from a word that never had one (データ, ハードウェア).
13
+ * knowsWithLongVowel is given by index.ts, which holds the analyser; each surface is asked once.
14
+ */
15
+ export const markDroppedLongVowels = (tokens: readonly Token[], knowsWithLongVowel: (surface: string) => boolean): Token[] =>
16
+ tokens.map((token) =>
17
+ mayHaveDropped(token) && knowsWithLongVowel(token.surface) ? { ...token, features: { ...token.features, LongVowelEnding: "Dropped" } } : token,
18
+ );
19
+
20
+ /** Whether text reads as one word the dictionary knows (a lemma, not an unknown run of katakana). */
21
+ export const knownWordReading =
22
+ (tokenize: (text: string) => readonly Token[] | undefined) =>
23
+ (surface: string): boolean => {
24
+ const read = tokenize(`${surface}${LONG_VOWEL}`);
25
+ return read?.length === 1 && read[0]?.lemma !== undefined && read[0].pos === "NOUN";
26
+ };
27
+
28
+ /** Remembers each answer: a document repeats its words, and the analyser is the cost. */
29
+ export const remembered = (ask: (surface: string) => boolean): ((surface: string) => boolean) => {
30
+ const answers = new Map<string, boolean>();
31
+ return (surface) => {
32
+ const known = answers.get(surface);
33
+ if (known !== undefined) return known;
34
+ const answer = ask(surface);
35
+ answers.set(surface, answer);
36
+ return answer;
37
+ };
38
+ };
@@ -0,0 +1,30 @@
1
+ /**
2
+ * lang-en/src/number-stop.ts の写し。和文の中の英文も同じ分割器で切るので同じ手当てが要る。アダプタは互いに依存しないので写しを置く。
3
+ *
4
+ * sentence-splitter は空白の後の「数字だけ + ピリオド」を、どこにあっても箇条書きの番号と読み、そこで文を切らない。
5
+ * 「in the spring of 2026. We shipped」が 1 文になる。
6
+ *
7
+ * 行の途中にあり、次の語が大文字で始まる番号は文末である。その数字だけを同じ長さの英字に替えて分割器に渡し、
8
+ * 普通の語として読ませる。長さが変わらないので、返る span は元の文字列にそのまま当てはまる。
9
+ * 行頭の番号(「1. First」)と、次が大文字でない番号は替えない。
10
+ */
11
+ const NUMBER_WORD = /(?<=\s)\d+(?=\.\s)/g;
12
+ // 次の語の頭。開き括弧・引用符の後の大文字も文頭である。
13
+ const CAPITAL_AFTER_SPACE = /\s+[\p{Ps}\p{Pi}"']*\p{Lu}/uy;
14
+ // 分割器の略語表に n だけでできた語は無い。数字の代わりに置いても、略語として読まれない。
15
+ const PLAIN_LETTER = "n";
16
+
17
+ const lineStartOf = (text: string, index: number): number => Math.max(text.lastIndexOf("\n", index - 1), text.lastIndexOf("\r", index - 1)) + 1;
18
+
19
+ const isFirstOnLine = (text: string, index: number): boolean => text.slice(lineStartOf(text, index), index).trim() === "";
20
+
21
+ const capitalFollows = (text: string, dot: number): boolean => {
22
+ CAPITAL_AFTER_SPACE.lastIndex = dot + 1;
23
+ return CAPITAL_AFTER_SPACE.test(text);
24
+ };
25
+
26
+ const isSentenceEnd = (text: string, index: number, digits: string): boolean => !isFirstOnLine(text, index) && capitalFollows(text, index + digits.length);
27
+
28
+ /** 文末の番号の数字を英字に替えた文字列。長さは変わらない。 */
29
+ export const unmarkNumberStops = (text: string): string =>
30
+ text.replace(NUMBER_WORD, (digits: string, index: number) => (isSentenceEnd(text, index, digits) ? PLAIN_LETTER.repeat(digits.length) : digits));
package/src/pos.ts CHANGED
@@ -146,16 +146,39 @@ export const isReady = (): boolean => state.ready !== undefined;
146
146
  * span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
147
147
  * 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
148
148
  */
149
- const toToken = (morpheme: Morpheme, start: number, passive: boolean, echo: boolean): Token => ({
149
+ const toToken = (morpheme: Morpheme, start: number, passive: boolean, echo: boolean, light: boolean): Token => ({
150
150
  span: { start, end: start + morpheme.surface_form.length },
151
151
  surface: morpheme.surface_form,
152
152
  // UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
153
153
  pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
154
154
  ...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
155
155
  ...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
156
- ...withEcho(featuresOf(morpheme, passive), echo),
156
+ ...withEcho(withWordStatus(featuresOf(morpheme, passive), isBound(morpheme), light), echo),
157
157
  });
158
158
 
159
+ /**
160
+ * 単独では語にならない形態素(IPADIC の接頭詞・接尾・非自立: ご案内の「ご」、こころさんの「さん」、勤めているの「いる」)は Bound=Yes。
161
+ * 語の中身を数える rule が、付いた先の語だけを数えるため。UD の FEATS の言語別拡張。
162
+ */
163
+ const isBound = (morpheme: Morpheme): boolean => morpheme.pos === "接頭詞" || morpheme.pos_detail_1 === "非自立" || morpheme.pos_detail_1 === "接尾";
164
+
165
+ /** サ変名詞を動詞にするだけの「する」「いたす」(説明します)。意味は名詞が持つ。UD の VerbType=Light。 */
166
+ const LIGHT_VERBS = new Set(["する", "いたす"]);
167
+
168
+ const isLightVerbAt = (sequence: readonly Morpheme[], index: number): boolean => {
169
+ const [previous, morpheme] = [sequence[index - 1], sequence[index]];
170
+ return morpheme?.pos === "動詞" && LIGHT_VERBS.has(morpheme.basic_form) && previous?.pos === "名詞" && previous.pos_detail_1 === "サ変接続";
171
+ };
172
+
173
+ const withWordStatus = (
174
+ found: { features?: Readonly<Record<string, string>> },
175
+ bound: boolean,
176
+ light: boolean,
177
+ ): { features?: Readonly<Record<string, string>> } => {
178
+ if (!bound && !light) return found;
179
+ return { features: { ...found.features, ...(bound ? { Bound: "Yes" } : {}), ...(light ? { VerbType: "Light" } : {}) } };
180
+ };
181
+
159
182
  /** 重ね言葉の二つ目(UD の Echo=Rdp)。ほかの印は残す。 */
160
183
  const withEcho = (found: { features?: Readonly<Record<string, string>> }, echo: boolean): { features?: Readonly<Record<string, string>> } =>
161
184
  echo ? { features: { ...found.features, Echo: "Rdp" } } : found;
@@ -246,7 +269,13 @@ export const tokenize = (text: string): Token[] | undefined => {
246
269
  const sequence = read.map(({ morpheme }) => morpheme);
247
270
  const inflections = read.map(({ morpheme, start }) => inflectionOf(morpheme, start));
248
271
  return read.map(({ morpheme, start }, index) =>
249
- toToken(morpheme, start, readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index), isEchoAt(inflections, index)),
272
+ toToken(
273
+ morpheme,
274
+ start,
275
+ readsAsPassive(sequence, index, PASSIVE_VOCABULARY) && !outsideTheReport(sequence, index),
276
+ isEchoAt(inflections, index),
277
+ isLightVerbAt(sequence, index),
278
+ ),
250
279
  );
251
280
  };
252
281
 
package/src/quantities.ts CHANGED
@@ -93,6 +93,17 @@ const LOOKAHEAD = 8;
93
93
  */
94
94
  const rereadAfterSpace = (number: string, rest: string): Morph[] => morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
95
95
 
96
+ /**
97
+ * 空白の後ろだけを読むと、名詞につく接頭詞と名詞で始まる(「1.4 本利用ルール」「1.2 本規約」の「本」)。詰めて読み直すと
98
+ * 助数詞に見えるが、行頭の番号の後ろなら、書き手が空けたのは番号と題のあいだ。「3 本の鉛筆」の「本」は単独で名詞、
99
+ * 「0.5 枚の紙」の「枚」は数につく接頭詞と読まれるので、どちらも数量のまま。
100
+ * 文中の数量には使わない。「10 両編成」の「両」も単独では名詞につく接頭詞と読まれる。
101
+ */
102
+ const opensWithPrefix = (rest: string): boolean => {
103
+ const [first, second] = morphemes(toHalfWidth(rest.slice(0, LOOKAHEAD))) ?? [];
104
+ return first?.pos === "接頭詞" && first.detail1 === "名詞接続" && second?.pos === "名詞";
105
+ };
106
+
96
107
  const unitAfterSpace = (number: string, rest: string): string | undefined => {
97
108
  const next = rereadAfterSpace(number, rest).find((morph) => morph.start >= number.length);
98
109
  return next !== undefined && isCounter(next) ? next.surface : undefined;
@@ -285,5 +296,5 @@ const countedByTableAfter = (rest: string): boolean => {
285
296
  */
286
297
  export const countedAfter = (number: string, rest: string): boolean => {
287
298
  if (!isReady()) return countedByTableAfter(rest);
288
- return unitAfterSpace(number, rest) !== undefined || multiplierAfter(number, rest);
299
+ return (unitAfterSpace(number, rest) !== undefined && !opensWithPrefix(rest)) || multiplierAfter(number, rest);
289
300
  };
@@ -0,0 +1,53 @@
1
+ // lang-en/src/quoted-stop.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ import type { Span } from "chaffjs/plugin";
3
+ import { isAbbreviation } from "./sentence-split.ts";
4
+
5
+ /**
6
+ * 米国式の引用は、文末のピリオドを閉じ引用符の内側に置く(…called it "a fair trial." Plainly, …)。
7
+ * sentence-splitter は引用符の中の句点で文を切らず、引用符が閉じた後の空白でも切らないので、次の文が前の文につながる。
8
+ *
9
+ * 分割器が返した一つの文を、文末の記号 + 閉じ引用符 + 空白 + 次の文の頭(大文字。開き括弧・引用符の後の大文字も)で切る。
10
+ * 引用符の中の語が略語・頭文字("U.S." "Dr." "J.")のときと、括弧が開いたままのときは切らない。
11
+ */
12
+ /** 次の文の頭。空白の後の大文字。開き括弧・引用符の後の大文字も文頭である。 */
13
+ export const NEXT_SENTENCE_HEAD = String.raw`\s+[\p{Ps}\p{Pi}"']*\p{Lu}`;
14
+ const QUOTED_STOP = new RegExp(String.raw`[.?!]["'”’]+(?=${NEXT_SENTENCE_HEAD})`, "gu");
15
+ // ピリオドの前の語は、空白・開き引用符・開き括弧の後から数える。
16
+ const WORD_START = /[\s\p{Ps}\p{Pi}"']/u;
17
+ // 一字ずつピリオドを打った語(J. / U.S. / e.g.)は、略語の一覧に無くても略語。
18
+ const INITIALS = /^(?:\p{L}\.)+$/u;
19
+ // 分割器が一組として読む括弧(( [ { ( [ { 【 《 「 『 …)は、Unicode の開き・閉じ括弧の分類にすべて入る。
20
+ const OPENER = /\p{Ps}/u;
21
+ const CLOSER = /\p{Pe}/u;
22
+
23
+ const endsWithAbbreviation = (sentence: string, stop: number): boolean => {
24
+ if (sentence[stop] !== ".") return false;
25
+ const word = `${sentence.slice(0, stop).split(WORD_START).at(-1) ?? ""}.`;
26
+ return isAbbreviation(word) || INITIALS.test(word);
27
+ };
28
+
29
+ // 対の無い閉じ括弧(「1) 最初の項目」の番号)は、後から開く括弧を打ち消さない。
30
+ const depthAfter = (depth: number, char: string): number => {
31
+ if (OPENER.test(char)) return depth + 1;
32
+ return CLOSER.test(char) ? Math.max(0, depth - 1) : depth;
33
+ };
34
+
35
+ const bracketOpenAt = (sentence: string, index: number): boolean => Array.from(sentence.slice(0, index)).reduce(depthAfter, 0) > 0;
36
+
37
+ type Cut = { readonly end: number; readonly next: number };
38
+
39
+ const cutsIn = (sentence: string): Cut[] =>
40
+ [...sentence.matchAll(QUOTED_STOP)]
41
+ .filter((match) => !endsWithAbbreviation(sentence, match.index) && !bracketOpenAt(sentence, match.index))
42
+ .map((match) => {
43
+ const end = match.index + match[0].length;
44
+ return { end, next: end + (/^\s+/u.exec(sentence.slice(end))?.[0].length ?? 0) };
45
+ });
46
+
47
+ /** span の文を、閉じ引用符の内側で閉じた文ごとに分けた span。切れ目が無ければ span そのもの一つ。 */
48
+ export const splitAtQuotedStops = (text: string, span: Span): Span[] => {
49
+ const cuts = cutsIn(text.slice(span.start, span.end));
50
+ const starts = [0, ...cuts.map((cut) => cut.next)];
51
+ const ends = [...cuts.map((cut) => cut.end), span.end - span.start];
52
+ return starts.map((start, index) => ({ start: span.start + start, end: span.start + (ends[index] ?? start) }));
53
+ };
package/src/regexp.ts ADDED
@@ -0,0 +1,3 @@
1
+ // lang-en/src/regexp.ts の写し。アダプタは互いに依存しないので、和文の中の英文に同じ手当てをするために置く。元とのずれは test_mixed_language_stops.ts で比べる。
2
+ /** A word from a lexicon, matched as written: a sign in it is that character, not a pattern. */
3
+ export const escapeRegExp = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/gu, String.raw`\$&`);
@@ -0,0 +1,10 @@
1
+ /**
2
+ * 文が閉じているか。句点の後に閉じ括弧と空白が続いてよい。
3
+ *
4
+ * 「.」は、論文などが「,.」で書くときの句点。仮名・漢字か閉じ括弧の後だけ句点と読む。
5
+ * 数字の後の「.」は番号か小数点なので(1.はじめに、3.5%)、文を閉じない。
6
+ */
7
+ const CLOSED = /[。!?!?]["))」』]*\s*$/u;
8
+ const FULL_STOP_AFTER_WORD = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}ー))」』].["))」』]*\s*$/u;
9
+
10
+ export const closesSentence = (text: string): boolean => CLOSED.test(text) || FULL_STOP_AFTER_WORD.test(text);
@@ -45,6 +45,8 @@ const { language } = DefaultAbbrMarkerOptions;
45
45
  const ABBREVIATIONS = new Set(
46
46
  [...language.ABBREVIATIONS, ...language.PREPOSITIVE_ABBREVIATIONS, ...language.EXCLAMATION_WORDS].map((word) => word.toLowerCase()),
47
47
  );
48
+ /** 分割器が略語と読む語(ピリオドまで含めて渡す)。「Dr.」「U.S.」「Jan.」。 */
49
+ export const isAbbreviation = (word: string): boolean => ABBREVIATIONS.has(word.toLowerCase());
48
50
  // 「J. Smith」のような語は前の語を見て略語か決まる。前の語が切れ目の向こうにあると判定が変わる。
49
51
  const CAPITAL_DOT = /\p{Lu}\./gu;
50
52
 
@@ -76,7 +78,7 @@ const indexesOf = (text: string, pattern: RegExp): number[] => [...text.matchAll
76
78
  /** 句点の並びの直後。和文は直前が仮名・漢字の句点、英文は略語でない語のピリオド。 */
77
79
  const stopEnds = (text: string): number[] => {
78
80
  const japanese = [...text.matchAll(JAPANESE_STOPS)].filter((match) => isCJK(text[match.index - 1]));
79
- const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !ABBREVIATIONS.has(match[0].toLowerCase()));
81
+ const english = [...text.matchAll(ENGLISH_STOP)].filter((match) => !isAbbreviation(match[0]));
80
82
  return [...japanese, ...english].map((match) => match.index + match[0].length).toSorted((a, b) => a - b);
81
83
  };
82
84
 
@@ -26,7 +26,8 @@ const asCounter = (token: Token): Token => ({
26
26
  surface: token.surface,
27
27
  pos: "NOUN",
28
28
  ...(token.lemma === undefined ? {} : { lemma: token.lemma }),
29
- features: { NounType: "Class" },
29
+ // 解析器が読む助数詞(名詞,接尾,助数詞)と同じ印。数に付いて単独では語にならない。
30
+ features: { NounType: "Class", Bound: "Yes" },
30
31
  });
31
32
 
32
33
  export const markSpacedCounters = (tokens: readonly Token[], readsAsCounter: CounterReading): Token[] =>
package/src/structure.ts CHANGED
@@ -202,7 +202,9 @@ const DEFINITIONS = [/以下「(?<term>[^「」\n]{1,40})」という/gu, /「(?
202
202
  */
203
203
  const THIS_PART = /この[条項号款目節章編](?:及び[^、。]{1,20})?において$/u;
204
204
  const NEARBY_PART = /[前次同](?:各)?[項条号]に規定する$/u;
205
- const scopedLocally = (before: string): boolean => THIS_PART.test(before) || NEARBY_PART.test(before);
205
+ /** 「規則第7条に規定する『X』とは」「法第31条第1項第1号の『X』とは」は、その番地で使う X の意味。番地の末尾だけを見る。 */
206
+ const ADDRESSED_PART = new RegExp(`第${NUMBER}[条項号](?:の${NUMBER})?(?:に規定する|の)$`, "u");
207
+ const scopedLocally = (before: string): boolean => THIS_PART.test(before) || NEARBY_PART.test(before) || ADDRESSED_PART.test(before);
206
208
 
207
209
  const definitions = (text: string): Mention[] =>
208
210
  DEFINITIONS.flatMap((pattern) =>
@@ -252,10 +254,22 @@ const depthsOf = (text: string): Int32Array => {
252
254
  * 「民法第709条」のように他の文書の名前が前にあれば、その名前を document に入れる。この文書の木では引かない。
253
255
  */
254
256
  /** at より前で開いたまま閉じていない括弧の数。 */
255
- /** 間に挟まった括弧書き(中に参照があっても)を外す。入れ子は内側から外す。 */
257
+ /**
258
+ * 間に挟まった括弧書き(中に参照があっても)を外す。入れ子は内側から外れる。閉じない「(」と開かない「)」は残す。
259
+ * 内側を外すのを繰り返すと深さの数だけ読み直すので、組ごと 1 度で外す。
260
+ */
256
261
  const withoutClosedParentheses = (gap: string): string => {
257
- const once = gap.replace(PARENTHESES, "");
258
- return once === gap ? gap : withoutClosedParentheses(once);
262
+ const kept: string[] = [];
263
+ const opens: number[] = [];
264
+ for (const char of gap) {
265
+ const opened = char === ")" ? opens.pop() : undefined;
266
+ if (opened !== undefined) kept.splice(opened);
267
+ else {
268
+ if (char === "(") opens.push(kept.length);
269
+ kept.push(char);
270
+ }
271
+ }
272
+ return kept.join("");
259
273
  };
260
274
 
261
275
  /**
@@ -281,9 +295,17 @@ const LEXICONS = loadLexicons();
281
295
  const CITATION = citationVocabulary(LEXICONS);
282
296
  const SECTIONS = sectionVocabulary(LEXICONS);
283
297
  const NOT_MAGNITUDE = (LEXICONS["not-magnitude"] ?? []).map((entry) => entry.pattern);
298
+ const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
299
+
300
+ /** 記号のすぐ後ろにラテン文字・数字・ハイフンが続けば、単位ではなく語の頭(「2.1 mmap」)。 */
301
+ const CONTINUES_WORD = /^[\p{Script=Latin}\p{Nd}_-]/u;
302
+
303
+ /** 「1.5 mM の塩化マグネシウム」「37.5 ℃で」: 解析器が助数詞と読まない単位の記号。 */
304
+ const startsWithMeasureUnit = (rest: string): boolean => MEASURE_UNITS.some((unit) => rest.startsWith(unit) && !CONTINUES_WORD.test(rest.slice(unit.length)));
284
305
 
285
306
  /** 辞書が桁の語と助数詞に切る一語(「1.5 万葉の世界」)は数量の続きではないので、行頭の番号は節のまま。 */
286
- const countedAfterNumber = (number: string, rest: string): boolean => !NOT_MAGNITUDE.some((word) => rest.startsWith(word)) && countedAfter(number, rest);
307
+ const countedAfterNumber = (number: string, rest: string): boolean =>
308
+ startsWithMeasureUnit(rest) || (!NOT_MAGNITUDE.some((word) => rest.startsWith(word)) && countedAfter(number, rest));
287
309
 
288
310
  const sectionsOf = (text: string): Mention[] => sectionReferences(text, SECTIONS);
289
311