@chaffjs/lang-ja 0.12.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/dist/analyser-pieces.d.ts +6 -0
  2. package/dist/analyser-pieces.d.ts.map +1 -0
  3. package/dist/analyser-pieces.js +37 -0
  4. package/dist/analyser-pieces.js.map +1 -0
  5. package/dist/counter-tsu.d.ts +18 -0
  6. package/dist/counter-tsu.d.ts.map +1 -0
  7. package/dist/counter-tsu.js +29 -0
  8. package/dist/counter-tsu.js.map +1 -0
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +3 -2
  11. package/dist/index.js.map +1 -1
  12. package/dist/particle-after.d.ts +7 -0
  13. package/dist/particle-after.d.ts.map +1 -0
  14. package/dist/particle-after.js +9 -0
  15. package/dist/particle-after.js.map +1 -0
  16. package/dist/passive-reading.d.ts +24 -0
  17. package/dist/passive-reading.d.ts.map +1 -0
  18. package/dist/passive-reading.js +90 -0
  19. package/dist/passive-reading.js.map +1 -0
  20. package/dist/pos.d.ts +2 -0
  21. package/dist/pos.d.ts.map +1 -1
  22. package/dist/pos.js +75 -29
  23. package/dist/pos.js.map +1 -1
  24. package/dist/quantities.d.ts +7 -1
  25. package/dist/quantities.d.ts.map +1 -1
  26. package/dist/quantities.js +47 -9
  27. package/dist/quantities.js.map +1 -1
  28. package/dist/reduplication.d.ts +3 -1
  29. package/dist/reduplication.d.ts.map +1 -1
  30. package/dist/reduplication.js +16 -1
  31. package/dist/reduplication.js.map +1 -1
  32. package/dist/structure.d.ts.map +1 -1
  33. package/dist/structure.js +9 -2
  34. package/dist/structure.js.map +1 -1
  35. package/dist/tokens-within.d.ts +7 -0
  36. package/dist/tokens-within.d.ts.map +1 -0
  37. package/dist/tokens-within.js +29 -0
  38. package/dist/tokens-within.js.map +1 -0
  39. package/dist/well-formed.d.ts +3 -0
  40. package/dist/well-formed.d.ts.map +1 -0
  41. package/dist/well-formed.js +3 -0
  42. package/dist/well-formed.js.map +1 -0
  43. package/lexicons/common-acronym.yaml +4 -0
  44. package/lexicons/date-time-unit.yaml +11 -0
  45. package/lexicons/definition-verb.yaml +5 -1
  46. package/lexicons/distributive-noun.yaml +2 -1
  47. package/lexicons/doubled-word.yaml +5 -0
  48. package/lexicons/emphasis-word.yaml +1 -0
  49. package/lexicons/honorific-formula.yaml +10 -0
  50. package/lexicons/honorific.yaml +15 -0
  51. package/lexicons/http-method.yaml +14 -0
  52. package/lexicons/intransitive-verb.yaml +18 -0
  53. package/lexicons/naming-verb.yaml +9 -0
  54. package/lexicons/not-magnitude.yaml +12 -0
  55. package/lexicons/numbered-division.yaml +24 -0
  56. package/lexicons/quantity-noun.yaml +42 -0
  57. package/lexicons/spontaneous-verb.yaml +18 -0
  58. package/lexicons/stative-passive-verb.yaml +8 -0
  59. package/package.json +1 -1
  60. package/src/analyser-pieces.ts +38 -0
  61. package/src/counter-tsu.ts +60 -0
  62. package/src/index.ts +3 -5
  63. package/src/particle-after.ts +11 -0
  64. package/src/passive-reading.ts +125 -0
  65. package/src/pos.ts +77 -39
  66. package/src/quantities.ts +62 -10
  67. package/src/reduplication.ts +24 -4
  68. package/src/structure.ts +11 -2
  69. package/src/tokens-within.ts +27 -0
  70. package/src/well-formed.ts +2 -0
package/src/quantities.ts CHANGED
@@ -29,6 +29,7 @@ const UNITS = [
29
29
  "名",
30
30
  "倍",
31
31
  "個",
32
+ "つ",
32
33
  "%",
33
34
  "%",
34
35
  ];
@@ -64,6 +65,18 @@ const runEnd = (morphs: readonly Morph[], index: number): number => {
64
65
  /** 「第3条」の 3 は番号で、数量ではない。 */
65
66
  const isOrdinal = (morph: Morph | undefined): boolean => morph?.surface === "第";
66
67
 
68
+ /** 助数詞の後ろの「目」「め」(3つ目、2回目、1年目、一つめ)も順番で、数量ではない。「目標」は一語なので当たらない。 */
69
+ const ORDINAL_SUFFIXES: ReadonlySet<string> = new Set(["目", "め"]);
70
+
71
+ const isOrdinalSuffix = (morph: Morph | undefined): boolean => morph?.pos === "名詞" && ORDINAL_SUFFIXES.has(morph.surface);
72
+
73
+ /**
74
+ * 解析器の無いときに順番と読む単位の後ろ。漢字の続かない「目」(「2回目標」「5人目線」の目は次の語の頭)と、
75
+ * 「めど」でない「め」(「10日めどに」は期限)。「つ」の後ろの漢字は見ない。「3つ選ぶ」のように数えた後ろに漢字が来るほうが、
76
+ * 「三つ巴」のような一語より多い。
77
+ */
78
+ const ORDINAL_BY_TABLE = /^(?:目(?!\p{Script=Han})|め(?!ど))/u;
79
+
67
80
  /** 数と単位のあいだに置かれうる空白 1 文字。構造の型(structure.ts の SPACE)と同じく、全角空白とタブも含む。 */
68
81
  const GAP = new Set([" ", "\t", "\u3000"]);
69
82
 
@@ -71,23 +84,32 @@ const isSpace = (morph: Morph | undefined): morph is Morph => morph !== undefine
71
84
 
72
85
  const SPACES = /[ \t\u3000]/gu;
73
86
 
87
+ /** 数量の後ろを読むのに要る文字数。「万人が」「億円の」まで見えれば足りる。 */
88
+ const LOOKAHEAD = 8;
89
+
74
90
  /**
75
91
  * 「1.5 倍」の「倍」は、数とのあいだに空白があると解析器が普通の名詞と読む。
76
92
  * 空白を詰めて読み直し、数の直後に来る語が助数詞ならその語を返す。
77
93
  */
94
+ const rereadAfterSpace = (number: string, rest: string): Morph[] => morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
95
+
78
96
  const unitAfterSpace = (number: string, rest: string): string | undefined => {
79
- const next = morphemes(toHalfWidth(number + rest.slice(0, 8)))?.find((morph) => morph.start >= number.length);
97
+ const next = rereadAfterSpace(number, rest).find((morph) => morph.start >= number.length);
80
98
  return next !== undefined && isCounter(next) ? next.surface : undefined;
81
99
  };
82
100
 
83
- type Unit = { readonly unit: string; readonly end: number };
101
+ /** ordinal: 単位のすぐ後ろが順番の「目」「め」。空白を詰めて読み直したときは、読み直した語で見る(「3 つめ」の「つめ」は詰めると つ + め)。 */
102
+ type Unit = { readonly unit: string; readonly end: number; readonly ordinal: boolean };
84
103
 
85
104
  const counterAt = (text: string, morphs: readonly Morph[], number: string, end: number): Unit | undefined => {
86
105
  const next = morphs[end];
87
- if (next !== undefined && isCounter(next)) return { unit: next.surface, end: next.end };
106
+ if (next !== undefined && isCounter(next)) return { unit: next.surface, end: next.end, ordinal: isOrdinalSuffix(morphs[end + 1]) };
88
107
  if (!isSpace(next)) return undefined;
89
- const unit = unitAfterSpace(number, text.slice(next.end));
90
- return unit === undefined ? undefined : { unit, end: next.end + unit.length };
108
+ const rest = text.slice(next.end);
109
+ const unit = unitAfterSpace(number, rest);
110
+ if (unit === undefined) return undefined;
111
+ const ordinal = isOrdinalSuffix(rereadAfterSpace(number, rest).find((morph) => morph.start === number.length + unit.length));
112
+ return { unit, end: next.end + unit.length, ordinal };
91
113
  };
92
114
 
93
115
  const countedByMorphemes = (text: string, morphs: readonly Morph[]): Counted[] => {
@@ -100,7 +122,7 @@ const countedByMorphemes = (text: string, morphs: readonly Morph[]): Counted[] =
100
122
  const number = text.slice(first.start, morphs[end - 1]?.end ?? first.end).replace(SPACES, "");
101
123
  const value = parseJapaneseNumber(number);
102
124
  const counter = counterAt(text, morphs, number, end);
103
- if (value !== undefined && counter !== undefined) found.push({ start: first.start, end: counter.end, value, unit: counter.unit });
125
+ if (value !== undefined && counter !== undefined && !counter.ordinal) found.push({ start: first.start, end: counter.end, value, unit: counter.unit });
104
126
  }
105
127
  index = Math.max(end, index + 1);
106
128
  }
@@ -116,7 +138,7 @@ export const countedByTable = (text: string): Counted[] =>
116
138
  const after = match.index + match[0].length + (GAP.has(text[match.index + match[0].length] ?? "") ? 1 : 0);
117
139
  const unit = UNITS.find((candidate) => text.startsWith(candidate, after));
118
140
  const value = parseJapaneseNumber(toHalfWidth(match[0].replace(SPACES, "")));
119
- const ordinal = text[match.index - 1] === "第";
141
+ const ordinal = text[match.index - 1] === "第" || (unit !== undefined && ORDINAL_BY_TABLE.test(text.slice(after + unit.length)));
120
142
  return unit === undefined || value === undefined || ordinal ? [] : [{ start: match.index, end: after + unit.length, value, unit }];
121
143
  });
122
144
 
@@ -227,11 +249,41 @@ export const dates = (text: string): Mention[] =>
227
249
  return weekday === undefined ? date : { ...date, attrs: { ...date.attrs, weekday } };
228
250
  });
229
251
 
252
+ /** 数がそこで閉じる。助数詞(「万人」)か、名詞でない語(「万を」「万。」)か、行の終わり。 */
253
+ const closesNumber = (morph: Morph | undefined): boolean => morph === undefined || isCounter(morph) || morph.pos !== "名詞";
254
+
255
+ /**
256
+ * 桁の語が一語として続く。「1.5 万を超える」のように単位が無くても数の一部。
257
+ * 「万葉集」は一語で、「万葉の」「万一」は桁の語の後ろに名詞や数が続くので、どれも数の続きではない。
258
+ */
259
+ export const continuesWithMultiplier = (morphs: readonly Morph[], index: number): boolean =>
260
+ MULTIPLIERS.has(morphs[index]?.surface ?? "") && isNumeral(morphs[index]) && closesNumber(morphs[index + 1]);
261
+
262
+ const multiplierAfter = (number: string, rest: string): boolean => {
263
+ const morphs = morphemes(toHalfWidth(number + rest.slice(0, LOOKAHEAD))) ?? [];
264
+ return continuesWithMultiplier(
265
+ morphs,
266
+ morphs.findIndex((morph) => morph.start >= number.length),
267
+ );
268
+ };
269
+
270
+ const startsWithUnit = (text: string): boolean => UNITS.some((unit) => text.startsWith(unit));
271
+
272
+ /** 解析器の無いときの、数がそこで閉じる字。行の終わり、表の単位、仮名の助詞や句読点(「万を」「万。」)。漢字(「万葉」「万全」)は閉じない。 */
273
+ const CLOSES_BY_TABLE = /^(?:$|[\p{Script=Hiragana}\p{P}\s])/u;
274
+
275
+ /** 解析器の無いときの読み方。桁の語が先にあれば、その後ろで数が閉じるかを見る。 */
276
+ const countedByTableAfter = (rest: string): boolean => {
277
+ if (!MULTIPLIERS.has(rest[0] ?? "")) return startsWithUnit(rest);
278
+ const after = rest.slice(1);
279
+ return startsWithUnit(after) || CLOSES_BY_TABLE.test(after);
280
+ };
281
+
230
282
  /**
231
- * 「1.5 倍になった。」の 1.5 は章番号ではない。番号と続く語のあいだの空白を詰めて読み直し、
283
+ * 「1.5 倍になった。」「1.5 万人が参加した。」の 1.5 は章番号ではない。番号と続く語のあいだの空白を詰めて読み直し、
232
284
  * 続く語が助数詞なら数量の一部と分かる。空白のあるままだと、解析器は「倍」を普通の名詞と読む。
233
285
  */
234
286
  export const countedAfter = (number: string, rest: string): boolean => {
235
- if (!isReady()) return UNITS.some((unit) => rest.startsWith(unit));
236
- return unitAfterSpace(number, rest) !== undefined;
287
+ if (!isReady()) return countedByTableAfter(rest);
288
+ return unitAfterSpace(number, rest) !== undefined || multiplierAfter(number, rest);
237
289
  };
@@ -26,7 +26,27 @@ const isParticleAfter = (next: Token | undefined, second: Token, particles: Read
26
26
 
27
27
  const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
28
28
 
29
- export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive): Token[] =>
30
- tokens.map((token, index) =>
31
- isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token,
32
- );
29
+ const isDistributive = (tokens: readonly Token[], index: number, vocabulary: Distributive): boolean => {
30
+ const token = tokens[index];
31
+ return token !== undefined && isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles);
32
+ };
33
+
34
+ /** 文字列を解析器が一語の副詞と読むか。解析器を持つ index.ts が渡す。 */
35
+ export type ReadsAsAdverb = (text: string) => boolean;
36
+
37
+ const CONTENT_WORD = new Set(["NOUN", "VERB", "ADJ"]);
38
+
39
+ /**
40
+ * 重ね言葉の副詞(がんがん)は、前の語に引かれると同じ名詞二つに切られる(労働者ががんがん → が・がん・がん)。
41
+ * 二語を続けて読み直して一語の副詞なら、重ね言葉。機能語の重なりは読み直さない。書き損じの「行ったたので」の「たた」も一語の副詞に読める。
42
+ */
43
+ const isSplitAdverb = (first: Token | undefined, second: Token, readsAsAdverb: ReadsAsAdverb): boolean =>
44
+ first !== undefined &&
45
+ first.surface === second.surface &&
46
+ first.pos === second.pos &&
47
+ CONTENT_WORD.has(second.pos) &&
48
+ first.span.end === second.span.start &&
49
+ readsAsAdverb(`${first.surface}${second.surface}`);
50
+
51
+ export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive, readsAsAdverb: ReadsAsAdverb): Token[] =>
52
+ tokens.map((token, index) => (isDistributive(tokens, index, vocabulary) || isSplitAdverb(tokens[index - 1], token, readsAsAdverb) ? echoed(token) : token));
package/src/structure.ts CHANGED
@@ -4,6 +4,7 @@ import { citationVocabulary, citedDocument } from "./citation.ts";
4
4
  import { loadLexicons } from "./lexicons.ts";
5
5
  import { countedAfter, dates, quantities } from "./quantities.ts";
6
6
  import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
7
+ import { startsWithParticle } from "./particle-after.ts";
7
8
 
8
9
  // 契約書・規程・法令の番号の書き方。core は番号の書き方を知らず、ここで読んだものを入れ子にする。
9
10
 
@@ -202,6 +203,9 @@ const definitions = (text: string): Mention[] =>
202
203
  }),
203
204
  );
204
205
 
206
+ /** 条を数える語。見出しに「1 目的」と番号を振っただけの文書には、この語で数えた条が無い。 */
207
+ const ARTICLE_UNIT = "条";
208
+
205
209
  const REFERENCE = new RegExp(`第(?<a>${NUMBER})条(?:の(?<s>${NUMBER}))?(?:第(?<p>${NUMBER})項)?(?:第(?<i>${NUMBER})号)?`, "gu");
206
210
 
207
211
  /**
@@ -266,6 +270,10 @@ const addressOfReference = (
266
270
  const LEXICONS = loadLexicons();
267
271
  const CITATION = citationVocabulary(LEXICONS);
268
272
  const SECTIONS = sectionVocabulary(LEXICONS);
273
+ const NOT_MAGNITUDE = (LEXICONS["not-magnitude"] ?? []).map((entry) => entry.pattern);
274
+
275
+ /** 辞書が桁の語と助数詞に切る一語(「1.5 万葉の世界」)は数量の続きではないので、行頭の番号は節のまま。 */
276
+ const countedAfterNumber = (number: string, rest: string): boolean => !NOT_MAGNITUDE.some((word) => rest.startsWith(word)) && countedAfter(number, rest);
269
277
 
270
278
  const sectionsOf = (text: string): Mention[] => sectionReferences(text, SECTIONS);
271
279
 
@@ -274,7 +282,7 @@ const articlesOf = (text: string): Mention[] =>
274
282
  const address = addressOfReference(match.groups ?? {});
275
283
  if (address === undefined) return [];
276
284
  const { target, fallback } = address;
277
- const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }) };
285
+ const attrs = { target, label: match[0], unitWord: ARTICLE_UNIT, ...(fallback === undefined ? {} : { fallback }) };
278
286
  return [{ start: match.index, end: match.index + match[0].length, attrs }];
279
287
  });
280
288
 
@@ -355,6 +363,7 @@ export const structure: StructurePatterns = {
355
363
  obligations,
356
364
  quantities: quantitiesOutsideSections,
357
365
  dates,
358
- countedAfter,
366
+ countedAfter: countedAfterNumber,
367
+ continuesSentence: startsWithParticle,
359
368
  number,
360
369
  };
@@ -0,0 +1,27 @@
1
+ import type { Span, Token } from "chaffjs/plugin";
2
+
3
+ /** start が from 以上になる最初の位置。tokens は start の昇順。 */
4
+ const firstFrom = (tokens: readonly Token[], from: number): number => {
5
+ let low = 0;
6
+ let high = tokens.length;
7
+ while (low < high) {
8
+ const middle = (low + high) >> 1;
9
+ if ((tokens[middle]?.span.start ?? Number.POSITIVE_INFINITY) < from) low = middle + 1;
10
+ else high = middle;
11
+ }
12
+ return low;
13
+ };
14
+
15
+ /**
16
+ * span に収まる token。tokens は start の昇順(解析器が本文の頭から出した順)。
17
+ * 文ごとに全 token をなめると、空行の無い何千行もの .txt で文の数 × token の数になる。
18
+ */
19
+ export const tokensWithin = (tokens: readonly Token[], span: Span): Token[] => {
20
+ const found: Token[] = [];
21
+ for (let at = firstFrom(tokens, span.start); at < tokens.length; at += 1) {
22
+ const token = tokens[at];
23
+ if (token === undefined || token.span.start > span.end) break;
24
+ if (token.span.end <= span.end) found.push(token);
25
+ }
26
+ return found;
27
+ };
@@ -0,0 +1,2 @@
1
+ /** 対になっていないサロゲートを U+FFFD に置き換える。長さを変えないので、位置はそのまま同じ文字を指す。 */
2
+ export const wellFormed = (text: string): string => text.replace(/\p{Surrogate}/gu, "\uFFFD");