@chaffjs/lang-ja 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/dist/analyser-pieces.d.ts +6 -0
  2. package/dist/analyser-pieces.d.ts.map +1 -0
  3. package/dist/analyser-pieces.js +37 -0
  4. package/dist/analyser-pieces.js.map +1 -0
  5. package/dist/counter-tsu.d.ts +1 -0
  6. package/dist/counter-tsu.d.ts.map +1 -1
  7. package/dist/counter-tsu.js.map +1 -1
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +3 -2
  10. package/dist/index.js.map +1 -1
  11. package/dist/particle-after.d.ts +7 -0
  12. package/dist/particle-after.d.ts.map +1 -0
  13. package/dist/particle-after.js +9 -0
  14. package/dist/particle-after.js.map +1 -0
  15. package/dist/passive-reading.d.ts +24 -0
  16. package/dist/passive-reading.d.ts.map +1 -0
  17. package/dist/passive-reading.js +90 -0
  18. package/dist/passive-reading.js.map +1 -0
  19. package/dist/pos.d.ts +2 -0
  20. package/dist/pos.d.ts.map +1 -1
  21. package/dist/pos.js +56 -19
  22. package/dist/pos.js.map +1 -1
  23. package/dist/reduplication.d.ts +3 -1
  24. package/dist/reduplication.d.ts.map +1 -1
  25. package/dist/reduplication.js +16 -1
  26. package/dist/reduplication.js.map +1 -1
  27. package/dist/structure.d.ts.map +1 -1
  28. package/dist/structure.js +5 -1
  29. package/dist/structure.js.map +1 -1
  30. package/dist/tokens-within.d.ts +7 -0
  31. package/dist/tokens-within.d.ts.map +1 -0
  32. package/dist/tokens-within.js +29 -0
  33. package/dist/tokens-within.js.map +1 -0
  34. package/dist/well-formed.d.ts +3 -0
  35. package/dist/well-formed.d.ts.map +1 -0
  36. package/dist/well-formed.js +3 -0
  37. package/dist/well-formed.js.map +1 -0
  38. package/lexicons/common-acronym.yaml +4 -0
  39. package/lexicons/date-time-unit.yaml +11 -0
  40. package/lexicons/distributive-noun.yaml +2 -1
  41. package/lexicons/doubled-word.yaml +5 -0
  42. package/lexicons/emphasis-word.yaml +1 -0
  43. package/lexicons/honorific-formula.yaml +10 -0
  44. package/lexicons/honorific.yaml +15 -0
  45. package/lexicons/http-method.yaml +14 -0
  46. package/lexicons/intransitive-verb.yaml +18 -0
  47. package/lexicons/naming-verb.yaml +9 -0
  48. package/lexicons/quantity-noun.yaml +42 -0
  49. package/lexicons/spontaneous-verb.yaml +18 -0
  50. package/lexicons/stative-passive-verb.yaml +8 -0
  51. package/package.json +1 -1
  52. package/src/analyser-pieces.ts +38 -0
  53. package/src/counter-tsu.ts +1 -0
  54. package/src/index.ts +3 -5
  55. package/src/particle-after.ts +11 -0
  56. package/src/passive-reading.ts +125 -0
  57. package/src/pos.ts +59 -19
  58. package/src/reduplication.ts +24 -4
  59. package/src/structure.ts +6 -1
  60. package/src/tokens-within.ts +27 -0
  61. package/src/well-formed.ts +2 -0
package/src/pos.ts CHANGED
@@ -1,5 +1,9 @@
1
1
  import { surfaceStarts } from "./surface-starts.ts";
2
+ import { wellFormed } from "./well-formed.ts";
3
+ import { analyserPieces } from "./analyser-pieces.ts";
2
4
  import { readCounterTsu, type Morpheme } from "./counter-tsu.ts";
5
+ import { isPassiveForm, passiveVocabulary, readsAsPassive } from "./passive-reading.ts";
6
+ import { loadLexicons } from "./lexicons.ts";
3
7
  import { createRequire } from "node:module";
4
8
  import { dirname, join } from "node:path";
5
9
  import type { Token } from "chaffjs/plugin";
@@ -22,20 +26,31 @@ const isCallable = (value: unknown): value is (...args: readonly unknown[]) => u
22
26
 
23
27
  const isTokenizer = (value: unknown): value is Tokenizer => isRecord(value) && isCallable(value["tokenize"]);
24
28
 
25
- /** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ *。 */
29
+ /** kuromoji の形態素。形の違うものは落とす。二段目の細分類(助数詞・地域)は無ければ *。三段目(人名の姓・名)は無ければ持たない。 */
26
30
  const toMorpheme = (value: unknown): Morpheme[] => {
27
31
  if (!isRecord(value)) return [];
28
- const [surface, pos, detail1, detail2, basic, reading] = [
32
+ const [surface, pos, detail1, detail2, detail3, basic, reading] = [
29
33
  value["surface_form"],
30
34
  value["pos"],
31
35
  value["pos_detail_1"],
32
36
  value["pos_detail_2"],
37
+ value["pos_detail_3"],
33
38
  value["basic_form"],
34
39
  value["reading"],
35
40
  ];
36
41
  if (typeof surface !== "string" || typeof pos !== "string" || typeof detail1 !== "string" || typeof basic !== "string") return [];
37
42
  const detail = typeof detail2 === "string" ? detail2 : "*";
38
- return [{ surface_form: surface, pos, pos_detail_1: detail1, pos_detail_2: detail, basic_form: basic, ...(typeof reading === "string" ? { reading } : {}) }];
43
+ return [
44
+ {
45
+ surface_form: surface,
46
+ pos,
47
+ pos_detail_1: detail1,
48
+ pos_detail_2: detail,
49
+ ...(typeof detail3 === "string" ? { pos_detail_3: detail3 } : {}),
50
+ basic_form: basic,
51
+ ...(typeof reading === "string" ? { reading } : {}),
52
+ },
53
+ ];
39
54
  };
40
55
 
41
56
  const dictionaryPath = (): string => join(dirname(require.resolve("@sglkc/kuromoji/package.json")), "dict");
@@ -89,13 +104,10 @@ const BY_POS: Readonly<Record<string, string>> = {
89
104
  export const upos = (pos: string, detail: string): string => BY_DETAIL[detail] ?? BY_POS[pos] ?? "X";
90
105
 
91
106
  /**
92
- * 受動の「れる/られる」。IPADIC では動詞の接尾として出る。
93
- * ただしこの語は可能・尊敬・自発も表す。区別は文脈が要るのでここではしない。
94
- * 受動と言い切らず「受動の形」として印を付け、どう扱うかは rule 側に委ねる。
107
+ * 受動の「れる/られる」。この語は可能・尊敬・自発も表す。形のうえで受動でないと言えるもの
108
+ * (自発や関係を表す動詞、名付けの「と呼ばれる」、尊敬の形と決まり文句)は passive-reading.ts が外す。残りは「受動の形」として印を付ける。
95
109
  */
96
- const PASSIVE_LEMMA = new Set(["れる", "られる"]);
97
-
98
- const isPassive = (morpheme: Morpheme): boolean => morpheme.pos === "動詞" && morpheme.pos_detail_1 === "接尾" && PASSIVE_LEMMA.has(morpheme.basic_form);
110
+ const PASSIVE_VOCABULARY = passiveVocabulary(loadLexicons());
99
111
 
100
112
  /**
101
113
  * 非自立名詞(の・こと・もの・ため・はず)。品詞は名詞だが、単独では何も指さない。
@@ -128,14 +140,14 @@ export const isReady = (): boolean => state.ready !== undefined;
128
140
  * span は渡した文字列の先頭を 0 とする。位置は kuromoji の word_position ではなく、語の文字を本文と照らして決める(surface-starts.ts)。
129
141
  * 形の違うものが混ざったら、その 1 つを落とす。位置が NaN の token を下流に流さない。
130
142
  */
131
- const toToken = (morpheme: Morpheme, start: number): Token => ({
143
+ const toToken = (morpheme: Morpheme, start: number, passive: boolean): Token => ({
132
144
  span: { start, end: start + morpheme.surface_form.length },
133
145
  surface: morpheme.surface_form,
134
146
  // UD の日本語では「れる/られる」は AUX。IPADIC の「動詞,接尾」をそこへ寄せる。
135
- pos: isPassive(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
147
+ pos: isPassiveForm(morpheme) ? "AUX" : upos(morpheme.pos, morpheme.pos_detail_1),
136
148
  ...(morpheme.basic_form === "*" ? {} : { lemma: morpheme.basic_form }),
137
149
  ...(typeof morpheme.reading !== "string" || morpheme.reading === "*" ? {} : { reading: morpheme.reading }),
138
- ...featuresOf(morpheme),
150
+ ...featuresOf(morpheme, passive),
139
151
  });
140
152
 
141
153
  /**
@@ -157,15 +169,25 @@ const placeType = (morpheme: Morpheme): string | undefined => {
157
169
  return morpheme.pos_detail_1 === "接尾" ? "GeoUnit" : undefined;
158
170
  };
159
171
 
172
+ /** 人名(IPADIC の「固有名詞,人名」)の三段目。姓は Sur、名は Giv、どちらとも言えない人名は Prs(UD の NameType)。 */
173
+ const PERSON: Readonly<Record<string, string>> = { 姓: "Sur", 名: "Giv", 一般: "Prs" };
174
+
175
+ /** 人名と組織名(「固有名詞,組織」は UD の NameType=Com)。地名は placeType が見る。 */
176
+ const personOrOrganisation = (morpheme: Morpheme): string | undefined => {
177
+ if (morpheme.pos !== "名詞" || morpheme.pos_detail_1 !== "固有名詞") return undefined;
178
+ if (morpheme.pos_detail_2 === "人名") return PERSON[morpheme.pos_detail_3 ?? ""];
179
+ return morpheme.pos_detail_2 === "組織" ? "Com" : undefined;
180
+ };
181
+
160
182
  /** 数を数える単位(IPADIC の「接尾,助数詞」: 丁目・件・人)。UD では NounType=Class。 */
161
183
  const isCounter = (morpheme: Morpheme): boolean => morpheme.pos === "名詞" && morpheme.pos_detail_1 === "接尾" && morpheme.pos_detail_2 === "助数詞";
162
184
 
163
- const featuresOf = (morpheme: Morpheme): { features?: Readonly<Record<string, string>> } => {
164
- if (isPassive(morpheme)) return { features: { Voice: "Pass" } };
185
+ const featuresOf = (morpheme: Morpheme, passive: boolean): { features?: Readonly<Record<string, string>> } => {
186
+ if (passive) return { features: { Voice: "Pass" } };
165
187
  if (isDependentNoun(morpheme)) return { features: { NounType: "Dependent" } };
166
188
  if (isNumeral(morpheme)) return { features: { NumType: "Card" } };
167
- const place = placeType(morpheme);
168
- if (place !== undefined) return { features: { NameType: place } };
189
+ const name = placeType(morpheme) ?? personOrOrganisation(morpheme);
190
+ if (name !== undefined) return { features: { NameType: name } };
169
191
  if (isCounter(morpheme)) return { features: { NounType: "Class" } };
170
192
  return {};
171
193
  };
@@ -201,10 +223,22 @@ export const predicateOnly = (tokens: readonly Token[]): Token[] =>
201
223
  export const tokenize = (text: string): Token[] | undefined => {
202
224
  const tokenizer = state.ready;
203
225
  if (tokenizer === undefined) return undefined;
204
- return placed(text, analyse(tokenizer, text)).map(({ morpheme, start }) => toToken(morpheme, start));
226
+ const read = readAll(tokenizer, text);
227
+ const sequence = read.map(({ morpheme }) => morpheme);
228
+ return read.map(({ morpheme, start }, index) => toToken(morpheme, start, readsAsPassive(sequence, index, PASSIVE_VOCABULARY)));
205
229
  };
206
230
 
207
- const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] => readCounterTsu(toArray(callMethod(tokenizer, "tokenize", [text])).flatMap(toMorpheme));
231
+ /** 句読点の無い並びをこれより長く渡さない。解析器は並びの長さの二乗で遅くなり、20 万字の 1 行では何分も返らない。 */
232
+ const PIECE_LIMIT = 1000;
233
+
234
+ const analyse = (tokenizer: Tokenizer, text: string): Morpheme[] =>
235
+ readCounterTsu(analyserPieces(text, PIECE_LIMIT).flatMap((piece) => toArray(callMethod(tokenizer, "tokenize", [piece])).flatMap(toMorpheme)));
236
+
237
+ /** 解析器は片割れのサロゲートで例外を投げる。数量の後ろを数文字だけ読み直すと、絵文字を半分に切ることがある。 */
238
+ const readAll = (tokenizer: Tokenizer, text: string): { readonly morpheme: Morpheme; readonly start: number }[] => {
239
+ const readable = wellFormed(text);
240
+ return placed(readable, analyse(tokenizer, readable));
241
+ };
208
242
 
209
243
  /** 形態素と、本文の中での始まり。本文に見つからないものは落とす。 */
210
244
  const placed = (text: string, raws: readonly Morpheme[]): { readonly morpheme: Morpheme; readonly start: number }[] => {
@@ -235,7 +269,7 @@ export type Morph = {
235
269
  export const morphemes = (text: string): Morph[] | undefined => {
236
270
  const tokenizer = state.ready;
237
271
  if (tokenizer === undefined) return undefined;
238
- return placed(text, analyse(tokenizer, text)).map(({ morpheme: raw, start }) => ({
272
+ return readAll(tokenizer, text).map(({ morpheme: raw, start }) => ({
239
273
  start,
240
274
  end: start + raw.surface_form.length,
241
275
  surface: raw.surface_form,
@@ -250,6 +284,12 @@ const isCounterMorph = (morph: Morph): boolean => morph.pos === "名詞" && morp
250
284
  /** 桁の語(万・億)。「26.7 万行」の「万行」は、詰めれば 26.7万 と 行 に分かれる。 */
251
285
  const isNumeralMorph = (morph: Morph): boolean => morph.pos === "名詞" && morph.detail1 === "数";
252
286
 
287
+ /** 一語の副詞と読むか(がんがん)。前後の語に引かれない、その文字列だけの読み。 */
288
+ export const readsAsOneAdverb = (text: string): boolean => {
289
+ const read = tokenize(text);
290
+ return read?.length === 1 && read[0]?.pos === "ADV";
291
+ };
292
+
253
293
  /**
254
294
  * 数のすぐ後ろに詰めて書いたとき、word が数につく語として読まれるか(spaced-counter.ts が使う)。
255
295
  * word がそのまま一語の助数詞になるか、桁の語で始まるとき。「件名」は詰めると 件 と 名 に割れるので、助数詞とは読まない。
@@ -26,7 +26,27 @@ const isParticleAfter = (next: Token | undefined, second: Token, particles: Read
26
26
 
27
27
  const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
28
28
 
29
- export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive): Token[] =>
30
- tokens.map((token, index) =>
31
- isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token,
32
- );
29
+ const isDistributive = (tokens: readonly Token[], index: number, vocabulary: Distributive): boolean => {
30
+ const token = tokens[index];
31
+ return token !== undefined && isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles);
32
+ };
33
+
34
+ /** 文字列を解析器が一語の副詞と読むか。解析器を持つ index.ts が渡す。 */
35
+ export type ReadsAsAdverb = (text: string) => boolean;
36
+
37
+ const CONTENT_WORD = new Set(["NOUN", "VERB", "ADJ"]);
38
+
39
+ /**
40
+ * 重ね言葉の副詞(がんがん)は、前の語に引かれると同じ名詞二つに切られる(労働者ががんがん → が・がん・がん)。
41
+ * 二語を続けて読み直して一語の副詞なら、重ね言葉。機能語の重なりは読み直さない。書き損じの「行ったたので」の「たた」も一語の副詞に読める。
42
+ */
43
+ const isSplitAdverb = (first: Token | undefined, second: Token, readsAsAdverb: ReadsAsAdverb): boolean =>
44
+ first !== undefined &&
45
+ first.surface === second.surface &&
46
+ first.pos === second.pos &&
47
+ CONTENT_WORD.has(second.pos) &&
48
+ first.span.end === second.span.start &&
49
+ readsAsAdverb(`${first.surface}${second.surface}`);
50
+
51
+ export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive, readsAsAdverb: ReadsAsAdverb): Token[] =>
52
+ tokens.map((token, index) => (isDistributive(tokens, index, vocabulary) || isSplitAdverb(tokens[index - 1], token, readsAsAdverb) ? echoed(token) : token));
package/src/structure.ts CHANGED
@@ -4,6 +4,7 @@ import { citationVocabulary, citedDocument } from "./citation.ts";
4
4
  import { loadLexicons } from "./lexicons.ts";
5
5
  import { countedAfter, dates, quantities } from "./quantities.ts";
6
6
  import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
7
+ import { startsWithParticle } from "./particle-after.ts";
7
8
 
8
9
  // 契約書・規程・法令の番号の書き方。core は番号の書き方を知らず、ここで読んだものを入れ子にする。
9
10
 
@@ -202,6 +203,9 @@ const definitions = (text: string): Mention[] =>
202
203
  }),
203
204
  );
204
205
 
206
+ /** 条を数える語。見出しに「1 目的」と番号を振っただけの文書には、この語で数えた条が無い。 */
207
+ const ARTICLE_UNIT = "条";
208
+
205
209
  const REFERENCE = new RegExp(`第(?<a>${NUMBER})条(?:の(?<s>${NUMBER}))?(?:第(?<p>${NUMBER})項)?(?:第(?<i>${NUMBER})号)?`, "gu");
206
210
 
207
211
  /**
@@ -278,7 +282,7 @@ const articlesOf = (text: string): Mention[] =>
278
282
  const address = addressOfReference(match.groups ?? {});
279
283
  if (address === undefined) return [];
280
284
  const { target, fallback } = address;
281
- const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }) };
285
+ const attrs = { target, label: match[0], unitWord: ARTICLE_UNIT, ...(fallback === undefined ? {} : { fallback }) };
282
286
  return [{ start: match.index, end: match.index + match[0].length, attrs }];
283
287
  });
284
288
 
@@ -360,5 +364,6 @@ export const structure: StructurePatterns = {
360
364
  quantities: quantitiesOutsideSections,
361
365
  dates,
362
366
  countedAfter: countedAfterNumber,
367
+ continuesSentence: startsWithParticle,
363
368
  number,
364
369
  };
@@ -0,0 +1,27 @@
1
+ import type { Span, Token } from "chaffjs/plugin";
2
+
3
+ /** start が from 以上になる最初の位置。tokens は start の昇順。 */
4
+ const firstFrom = (tokens: readonly Token[], from: number): number => {
5
+ let low = 0;
6
+ let high = tokens.length;
7
+ while (low < high) {
8
+ const middle = (low + high) >> 1;
9
+ if ((tokens[middle]?.span.start ?? Number.POSITIVE_INFINITY) < from) low = middle + 1;
10
+ else high = middle;
11
+ }
12
+ return low;
13
+ };
14
+
15
+ /**
16
+ * span に収まる token。tokens は start の昇順(解析器が本文の頭から出した順)。
17
+ * 文ごとに全 token をなめると、空行の無い何千行もの .txt で文の数 × token の数になる。
18
+ */
19
+ export const tokensWithin = (tokens: readonly Token[], span: Span): Token[] => {
20
+ const found: Token[] = [];
21
+ for (let at = firstFrom(tokens, span.start); at < tokens.length; at += 1) {
22
+ const token = tokens[at];
23
+ if (token === undefined || token.span.start > span.end) break;
24
+ if (token.span.end <= span.end) found.push(token);
25
+ }
26
+ return found;
27
+ };
@@ -0,0 +1,2 @@
1
+ /** 対になっていないサロゲートを U+FFFD に置き換える。長さを変えないので、位置はそのまま同じ文字を指す。 */
2
+ export const wellFormed = (text: string): string => text.replace(/\p{Surrogate}/gu, "\uFFFD");