@chaffjs/lang-ja 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +2 -0
- package/dist/lexicons.js.map +1 -1
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +48 -12
- package/dist/pos.js.map +1 -1
- package/dist/quantities.d.ts +8 -0
- package/dist/quantities.d.ts.map +1 -1
- package/dist/quantities.js +11 -5
- package/dist/quantities.js.map +1 -1
- package/dist/reduplication.d.ts +12 -0
- package/dist/reduplication.d.ts.map +1 -0
- package/dist/reduplication.js +15 -0
- package/dist/reduplication.js.map +1 -0
- package/dist/section-reference.d.ts +14 -0
- package/dist/section-reference.d.ts.map +1 -0
- package/dist/section-reference.js +37 -0
- package/dist/section-reference.js.map +1 -0
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +38 -15
- package/dist/structure.js.map +1 -1
- package/dist/surface-starts.d.ts +6 -0
- package/dist/surface-starts.d.ts.map +1 -0
- package/dist/surface-starts.js +15 -0
- package/dist/surface-starts.js.map +1 -0
- package/lexicons/ai-tell.yaml +3 -3
- package/lexicons/closing-cliche.yaml +1 -1
- package/lexicons/common-acronym.yaml +72 -0
- package/lexicons/comparison-marker.yaml +8 -0
- package/lexicons/date-stamp-label.yaml +14 -0
- package/lexicons/definition-marker.yaml +6 -0
- package/lexicons/definition-verb.yaml +8 -0
- package/lexicons/distributive-noun.yaml +25 -0
- package/lexicons/distributive-particle.yaml +11 -0
- package/lexicons/double-keigo.yaml +10 -10
- package/lexicons/doubled-word.yaml +10 -0
- package/lexicons/lead-in.yaml +10 -0
- package/lexicons/not-section-after.yaml +19 -0
- package/lexicons/not-section-before.yaml +11 -0
- package/lexicons/sasete-itadaku.yaml +3 -3
- package/lexicons/superlative-scope.yaml +7 -0
- package/package.json +1 -1
- package/src/index.ts +7 -1
- package/src/lexicons.ts +3 -0
- package/src/pos.ts +51 -11
- package/src/quantities.ts +15 -5
- package/src/reduplication.ts +32 -0
- package/src/section-reference.ts +56 -0
- package/src/structure.ts +44 -17
- package/src/surface-starts.ts +13 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { Lexicon, Mention } from "chaffjs/plugin";
|
|
2
|
+
import { toHalfWidth } from "./numbers.ts";
|
|
3
|
+
|
|
4
|
+
// 算用数字で番号を振った文書の中の参照(「3.2節」「第3章」「3.2.1項」)。番地は木の通し番号(### 3.2 → 3.2)と同じにする。
|
|
5
|
+
// 漢数字の「第三章」「第二節」は法令の書き方で、法令の番地は章ごと・編ごとに振り直すので、ここでは読まない。
|
|
6
|
+
|
|
7
|
+
export type SectionVocabulary = {
|
|
8
|
+
/** 番号の直前にあると番地でなくなる語(全3章・バージョン3.2)。 */
|
|
9
|
+
readonly notBefore: readonly string[];
|
|
10
|
+
/** 単位の字で始まる別の語(3.2節約・3項目)。 */
|
|
11
|
+
readonly notAfter: readonly string[];
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const patternsOf = (lexicon: Lexicon | undefined): string[] => (lexicon ?? []).map((entry) => entry.pattern);
|
|
15
|
+
|
|
16
|
+
export const sectionVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): SectionVocabulary => ({
|
|
17
|
+
notBefore: patternsOf(lexicons["not-section-before"]),
|
|
18
|
+
notAfter: patternsOf(lexicons["not-section-after"]),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
/** 部品は 3 桁まで・深さは 6 まで。木の通し番号と同じ上限。 */
|
|
22
|
+
const SECTION = /(?:第[ \u3000]?)?(?<n>[0-90-9]{1,3}(?:[..][0-90-9]{1,3}){0,5})[ \u3000]?(?<unit>[章節項])/gu;
|
|
23
|
+
|
|
24
|
+
/** 番号の一部になる字が前にあれば、長い番号(2024.3.2)や版(v3.2)の途中。 */
|
|
25
|
+
const NUMBER_CHAR = /[0-90-9..A-Za-zA-Za-z]/u;
|
|
26
|
+
|
|
27
|
+
/** 章は「3章」、節と項は「3.2節」「3.2.1項」。点の無い「2節」「第2項」は数か法令の番地で、点のある「3.2章」は書かない。 */
|
|
28
|
+
const hasDots = (unit: string): boolean => unit !== "章";
|
|
29
|
+
|
|
30
|
+
/** 行頭の「3.2節 データ」「3.2節(データ)」「### 3.2節」は、その行の見出しの番号。本文の「3.2節で述べた」は参照。 */
|
|
31
|
+
const isLineLabel = (text: string, start: number, end: number): boolean => /^[#\s]*$/u.test(text.slice(0, start)) && /^(?:[\s((::]|$)/u.test(text.slice(end));
|
|
32
|
+
|
|
33
|
+
/** 「全 3 章」のように数える語と番号のあいだに空白があっても、数える語。見る長さは語彙の語が収まる分だけ。 */
|
|
34
|
+
const BEFORE_WINDOW = 20;
|
|
35
|
+
const wordBefore = (text: string, start: number): string => text.slice(Math.max(0, start - BEFORE_WINDOW), start).trimEnd();
|
|
36
|
+
|
|
37
|
+
const blocked = (text: string, start: number, unitAt: number, vocabulary: SectionVocabulary): boolean =>
|
|
38
|
+
NUMBER_CHAR.test(text[start - 1] ?? "") ||
|
|
39
|
+
vocabulary.notBefore.some((word) => wordBefore(text, start).endsWith(word)) ||
|
|
40
|
+
vocabulary.notAfter.some((word) => text.startsWith(word, unitAt));
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* 「3.2節」→ 3.2、「第3章」「3章」→ ch3。見出し「## 第3章」は木に ch3 として入る。見出しを「## 3. 構成」と書く文書では 3 なので、番号だけの見出しの 3 を fallback にする(第3条には当てない)。
|
|
44
|
+
* 他の文書の名前(民法第3章)は、条の参照と一緒に structure.ts が付ける。
|
|
45
|
+
*/
|
|
46
|
+
export const sectionReferences = (text: string, vocabulary: SectionVocabulary): Mention[] =>
|
|
47
|
+
[...text.matchAll(SECTION)].flatMap((match) => {
|
|
48
|
+
const groups = match.groups ?? {};
|
|
49
|
+
const [written, unit] = [groups["n"] ?? "", groups["unit"] ?? ""];
|
|
50
|
+
const number = toHalfWidth(written).replace(/./gu, ".");
|
|
51
|
+
const end = match.index + match[0].length;
|
|
52
|
+
if (number.includes(".") !== hasDots(unit)) return [];
|
|
53
|
+
if (blocked(text, match.index, end - unit.length, vocabulary) || isLineLabel(text, match.index, end)) return [];
|
|
54
|
+
const address = unit === "章" ? { target: `ch${number}`, fallback: number, fallbackLabel: number } : { target: number };
|
|
55
|
+
return [{ start: match.index, end, attrs: { ...address, label: match[0] } }];
|
|
56
|
+
});
|
package/src/structure.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { parseJapaneseNumber, toHalfWidth } from "./numbers.ts";
|
|
|
3
3
|
import { citationVocabulary, citedDocument } from "./citation.ts";
|
|
4
4
|
import { loadLexicons } from "./lexicons.ts";
|
|
5
5
|
import { countedAfter, dates, quantities } from "./quantities.ts";
|
|
6
|
+
import { sectionReferences, sectionVocabulary } from "./section-reference.ts";
|
|
6
7
|
|
|
7
8
|
// 契約書・規程・法令の番号の書き方。core は番号の書き方を知らず、ここで読んだものを入れ子にする。
|
|
8
9
|
|
|
@@ -261,27 +262,44 @@ const addressOfReference = (
|
|
|
261
262
|
return { target, fallback: paragraph === "1" && item === undefined ? article : undefined };
|
|
262
263
|
};
|
|
263
264
|
|
|
264
|
-
/**
|
|
265
|
-
const
|
|
265
|
+
/** 他の文書の名前を読む語と、章・節の番地でない言い方。語彙表から一度だけ作る。 */
|
|
266
|
+
const LEXICONS = loadLexicons();
|
|
267
|
+
const CITATION = citationVocabulary(LEXICONS);
|
|
268
|
+
const SECTIONS = sectionVocabulary(LEXICONS);
|
|
266
269
|
|
|
267
|
-
const
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
const groups = match.groups ?? {};
|
|
273
|
-
const address = addressOfReference(groups);
|
|
270
|
+
const sectionsOf = (text: string): Mention[] => sectionReferences(text, SECTIONS);
|
|
271
|
+
|
|
272
|
+
const articlesOf = (text: string): Mention[] =>
|
|
273
|
+
[...text.matchAll(REFERENCE)].flatMap((match) => {
|
|
274
|
+
const address = addressOfReference(match.groups ?? {});
|
|
274
275
|
if (address === undefined) return [];
|
|
275
276
|
const { target, fallback } = address;
|
|
276
|
-
const
|
|
277
|
-
const previous = chains.get(depth);
|
|
278
|
-
const inherited =
|
|
279
|
-
previous?.document !== undefined && isContinuation(withoutClosedParentheses(text.slice(previous.end, match.index))) ? previous.document : undefined;
|
|
280
|
-
const document = citedDocument(text, match.index, CITATION) ?? inherited;
|
|
281
|
-
chains.set(depth, { end: match.index + match[0].length, document });
|
|
282
|
-
const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }), ...(document === undefined ? {} : { document }) };
|
|
277
|
+
const attrs = { target, label: match[0], ...(fallback === undefined ? {} : { fallback }) };
|
|
283
278
|
return [{ start: match.index, end: match.index + match[0].length, attrs }];
|
|
284
279
|
});
|
|
280
|
+
|
|
281
|
+
/** 条の参照と章・節の参照を文書の順に並べ、他の文書の名前を付ける。「民法第3章及び第4章」の第4章も民法の章。 */
|
|
282
|
+
const references = (text: string): Mention[] => {
|
|
283
|
+
const depths = depthsOf(text);
|
|
284
|
+
// 括弧書きの中の参照(「(同法第五十九条において準用する場合を含む。)」)は、外の並びを切らない。並びは括弧の深さごとに持つ。
|
|
285
|
+
const chains = new Map<number, { readonly end: number; readonly document: string | undefined }>();
|
|
286
|
+
return [...articlesOf(text), ...sectionsOf(text)]
|
|
287
|
+
.sort((left, right) => left.start - right.start)
|
|
288
|
+
.map((mention) => {
|
|
289
|
+
const depth = depths[mention.start] ?? 0;
|
|
290
|
+
const previous = chains.get(depth);
|
|
291
|
+
const inherited =
|
|
292
|
+
previous?.document !== undefined && isContinuation(withoutClosedParentheses(text.slice(previous.end, mention.start))) ? previous.document : undefined;
|
|
293
|
+
const document = citedDocument(text, mention.start, CITATION) ?? inherited;
|
|
294
|
+
chains.set(depth, { end: mention.end, document });
|
|
295
|
+
return document === undefined ? mention : { ...mention, attrs: { ...mention.attrs, document } };
|
|
296
|
+
});
|
|
297
|
+
};
|
|
298
|
+
|
|
299
|
+
/** 「3.2節」の 3.2 は節の番地で、節の数ではない。 */
|
|
300
|
+
const quantitiesOutsideSections = (text: string): Mention[] => {
|
|
301
|
+
const sections = sectionsOf(text);
|
|
302
|
+
return quantities(text).filter((quantity) => !sections.some((section) => quantity.start < section.end && section.start < quantity.end));
|
|
285
303
|
};
|
|
286
304
|
|
|
287
305
|
/** 長いものから探し、重なったら先に見つけたものを残す。「しなければならない」を「なければならない」と二重に数えない。 */
|
|
@@ -330,4 +348,13 @@ const obligations = (text: string): Mention[] => {
|
|
|
330
348
|
/** 「二十二」「2」を数にする。相対の参照(前二項・前条第二項)を core が読むときに使う。 */
|
|
331
349
|
const number = (text: string): number | undefined => parseJapaneseNumber(toHalfWidth(text));
|
|
332
350
|
|
|
333
|
-
export const structure: StructurePatterns = {
|
|
351
|
+
export const structure: StructurePatterns = {
|
|
352
|
+
numbered,
|
|
353
|
+
definitions,
|
|
354
|
+
references,
|
|
355
|
+
obligations,
|
|
356
|
+
quantities: quantitiesOutsideSections,
|
|
357
|
+
dates,
|
|
358
|
+
countedAfter,
|
|
359
|
+
number,
|
|
360
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 語の文字を前から順に本文と照らし、それぞれの始まり(UTF-16 の位置)を返す。見つからない語は undefined。
|
|
3
|
+
* kuromoji の word_position は、記号のまとまり(「)、」「**、」)や絵文字の後ろで本文とずれる。語をつなげた文字は本文と同じなので、照らせば正しい位置が出る。
|
|
4
|
+
*/
|
|
5
|
+
export const surfaceStarts = (text: string, surfaces: readonly string[]): (number | undefined)[] => {
|
|
6
|
+
const scan = { cursor: 0 };
|
|
7
|
+
return surfaces.map((surface) => {
|
|
8
|
+
const at = surface === "" ? -1 : text.indexOf(surface, scan.cursor);
|
|
9
|
+
if (at === -1) return undefined;
|
|
10
|
+
scan.cursor = at + surface.length;
|
|
11
|
+
return at;
|
|
12
|
+
});
|
|
13
|
+
};
|