@chaffjs/lang-ja 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -1
- package/dist/index.js.map +1 -1
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +2 -0
- package/dist/lexicons.js.map +1 -1
- package/dist/reduplication.d.ts +12 -0
- package/dist/reduplication.d.ts.map +1 -0
- package/dist/reduplication.js +15 -0
- package/dist/reduplication.js.map +1 -0
- package/lexicons/common-acronym.yaml +72 -0
- package/lexicons/comparison-marker.yaml +8 -0
- package/lexicons/date-stamp-label.yaml +14 -0
- package/lexicons/definition-marker.yaml +6 -0
- package/lexicons/definition-verb.yaml +8 -0
- package/lexicons/distributive-noun.yaml +25 -0
- package/lexicons/distributive-particle.yaml +11 -0
- package/lexicons/doubled-word.yaml +10 -0
- package/lexicons/lead-in.yaml +10 -0
- package/lexicons/superlative-scope.yaml +7 -0
- package/package.json +1 -1
- package/src/index.ts +7 -1
- package/src/lexicons.ts +3 -0
- package/src/reduplication.ts +32 -0
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAgB,eAAe,EAAgC,MAAM,gBAAgB,CAAC;AA2DlG,eAAO,MAAM,OAAO,EAAE,eA0BrB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -2,6 +2,7 @@ import { loadLexicons } from "./lexicons.js";
|
|
|
2
2
|
import { sentenceSpans } from "./sentence-split.js";
|
|
3
3
|
import { structure } from "./structure.js";
|
|
4
4
|
import { isReady, predicateOnly, prepare, tokenize } from "./pos.js";
|
|
5
|
+
import { distributiveVocabulary, markReduplication } from "./reduplication.js";
|
|
5
6
|
// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
|
|
6
7
|
/**
|
|
7
8
|
* 日本語の文末は「。!?」に限られる。
|
|
@@ -38,6 +39,7 @@ const merge = (source, spans) => spans
|
|
|
38
39
|
* token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
|
|
39
40
|
* 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
|
|
40
41
|
*/
|
|
42
|
+
const DISTRIBUTIVE = distributiveVocabulary(loadLexicons());
|
|
41
43
|
const withTokens = (source, sentences) => {
|
|
42
44
|
const tokens = tokenize(source);
|
|
43
45
|
if (tokens === undefined)
|
|
@@ -45,7 +47,7 @@ const withTokens = (source, sentences) => {
|
|
|
45
47
|
return sentences.map((sentence) => ({
|
|
46
48
|
...sentence,
|
|
47
49
|
// 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
|
|
48
|
-
tokens: predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
|
|
50
|
+
tokens: markReduplication(predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)), DISTRIBUTIVE),
|
|
49
51
|
}));
|
|
50
52
|
};
|
|
51
53
|
export const adapter = {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AACrE,OAAO,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAG/E,2CAA2C;AAE3C;;;;;;;;;GASG;AACH,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC,MAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,MAAM,SAAS,GAAG,MAAM,CAAC;AAEzB,MAAM,MAAM,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEvF;;;GAGG;AACH,MAAM,KAAK,GAAG,CAAC,MAAc,EAAE,KAAsB,EAAc,EAAE,CACnE,KAAK;KACF,MAAM,CAAS,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC5B,MAAM,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxB,yCAAyC;IACzC,uCAAuC;IACvC,6BAA6B;IAC7B,MAAM,WAAW,GAAG,IAAI,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5F,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QACrF,OAAO,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,CAAC,GAAG,GAAG,EAAE,IAAI,CAAC,CAAC;AACxB,CAAC,EAAE,EAAE,CAAC;KACL,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AAEzE;;;GAGG;AACH,MAAM,YAAY,GAAG,sBAAsB,CAAC,YAAY,EAAE,CAAC,CAAC;AAE5D,MAAM,UAAU,GAAG,CAAC,MAAc,EAAE,SAA8B,EAAc,EAAE;IAChF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,CAAC,GAAG,SAAS,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAClC,GAAG,QAAQ;QACX,qCAAqC;QACrC,MAAM,EAAE,iBAAiB,CACvB,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvH,YAAY,CACb;KACF,CAAC,CAAC,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAoB;IACtC,IAAI,EAAE,UAAU;IAChB,EAAE,EAAE,IAAI;IACR,UAAU,EAAE,CAAC;IACb,YAAY,EAAE;QACZ,aAAa,EAAE,IAAI;QACnB,gDAAgD;QAChD,SAAS,EAAE,IAAI;QACf,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,IAAI;QACX,UAAU,EAAE,MAAM;KACnB;IACD,OAAO,EAAE,KAAK,EAAE,IAAkB,EAAiB,EAAE;QACnD,IAAI,IAAI,CAAC,GAAG;YAAE,MAAM,OAAO,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,EAAE,CAAC,MAAc,EAAU,EAAE;QACjC,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;QACrD,IAAI,KAAK,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC;IACvD,CAAC;IACD,QAAQ,EAAE,YAAY,EAAE;IACxB,SAAS;IACT,OAAO,EAAE,CAAC,IAAY,EAAgB,EAAE;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;QACnD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;IAC5E,CAAC;CACF,CAAC"}
|
package/dist/lexicons.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAoB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAQ7D,CAAC"}
|
package/dist/lexicons.js
CHANGED
|
@@ -4,6 +4,7 @@ import { dirname, join } from "node:path";
|
|
|
4
4
|
import { parse } from "yaml";
|
|
5
5
|
const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
|
|
6
6
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7
|
+
const POSITIONS = ["before", "after"];
|
|
7
8
|
const toEntry = (raw) => {
|
|
8
9
|
if (!isRecord(raw) || typeof raw["pattern"] !== "string")
|
|
9
10
|
return undefined;
|
|
@@ -13,6 +14,7 @@ const toEntry = (raw) => {
|
|
|
13
14
|
pattern: raw["pattern"],
|
|
14
15
|
weight: typeof weight === "number" ? weight : undefined,
|
|
15
16
|
instead_of: typeof instead === "string" ? instead : undefined,
|
|
17
|
+
position: POSITIONS.find((position) => position === raw["position"]),
|
|
16
18
|
};
|
|
17
19
|
};
|
|
18
20
|
/**
|
package/dist/lexicons.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;
|
|
1
|
+
{"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAU,CAAC;AAE/C,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,MAAM,CAA0B,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAClG,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC,EAAE,EAAE,CAAC,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Lexicon, Token } from "chaffjs/plugin";
|
|
2
|
+
/**
|
|
3
|
+
* 名詞を重ねて「それぞれの」を言う形(会社会社で、部署部署の)。重ねられるのは、まとまりや場合を指す名詞だけで、
|
|
4
|
+
* 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp を付ける。「資料資料の」「確認確認する」は書き損じのまま。
|
|
5
|
+
*/
|
|
6
|
+
export type Distributive = {
|
|
7
|
+
readonly nouns: ReadonlySet<string>;
|
|
8
|
+
readonly particles: ReadonlySet<string>;
|
|
9
|
+
};
|
|
10
|
+
export declare const distributiveVocabulary: (lexicons: Readonly<Record<string, Lexicon>>) => Distributive;
|
|
11
|
+
export declare const markReduplication: (tokens: readonly Token[], vocabulary: Distributive) => Token[];
|
|
12
|
+
//# sourceMappingURL=reduplication.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reduplication.d.ts","sourceRoot":"","sources":["../src/reduplication.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAErD;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAAC,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;CAAE,CAAC;AAI5G,eAAO,MAAM,sBAAsB,GAAI,UAAU,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAG,YAGnF,CAAC;AAeH,eAAO,MAAM,iBAAiB,GAAI,QAAQ,SAAS,KAAK,EAAE,EAAE,YAAY,YAAY,KAAG,KAAK,EAGzF,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const patternsOf = (lexicon) => new Set((lexicon ?? []).map((entry) => entry.pattern));
|
|
2
|
+
export const distributiveVocabulary = (lexicons) => ({
|
|
3
|
+
nouns: patternsOf(lexicons["distributive-noun"]),
|
|
4
|
+
particles: patternsOf(lexicons["distributive-particle"]),
|
|
5
|
+
});
|
|
6
|
+
const isSameNoun = (first, second, nouns) => first !== undefined &&
|
|
7
|
+
first.pos === "NOUN" &&
|
|
8
|
+
second.pos === "NOUN" &&
|
|
9
|
+
first.surface === second.surface &&
|
|
10
|
+
first.span.end === second.span.start &&
|
|
11
|
+
nouns.has(second.surface);
|
|
12
|
+
const isParticleAfter = (next, second, particles) => next !== undefined && next.pos === "ADP" && next.span.start === second.span.end && particles.has(next.surface);
|
|
13
|
+
const echoed = (token) => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
|
|
14
|
+
export const markReduplication = (tokens, vocabulary) => tokens.map((token, index) => isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token);
|
|
15
|
+
//# sourceMappingURL=reduplication.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reduplication.js","sourceRoot":"","sources":["../src/reduplication.ts"],"names":[],"mappings":"AAQA,MAAM,UAAU,GAAG,CAAC,OAA4B,EAAuB,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;AAEjI,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,QAA2C,EAAgB,EAAE,CAAC,CAAC;IACpG,KAAK,EAAE,UAAU,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAChD,SAAS,EAAE,UAAU,CAAC,QAAQ,CAAC,uBAAuB,CAAC,CAAC;CACzD,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,KAAwB,EAAE,MAAa,EAAE,KAA0B,EAAW,EAAE,CAClG,KAAK,KAAK,SAAS;IACnB,KAAK,CAAC,GAAG,KAAK,MAAM;IACpB,MAAM,CAAC,GAAG,KAAK,MAAM;IACrB,KAAK,CAAC,OAAO,KAAK,MAAM,CAAC,OAAO;IAChC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK;IACpC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAE5B,MAAM,eAAe,GAAG,CAAC,IAAuB,EAAE,MAAa,EAAE,SAA8B,EAAW,EAAE,CAC1G,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEjH,MAAM,MAAM,GAAG,CAAC,KAAY,EAAS,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;AAErG,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAwB,EAAE,UAAwB,EAAW,EAAE,CAC/F,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAC1B,UAAU,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,IAAI,eAAe,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAClJ,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# 読み手が説明なしで通じると見なしてよい略語。展開すると逆に読みにくい。
|
|
2
|
+
id: common-acronym
|
|
3
|
+
language: ja
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: OK
|
|
6
|
+
- pattern: NG
|
|
7
|
+
- pattern: URL
|
|
8
|
+
- pattern: API
|
|
9
|
+
- pattern: CSS
|
|
10
|
+
- pattern: HTML
|
|
11
|
+
- pattern: JSON
|
|
12
|
+
- pattern: YAML
|
|
13
|
+
- pattern: HTTP
|
|
14
|
+
- pattern: HTTPS
|
|
15
|
+
- pattern: PDF
|
|
16
|
+
- pattern: CPU
|
|
17
|
+
- pattern: GPU
|
|
18
|
+
- pattern: RAM
|
|
19
|
+
- pattern: USB
|
|
20
|
+
- pattern: AI
|
|
21
|
+
- pattern: ID
|
|
22
|
+
- pattern: FAQ
|
|
23
|
+
- pattern: PR
|
|
24
|
+
- pattern: OS
|
|
25
|
+
- pattern: CLI
|
|
26
|
+
- pattern: UI
|
|
27
|
+
- pattern: UX
|
|
28
|
+
- pattern: SQL
|
|
29
|
+
- pattern: CSV
|
|
30
|
+
- pattern: XML
|
|
31
|
+
- pattern: TODO
|
|
32
|
+
- pattern: NOTE
|
|
33
|
+
- pattern: TIP
|
|
34
|
+
# 通信と符号化。技術文書でなくても説明なしで通じる。
|
|
35
|
+
- pattern: DNS
|
|
36
|
+
- pattern: IP
|
|
37
|
+
- pattern: TCP
|
|
38
|
+
- pattern: UDP
|
|
39
|
+
- pattern: SSH
|
|
40
|
+
- pattern: SSL
|
|
41
|
+
- pattern: TLS
|
|
42
|
+
- pattern: VPN
|
|
43
|
+
- pattern: LAN
|
|
44
|
+
- pattern: UTF
|
|
45
|
+
- pattern: ASCII
|
|
46
|
+
# 機器・形式・単位。
|
|
47
|
+
- pattern: PC
|
|
48
|
+
- pattern: IT
|
|
49
|
+
- pattern: SDK
|
|
50
|
+
- pattern: IDE
|
|
51
|
+
- pattern: PNG
|
|
52
|
+
- pattern: JPEG
|
|
53
|
+
- pattern: GIF
|
|
54
|
+
- pattern: SVG
|
|
55
|
+
- pattern: QR
|
|
56
|
+
- pattern: GPS
|
|
57
|
+
- pattern: SNS
|
|
58
|
+
- pattern: TV
|
|
59
|
+
- pattern: KB
|
|
60
|
+
- pattern: MB
|
|
61
|
+
- pattern: GB
|
|
62
|
+
- pattern: TB
|
|
63
|
+
# 役職と国・地域。
|
|
64
|
+
- pattern: CEO
|
|
65
|
+
- pattern: CTO
|
|
66
|
+
- pattern: CFO
|
|
67
|
+
- pattern: US
|
|
68
|
+
- pattern: USA
|
|
69
|
+
- pattern: UK
|
|
70
|
+
- pattern: EU
|
|
71
|
+
- pattern: UN
|
|
72
|
+
- pattern: DNA
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# 更新日・公開日の刻印のラベル。これと日付だけの段落は、ページの付帯情報として本題までの段落に数えない。
|
|
2
|
+
id: date-stamp-label
|
|
3
|
+
language: ja
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: 最終更新日
|
|
6
|
+
- pattern: 最終更新
|
|
7
|
+
- pattern: 更新日
|
|
8
|
+
- pattern: 公開日
|
|
9
|
+
- pattern: 掲載日
|
|
10
|
+
- pattern: 作成日
|
|
11
|
+
- pattern: 発行日
|
|
12
|
+
- pattern: 更新
|
|
13
|
+
- pattern: 公開
|
|
14
|
+
- pattern: 掲載
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# 重ねて「それぞれの」を言える名詞。まとまり(会社・部署・家庭)と、場合や時(場面・時代)を指すもの。
|
|
2
|
+
# 「会社会社で考え方が違う」「場面場面で使い分ける」。ここに無い名詞の重なり(資料資料の)は、doubled-word が書き損じとして数える。
|
|
3
|
+
id: distributive-noun
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: 会社
|
|
7
|
+
- pattern: 企業
|
|
8
|
+
- pattern: 組織
|
|
9
|
+
- pattern: 部署
|
|
10
|
+
- pattern: 部門
|
|
11
|
+
- pattern: チーム
|
|
12
|
+
- pattern: 店舗
|
|
13
|
+
- pattern: 現場
|
|
14
|
+
- pattern: 職場
|
|
15
|
+
- pattern: 学校
|
|
16
|
+
- pattern: 家庭
|
|
17
|
+
- pattern: 地域
|
|
18
|
+
- pattern: 国
|
|
19
|
+
- pattern: 個人
|
|
20
|
+
- pattern: 場面
|
|
21
|
+
- pattern: 場所
|
|
22
|
+
- pattern: 時代
|
|
23
|
+
- pattern: 時期
|
|
24
|
+
- pattern: 季節
|
|
25
|
+
- pattern: 案件
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# 重ねた名詞(distributive-noun)の後ろに続いて「それぞれの」を言う助詞。「会社会社で」「部署部署の」「場面場面に」。
|
|
2
|
+
# ここに無い助詞が続く重なり(会社会社を)は、doubled-word が書き損じとして数える。
|
|
3
|
+
id: distributive-particle
|
|
4
|
+
language: ja
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: で
|
|
7
|
+
- pattern: に
|
|
8
|
+
- pattern: の
|
|
9
|
+
- pattern: によって
|
|
10
|
+
- pattern: により
|
|
11
|
+
- pattern: による
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# doubled-word が数えない重なり。形態素解析が一語にしない重ね言葉だけを置く。
|
|
2
|
+
# 「まだまだ」「ますます」「いろいろ」「時々」は一語になり、「そうそう」「もっともっと」は副詞なので、ここには要らない。
|
|
3
|
+
# 名詞を重ねて「それぞれの」を言う形(会社会社で)は、後ろの助詞でアダプタが見分ける(distributive-particle)。
|
|
4
|
+
id: doubled-word
|
|
5
|
+
language: ja
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: 一つ一つ
|
|
8
|
+
- pattern: 見る見る
|
|
9
|
+
- pattern: 毎日毎日
|
|
10
|
+
- pattern: 毎回毎回
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { loadLexicons } from "./lexicons.ts";
|
|
|
2
2
|
import { sentenceSpans } from "./sentence-split.ts";
|
|
3
3
|
import { structure } from "./structure.ts";
|
|
4
4
|
import { isReady, predicateOnly, prepare, tokenize } from "./pos.ts";
|
|
5
|
+
import { distributiveVocabulary, markReduplication } from "./reduplication.ts";
|
|
5
6
|
import type { AdapterNeeds, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
|
|
6
7
|
|
|
7
8
|
// chaff からは型だけを取る。実行時の値依存を作らない。アダプタは単体で動く。
|
|
@@ -46,13 +47,18 @@ const merge = (source: string, spans: readonly Span[]): Sentence[] =>
|
|
|
46
47
|
* token の span は文ではなく、segment に渡した文字列を基準にする。文の span と同じ座標系。
|
|
47
48
|
* 文ごとに解析して足し戻すのではなく、一度解析して文へ配る。同じ文字列を二度読まない。
|
|
48
49
|
*/
|
|
50
|
+
const DISTRIBUTIVE = distributiveVocabulary(loadLexicons());
|
|
51
|
+
|
|
49
52
|
const withTokens = (source: string, sentences: readonly Sentence[]): Sentence[] => {
|
|
50
53
|
const tokens = tokenize(source);
|
|
51
54
|
if (tokens === undefined) return [...sentences];
|
|
52
55
|
return sentences.map((sentence) => ({
|
|
53
56
|
...sentence,
|
|
54
57
|
// 述語かどうかは文の中でしか決まらないので、文へ配ってから印を落とす。
|
|
55
|
-
tokens:
|
|
58
|
+
tokens: markReduplication(
|
|
59
|
+
predicateOnly(tokens.filter((token) => token.span.start >= sentence.span.start && token.span.end <= sentence.span.end)),
|
|
60
|
+
DISTRIBUTIVE,
|
|
61
|
+
),
|
|
56
62
|
}));
|
|
57
63
|
};
|
|
58
64
|
|
package/src/lexicons.ts
CHANGED
|
@@ -8,6 +8,8 @@ const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
|
|
|
8
8
|
|
|
9
9
|
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10
10
|
|
|
11
|
+
const POSITIONS = ["before", "after"] as const;
|
|
12
|
+
|
|
11
13
|
const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
12
14
|
if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
|
|
13
15
|
const weight = raw["weight"];
|
|
@@ -16,6 +18,7 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
|
16
18
|
pattern: raw["pattern"],
|
|
17
19
|
weight: typeof weight === "number" ? weight : undefined,
|
|
18
20
|
instead_of: typeof instead === "string" ? instead : undefined,
|
|
21
|
+
position: POSITIONS.find((position) => position === raw["position"]),
|
|
19
22
|
};
|
|
20
23
|
};
|
|
21
24
|
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Lexicon, Token } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* 名詞を重ねて「それぞれの」を言う形(会社会社で、部署部署の)。重ねられるのは、まとまりや場合を指す名詞だけで、
|
|
5
|
+
* 後ろに助詞が続く。二つ目の名詞に UD の Echo=Rdp を付ける。「資料資料の」「確認確認する」は書き損じのまま。
|
|
6
|
+
*/
|
|
7
|
+
export type Distributive = { readonly nouns: ReadonlySet<string>; readonly particles: ReadonlySet<string> };
|
|
8
|
+
|
|
9
|
+
const patternsOf = (lexicon: Lexicon | undefined): ReadonlySet<string> => new Set((lexicon ?? []).map((entry) => entry.pattern));
|
|
10
|
+
|
|
11
|
+
export const distributiveVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): Distributive => ({
|
|
12
|
+
nouns: patternsOf(lexicons["distributive-noun"]),
|
|
13
|
+
particles: patternsOf(lexicons["distributive-particle"]),
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
const isSameNoun = (first: Token | undefined, second: Token, nouns: ReadonlySet<string>): boolean =>
|
|
17
|
+
first !== undefined &&
|
|
18
|
+
first.pos === "NOUN" &&
|
|
19
|
+
second.pos === "NOUN" &&
|
|
20
|
+
first.surface === second.surface &&
|
|
21
|
+
first.span.end === second.span.start &&
|
|
22
|
+
nouns.has(second.surface);
|
|
23
|
+
|
|
24
|
+
const isParticleAfter = (next: Token | undefined, second: Token, particles: ReadonlySet<string>): boolean =>
|
|
25
|
+
next !== undefined && next.pos === "ADP" && next.span.start === second.span.end && particles.has(next.surface);
|
|
26
|
+
|
|
27
|
+
const echoed = (token: Token): Token => ({ ...token, features: { ...token.features, Echo: "Rdp" } });
|
|
28
|
+
|
|
29
|
+
export const markReduplication = (tokens: readonly Token[], vocabulary: Distributive): Token[] =>
|
|
30
|
+
tokens.map((token, index) =>
|
|
31
|
+
isSameNoun(tokens[index - 1], token, vocabulary.nouns) && isParticleAfter(tokens[index + 1], token, vocabulary.particles) ? echoed(token) : token,
|
|
32
|
+
);
|