@chaffjs/lang-en 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +2 -0
- package/dist/lexicons.js.map +1 -1
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +22 -3
- package/dist/pos.js.map +1 -1
- package/dist/proper-noun.d.ts +6 -0
- package/dist/proper-noun.d.ts.map +1 -0
- package/dist/proper-noun.js +23 -0
- package/dist/proper-noun.js.map +1 -0
- package/dist/stative-participle.d.ts +19 -0
- package/dist/stative-participle.d.ts.map +1 -0
- package/dist/stative-participle.js +23 -0
- package/dist/stative-participle.js.map +1 -0
- package/lexicons/article.yaml +8 -0
- package/lexicons/common-acronym.yaml +72 -0
- package/lexicons/comparison-marker.yaml +9 -0
- package/lexicons/date-stamp-label.yaml +15 -0
- package/lexicons/definition-marker.yaml +14 -0
- package/lexicons/definition-verb.yaml +5 -0
- package/lexicons/degree-adverb.yaml +7 -0
- package/lexicons/doubled-word.yaml +30 -0
- package/lexicons/lead-in.yaml +8 -0
- package/lexicons/object-or-possessive.yaml +7 -0
- package/lexicons/stative-participle.yaml +25 -0
- package/lexicons/superlative-scope.yaml +10 -0
- package/package.json +1 -1
- package/src/lexicons.ts +3 -0
- package/src/pos.ts +30 -4
- package/src/proper-noun.ts +21 -0
- package/src/stative-participle.ts +43 -0
package/dist/lexicons.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"lexicons.d.ts","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,OAAO,EAAgB,MAAM,gBAAgB,CAAC;AAoB5D;;;GAGG;AACH,eAAO,MAAM,YAAY,GAAI,MAAK,MAAY,KAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAQ7D,CAAC"}
|
package/dist/lexicons.js
CHANGED
|
@@ -4,6 +4,7 @@ import { dirname, join } from "node:path";
|
|
|
4
4
|
import { parse } from "yaml";
|
|
5
5
|
const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
|
|
6
6
|
const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
7
|
+
const POSITIONS = ["before", "after"];
|
|
7
8
|
const toEntry = (raw) => {
|
|
8
9
|
if (!isRecord(raw) || typeof raw["pattern"] !== "string")
|
|
9
10
|
return undefined;
|
|
@@ -13,6 +14,7 @@ const toEntry = (raw) => {
|
|
|
13
14
|
pattern: raw["pattern"],
|
|
14
15
|
weight: typeof weight === "number" ? weight : undefined,
|
|
15
16
|
instead_of: typeof instead === "string" ? instead : undefined,
|
|
17
|
+
position: POSITIONS.find((position) => position === raw["position"]),
|
|
16
18
|
};
|
|
17
19
|
};
|
|
18
20
|
/**
|
package/dist/lexicons.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;
|
|
1
|
+
{"version":3,"file":"lexicons.js","sourceRoot":"","sources":["../src/lexicons.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAG7B,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;AAE5E,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAE5I,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAU,CAAC;AAE/C,MAAM,OAAO,GAAG,CAAC,GAAY,EAA4B,EAAE;IACzD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,SAAS,CAAC,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC3E,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IAClC,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,SAAS,CAAC;QACvB,MAAM,EAAE,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,UAAU,EAAE,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC7D,QAAQ,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,KAAK,GAAG,CAAC,UAAU,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,GAAG,EAA2B,EAAE,CACzE,WAAW,CAAC,GAAG,CAAC;KACb,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;KACxC,MAAM,CAA0B,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;IAC7C,MAAM,GAAG,GAAY,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IAClE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAClG,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACnF,OAAO,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AAC1C,CAAC,EAAE,EAAE,CAAC,CAAC"}
|
package/dist/pos.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pos.d.ts","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"pos.d.ts","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AA+E5C,eAAO,MAAM,IAAI,GAAI,KAAK,MAAM,KAAG,MAAiE,CAAC;AAyFrG,eAAO,MAAM,OAAO,QAAO,IAE1B,CAAC;AAEF,eAAO,MAAM,OAAO,QAAO,OAAoC,CAAC;AAwBhE,eAAO,MAAM,QAAQ,GAAI,MAAM,MAAM,KAAG,KAAK,EAAE,GAAG,SAIjD,CAAC"}
|
package/dist/pos.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
|
+
import { loadLexicons } from "./lexicons.js";
|
|
3
|
+
import { properNounChecked } from "./proper-noun.js";
|
|
4
|
+
import { isStativeParticiple, stativeVocabulary } from "./stative-participle.js";
|
|
2
5
|
const require = createRequire(import.meta.url);
|
|
3
6
|
const isRecord = (value) => typeof value === "object" && value !== null;
|
|
4
7
|
const isCallable = (value) => typeof value === "function";
|
|
@@ -104,16 +107,32 @@ const inRelativeClause = (tagged, be) => {
|
|
|
104
107
|
const antecedent = tagged.slice(0, lead).findLast((entry) => entry.pos !== ",");
|
|
105
108
|
return antecedent !== undefined && NOMINAL_TAG.has(antecedent.pos);
|
|
106
109
|
};
|
|
110
|
+
const LEXICONS = loadLexicons();
|
|
111
|
+
const STATIVE = stativeVocabulary(LEXICONS);
|
|
107
112
|
const isPassive = (tagged, at) => {
|
|
108
113
|
if (tagged[at]?.pos !== "VBN")
|
|
109
114
|
return false;
|
|
110
115
|
const be = beBefore(tagged, at);
|
|
111
|
-
return be !== -1 && !inRelativeClause(tagged, be);
|
|
116
|
+
return be !== -1 && !inRelativeClause(tagged, be) && !isStativeParticiple(tagged, at, STATIVE);
|
|
117
|
+
};
|
|
118
|
+
const wordsOf = (name) => new Set((LEXICONS[name] ?? []).map((entry) => entry.pattern.toLowerCase()));
|
|
119
|
+
const ARTICLES = wordsOf("article");
|
|
120
|
+
/** 目的語にも所有にもなる語(give her the book / her book)。解析器はいつも PRP$ と付けるので、所有とは言えない。 */
|
|
121
|
+
const OBJECT_OR_POSSESSIVE = wordsOf("object-or-possessive");
|
|
122
|
+
const POSSESSIVE_TAG = new Set(["PRP$", "WP$"]);
|
|
123
|
+
/** 冠詞(PronType=Art)と所有の語(Poss=Yes)は、名詞の前に一つしか立たない。二つ並ぶ our the platform は書き損じ。 */
|
|
124
|
+
const determinerFeatures = (entry) => {
|
|
125
|
+
if (POSSESSIVE_TAG.has(entry.pos))
|
|
126
|
+
return OBJECT_OR_POSSESSIVE.has(entry.value.toLowerCase()) ? {} : { features: { Poss: "Yes" } };
|
|
127
|
+
return entry.pos === "DT" && ARTICLES.has(entry.value.toLowerCase()) ? { features: { PronType: "Art" } } : {};
|
|
112
128
|
};
|
|
113
129
|
/** 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び(fix the parser, ship it)と見分ける。 */
|
|
114
130
|
const featuresOf = (tagged, at) => {
|
|
115
|
-
|
|
131
|
+
const entry = tagged[at];
|
|
132
|
+
if (entry === undefined)
|
|
116
133
|
return {};
|
|
134
|
+
if (entry.pos !== "VBN")
|
|
135
|
+
return determinerFeatures(entry);
|
|
117
136
|
return { features: isPassive(tagged, at) ? { VerbForm: "Part", Voice: "Pass" } : { VerbForm: "Part" } };
|
|
118
137
|
};
|
|
119
138
|
const state = { ready: undefined };
|
|
@@ -133,7 +152,7 @@ const locate = (text, tagged) => tagged.reduce((acc, entry, at) => {
|
|
|
133
152
|
const token = {
|
|
134
153
|
span: { start, end },
|
|
135
154
|
surface: entry.value,
|
|
136
|
-
pos: upos(entry.pos),
|
|
155
|
+
pos: properNounChecked(entry.value, upos(entry.pos)),
|
|
137
156
|
...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
|
|
138
157
|
...featuresOf(tagged, at),
|
|
139
158
|
};
|
package/dist/pos.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pos.js","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"pos.js","sourceRoot":"","sources":["../src/pos.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjF,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAO/C,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAoC,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC;AAEnH,MAAM,UAAU,GAAG,CAAC,KAAc,EAAqD,EAAE,CAAC,OAAO,KAAK,KAAK,UAAU,CAAC;AAEtH,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAmB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,QAAQ,CAAC;AAEhJ,MAAM,OAAO,GAAG,CAAC,KAAc,EAAsB,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE5F,kEAAkE;AAClE,MAAM,UAAU,GAAG,CAAC,KAA8B,EAAE,IAAY,EAAE,IAAwB,EAAW,EAAE;IACrG,MAAM,MAAM,GAAY,KAAK,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,IAAI,WAAW,CAAC,CAAC;IAC/E,OAAO,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;AAC5C,CAAC,CAAC;AAEF,MAAM,KAAK,GAAG,GAAW,EAAE;IACzB,MAAM,OAAO,GAAY,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACpD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAChF,MAAM,MAAM,GAAY,OAAO,EAAE,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IAC7E,OAAO,MAAM,CAAC;AAChB,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,GAAqC;IAC/C,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,KAAK;IACT,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,OAAO;IACZ,IAAI,EAAE,OAAO;IACb,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,IAAI,EAAE,MAAM;IACZ,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,KAAK;IACT,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,MAAM;IACV,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,KAAK;IACV,EAAE,EAAE,MAAM;IACV,GAAG,EAAE,MAAM;IACX,GAAG,EAAE,KAAK;CACX,CAAC;AAEF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7E,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;AAErG;;;GAGG;AACH;;;;;;GAMG;AACH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;AAE3D,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAE9E,MAAM,IAAI,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;AAE/H,8BAA8B;AAC9B,MAAM,QAAQ,GAAG,CAAC,MAAyB,EAAE,EAAU,EAAU,EAAE;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IACrF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,KAAK,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;AAE5D,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEvD,MAAM,WAAW,GAAG,CAAC,KAAa,EAAW,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;AAErI,wDAAwD;AACxD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;AAE7E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;AAE5C,MAAM,gBAAgB,GAAG,CAAC,MAAyB,EAAE,EAAU,EAAW,EAAE;IAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5E,6DAA6D;IAC7D,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IAChF,OAAO,UAAU,KAAK,SAAS,IAAI,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AACrE,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;AAEhC,MAAM,OAAO,GAAG,iBAAiB,CAAC,QAAQ,CAAC,CAAC;AAE5C,MAAM,SAAS,GAAG,CAAC,MAAyB,EAAE,EAAU,EAAW,EAAE;IACnE,IAAI,MAAM,CAAC,EAAE,CAAC,EAAE,GAAG,KAAK,KAAK;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChC,OAAO,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,CAAC,CAAC;AACjG,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,IAAY,EAAuB,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAEnI,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;AAEpC,+EAA+E;AAC/E,MAAM,oBAAoB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;AAE7D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAIhD,kFAAkF;AAClF,MAAM,kBAAkB,GAAG,CAAC,KAAa,EAAY,EAAE;IACrD,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;QAAE,OAAO,oBAAoB,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;IACnI,OAAO,KAAK,CAAC,GAAG,KAAK,IAAI,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAChH,CAAC,CAAC;AAEF,iGAAiG;AACjG,MAAM,UAAU,GAAG,CAAC,MAAyB,EAAE,EAAU,EAAY,EAAE;IACrE,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACnC,IAAI,KAAK,CAAC,GAAG,KAAK,KAAK;QAAE,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,EAAE,QAAQ,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;AAC1G,CAAC,CAAC;AAEF,MAAM,KAAK,GAAkC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AAElE,MAAM,CAAC,MAAM,OAAO,GAAG,GAAS,EAAE;IAChC,KAAK,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;AAC1B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,OAAO,GAAG,GAAY,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;AAEhE;;;GAGG;AACH,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,MAAyB,EAAW,EAAE,CAClE,MAAM,CAAC,MAAM,CACX,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE;IACjB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IACpD,IAAI,KAAK,KAAK,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IAC7B,MAAM,GAAG,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;IACvC,MAAM,KAAK,GAAG;QACZ,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE;QACpB,OAAO,EAAE,KAAK,CAAC,KAAK;QACpB,GAAG,EAAE,iBAAiB,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACpD,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;KAC1B,CAAC;IACF,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AACzD,CAAC,EACD,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,CAC1B,CAAC,MAAM,CAAC;AAEX,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAuB,EAAE;IAC5D,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC;IAC3B,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC3C,OAAO,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,MAAM,EAAE,aAAa,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC3F,CAAC,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* 解析器が固有名詞とした語を、表記で確かめる。wink は知らない語を固有名詞にするが、英語の固有名詞は大文字で書く。
|
|
3
|
+
* 小文字だけで書いた語(linters、json、e.g. の e)は普通名詞、文字の無い語(—、$)は記号。大文字の無い文字(東京)の語は確かめようがないので触れない。
|
|
4
|
+
*/
|
|
5
|
+
export declare const properNounChecked: (surface: string, pos: string) => string;
|
|
6
|
+
//# sourceMappingURL=proper-noun.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proper-noun.d.ts","sourceRoot":"","sources":["../src/proper-noun.ts"],"names":[],"mappings":"AAYA;;;GAGG;AACH,eAAO,MAAM,iBAAiB,GAAI,SAAS,MAAM,EAAE,KAAK,MAAM,KAAG,MAIhE,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const LETTER = /\p{L}/u;
|
|
2
|
+
const CAPITAL = /\p{Lu}/u;
|
|
3
|
+
const SMALL = /\p{Ll}/u;
|
|
4
|
+
const DIGIT = /\p{N}/u;
|
|
5
|
+
const SYMBOL = /^[\p{Sc}\p{Sm}\p{So}]+$/u;
|
|
6
|
+
/** 文字の無い語の品詞。数字があれば数、通貨や数学の記号は記号、ほかは句読点(— や …)。 */
|
|
7
|
+
const nonWord = (surface) => {
|
|
8
|
+
if (DIGIT.test(surface))
|
|
9
|
+
return "NUM";
|
|
10
|
+
return SYMBOL.test(surface) ? "SYM" : "PUNCT";
|
|
11
|
+
};
|
|
12
|
+
/**
|
|
13
|
+
* 解析器が固有名詞とした語を、表記で確かめる。wink は知らない語を固有名詞にするが、英語の固有名詞は大文字で書く。
|
|
14
|
+
* 小文字だけで書いた語(linters、json、e.g. の e)は普通名詞、文字の無い語(—、$)は記号。大文字の無い文字(東京)の語は確かめようがないので触れない。
|
|
15
|
+
*/
|
|
16
|
+
export const properNounChecked = (surface, pos) => {
|
|
17
|
+
if (pos !== "PROPN")
|
|
18
|
+
return pos;
|
|
19
|
+
if (!LETTER.test(surface))
|
|
20
|
+
return nonWord(surface);
|
|
21
|
+
return SMALL.test(surface) && !CAPITAL.test(surface) ? "NOUN" : "PROPN";
|
|
22
|
+
};
|
|
23
|
+
//# sourceMappingURL=proper-noun.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proper-noun.js","sourceRoot":"","sources":["../src/proper-noun.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,GAAG,QAAQ,CAAC;AACxB,MAAM,OAAO,GAAG,SAAS,CAAC;AAC1B,MAAM,KAAK,GAAG,SAAS,CAAC;AACxB,MAAM,KAAK,GAAG,QAAQ,CAAC;AACvB,MAAM,MAAM,GAAG,0BAA0B,CAAC;AAE1C,mDAAmD;AACnD,MAAM,OAAO,GAAG,CAAC,OAAe,EAAU,EAAE;IAC1C,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,OAAO,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;AAChD,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,OAAe,EAAE,GAAW,EAAU,EAAE;IACxE,IAAI,GAAG,KAAK,OAAO;QAAE,OAAO,GAAG,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC,CAAC;IACnD,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;AAC1E,CAAC,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Lexicon } from "chaffjs/plugin";
|
|
2
|
+
/** 解析器が返す 1 語。pos は Penn Treebank。 */
|
|
3
|
+
type TaggedWord = {
|
|
4
|
+
readonly value: string;
|
|
5
|
+
readonly pos: string;
|
|
6
|
+
};
|
|
7
|
+
/** 状態を表す過去分詞の句(語の並び)と、程度の副詞。どちらも語彙表から引く。 */
|
|
8
|
+
export type StativeVocabulary = {
|
|
9
|
+
readonly phrases: readonly (readonly string[])[];
|
|
10
|
+
readonly degreeAdverbs: ReadonlySet<string>;
|
|
11
|
+
};
|
|
12
|
+
export declare const stativeVocabulary: (lexicons: Readonly<Record<string, Lexicon>>) => StativeVocabulary;
|
|
13
|
+
/**
|
|
14
|
+
* be の後の過去分詞が、誰かのした動作ではなく状態を表しているか。
|
|
15
|
+
* We are delighted to offer / is based on / are entitled to には名指すべき動作主がいない。
|
|
16
|
+
*/
|
|
17
|
+
export declare const isStativeParticiple: (tagged: readonly TaggedWord[], at: number, vocabulary: StativeVocabulary) => boolean;
|
|
18
|
+
export {};
|
|
19
|
+
//# sourceMappingURL=stative-participle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stative-participle.d.ts","sourceRoot":"","sources":["../src/stative-participle.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAE9C,sCAAsC;AACtC,KAAK,UAAU,GAAG;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnE,4CAA4C;AAC5C,MAAM,MAAM,iBAAiB,GAAG;IAC9B,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,SAAS,MAAM,EAAE,CAAC,EAAE,CAAC;IACjD,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC7C,CAAC;AAYF,eAAO,MAAM,iBAAiB,GAAI,UAAU,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,KAAG,iBAG9E,CAAC;AAaH;;;GAGG;AACH,eAAO,MAAM,mBAAmB,GAAI,QAAQ,SAAS,UAAU,EAAE,EAAE,IAAI,MAAM,EAAE,YAAY,iBAAiB,KAAG,OACK,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
const wordsOf = (lexicon) => (lexicon ?? [])
|
|
2
|
+
.map((entry) => entry.pattern
|
|
3
|
+
.toLowerCase()
|
|
4
|
+
.split(" ")
|
|
5
|
+
.filter((word) => word !== ""))
|
|
6
|
+
.filter((words) => words.length > 0);
|
|
7
|
+
export const stativeVocabulary = (lexicons) => ({
|
|
8
|
+
phrases: wordsOf(lexicons["stative-participle"]),
|
|
9
|
+
degreeAdverbs: new Set(wordsOf(lexicons["degree-adverb"]).flat()),
|
|
10
|
+
});
|
|
11
|
+
const wordAt = (tagged, at) => tagged[at]?.value.toLowerCase();
|
|
12
|
+
const startsPhrase = (tagged, at, phrase) => phrase.every((word, offset) => wordAt(tagged, at + offset) === word);
|
|
13
|
+
/** very は段階のある形容詞にしか付かない。very surprised は言えても very approved は言えない。 */
|
|
14
|
+
const afterDegreeAdverb = (tagged, at, vocabulary) => {
|
|
15
|
+
const before = wordAt(tagged, at - 1);
|
|
16
|
+
return before !== undefined && vocabulary.degreeAdverbs.has(before);
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* be の後の過去分詞が、誰かのした動作ではなく状態を表しているか。
|
|
20
|
+
* We are delighted to offer / is based on / are entitled to には名指すべき動作主がいない。
|
|
21
|
+
*/
|
|
22
|
+
export const isStativeParticiple = (tagged, at, vocabulary) => vocabulary.phrases.some((phrase) => startsPhrase(tagged, at, phrase)) || afterDegreeAdverb(tagged, at, vocabulary);
|
|
23
|
+
//# sourceMappingURL=stative-participle.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stative-participle.js","sourceRoot":"","sources":["../src/stative-participle.ts"],"names":[],"mappings":"AAWA,MAAM,OAAO,GAAG,CAAC,OAA4B,EAAc,EAAE,CAC3D,CAAC,OAAO,IAAI,EAAE,CAAC;KACZ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACb,KAAK,CAAC,OAAO;KACV,WAAW,EAAE;KACb,KAAK,CAAC,GAAG,CAAC;KACV,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CACjC;KACA,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAEzC,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,QAA2C,EAAqB,EAAE,CAAC,CAAC;IACpG,OAAO,EAAE,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;IAChD,aAAa,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;CAClE,CAAC,CAAC;AAEH,MAAM,MAAM,GAAG,CAAC,MAA6B,EAAE,EAAU,EAAsB,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC;AAElH,MAAM,YAAY,GAAG,CAAC,MAA6B,EAAE,EAAU,EAAE,MAAyB,EAAW,EAAE,CACrG,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC;AAEvE,sEAAsE;AACtE,MAAM,iBAAiB,GAAG,CAAC,MAA6B,EAAE,EAAU,EAAE,UAA6B,EAAW,EAAE;IAC9G,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;IACtC,OAAO,MAAM,KAAK,SAAS,IAAI,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,MAA6B,EAAE,EAAU,EAAE,UAA6B,EAAW,EAAE,CACvH,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,EAAE,UAAU,CAAC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# The articles. The tagger gives every determiner the same tag (DT), so the adapter marks these PronType=Art, as
|
|
2
|
+
# Universal Dependencies does: a noun takes one article or possessive, and two in a row ("our the platform") is a slip.
|
|
3
|
+
id: article
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: a
|
|
7
|
+
- pattern: an
|
|
8
|
+
- pattern: the
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# Acronyms a reader understands without the spelled-out form. Spelling them out would make the text harder to read.
|
|
2
|
+
id: common-acronym
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: OK
|
|
6
|
+
- pattern: NG
|
|
7
|
+
- pattern: URL
|
|
8
|
+
- pattern: API
|
|
9
|
+
- pattern: CSS
|
|
10
|
+
- pattern: HTML
|
|
11
|
+
- pattern: JSON
|
|
12
|
+
- pattern: YAML
|
|
13
|
+
- pattern: HTTP
|
|
14
|
+
- pattern: HTTPS
|
|
15
|
+
- pattern: PDF
|
|
16
|
+
- pattern: CPU
|
|
17
|
+
- pattern: GPU
|
|
18
|
+
- pattern: RAM
|
|
19
|
+
- pattern: USB
|
|
20
|
+
- pattern: AI
|
|
21
|
+
- pattern: ID
|
|
22
|
+
- pattern: FAQ
|
|
23
|
+
- pattern: PR
|
|
24
|
+
- pattern: OS
|
|
25
|
+
- pattern: CLI
|
|
26
|
+
- pattern: UI
|
|
27
|
+
- pattern: UX
|
|
28
|
+
- pattern: SQL
|
|
29
|
+
- pattern: CSV
|
|
30
|
+
- pattern: XML
|
|
31
|
+
- pattern: TODO
|
|
32
|
+
- pattern: NOTE
|
|
33
|
+
- pattern: TIP
|
|
34
|
+
# 通信と符号化。技術文書でなくても説明なしで通じる。
|
|
35
|
+
- pattern: DNS
|
|
36
|
+
- pattern: IP
|
|
37
|
+
- pattern: TCP
|
|
38
|
+
- pattern: UDP
|
|
39
|
+
- pattern: SSH
|
|
40
|
+
- pattern: SSL
|
|
41
|
+
- pattern: TLS
|
|
42
|
+
- pattern: VPN
|
|
43
|
+
- pattern: LAN
|
|
44
|
+
- pattern: UTF
|
|
45
|
+
- pattern: ASCII
|
|
46
|
+
# 機器・形式・単位。
|
|
47
|
+
- pattern: PC
|
|
48
|
+
- pattern: IT
|
|
49
|
+
- pattern: SDK
|
|
50
|
+
- pattern: IDE
|
|
51
|
+
- pattern: PNG
|
|
52
|
+
- pattern: JPEG
|
|
53
|
+
- pattern: GIF
|
|
54
|
+
- pattern: SVG
|
|
55
|
+
- pattern: QR
|
|
56
|
+
- pattern: GPS
|
|
57
|
+
- pattern: SNS
|
|
58
|
+
- pattern: TV
|
|
59
|
+
- pattern: KB
|
|
60
|
+
- pattern: MB
|
|
61
|
+
- pattern: GB
|
|
62
|
+
- pattern: TB
|
|
63
|
+
# 役職と国・地域。
|
|
64
|
+
- pattern: CEO
|
|
65
|
+
- pattern: CTO
|
|
66
|
+
- pattern: CFO
|
|
67
|
+
- pattern: US
|
|
68
|
+
- pattern: USA
|
|
69
|
+
- pattern: UK
|
|
70
|
+
- pattern: EU
|
|
71
|
+
- pattern: UN
|
|
72
|
+
- pattern: DNA
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Words that name what a superlative is measured against. With one in the sentence, "the fastest" is a claim, not a boast.
|
|
2
|
+
id: comparison-marker
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: among
|
|
6
|
+
- pattern: than
|
|
7
|
+
- pattern: compared
|
|
8
|
+
- pattern: based on
|
|
9
|
+
- pattern: according
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Labels of a page's date stamp. A paragraph holding only one of these and a date is page metadata, not preamble.
|
|
2
|
+
id: date-stamp-label
|
|
3
|
+
language: en
|
|
4
|
+
entries:
|
|
5
|
+
- pattern: Last updated
|
|
6
|
+
- pattern: Last updated on
|
|
7
|
+
- pattern: Updated
|
|
8
|
+
- pattern: Updated on
|
|
9
|
+
- pattern: Last modified
|
|
10
|
+
- pattern: Last reviewed
|
|
11
|
+
- pattern: Published
|
|
12
|
+
- pattern: Published on
|
|
13
|
+
- pattern: Posted
|
|
14
|
+
- pattern: Posted on
|
|
15
|
+
- pattern: Date
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# Words written inside brackets before an acronym to define it: Service Level Agreement (hereinafter "SLA"),
|
|
2
|
+
# (the "SLA"), (aka the "GDPR"). They count only when the brackets hold nothing but these words and the acronym.
|
|
3
|
+
# Matched without regard to case.
|
|
4
|
+
id: definition-marker
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: the
|
|
8
|
+
- pattern: hereinafter
|
|
9
|
+
- pattern: hereafter
|
|
10
|
+
- pattern: referred to as
|
|
11
|
+
- pattern: called
|
|
12
|
+
- pattern: aka
|
|
13
|
+
- pattern: a.k.a.
|
|
14
|
+
- pattern: also known as
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# Words written inside brackets after an acronym to close its definition. English puts the defining words before the
|
|
2
|
+
# acronym (hereinafter "SLA"), so there are none; the list exists so both languages name the same two lists.
|
|
3
|
+
id: definition-verb
|
|
4
|
+
language: en
|
|
5
|
+
entries: []
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Adverbs of degree that only a gradable adjective takes: "very surprised" is a state, "very approved" is not English.
|
|
2
|
+
# A past participle right after one of these is not a passive. "more" and "overly" are left out: they also go with a
|
|
3
|
+
# verb ("No issue was more discussed", "The model was overly simplified"), and so does "so" ("to be so owned").
|
|
4
|
+
id: degree-adverb
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: very
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Word pairs that doubled-word leaves alone: repeats that grammar allows. "had had" is the past perfect of have;
|
|
2
|
+
# "that that" a conjunction and a demonstrative, which the tagger does not always tell apart; "do do" is emphatic do
|
|
3
|
+
# ("We do do manual reviews"); "her her" an object and a possessive ("give her her copy"); "no no" an interjection
|
|
4
|
+
# the tagger reads as two determiners. "is is" is left out: "What it is is a plan" is rare in business text, "The
|
|
5
|
+
# plan is is ready" is a common slip.
|
|
6
|
+
#
|
|
7
|
+
# A phrasal verb's particle followed by the same preposition ("sign in in advance", "set up up to 10") is listed with
|
|
8
|
+
# the words around it, so that "stored in in the folder" and "set up up the environment" are still slips. An entry
|
|
9
|
+
# matches when the sentence has all its words in a row, the pair among them.
|
|
10
|
+
id: doubled-word
|
|
11
|
+
language: en
|
|
12
|
+
entries:
|
|
13
|
+
- pattern: had had
|
|
14
|
+
- pattern: that that
|
|
15
|
+
- pattern: do do
|
|
16
|
+
- pattern: her her
|
|
17
|
+
- pattern: no no
|
|
18
|
+
- pattern: sign in in
|
|
19
|
+
- pattern: signed in in
|
|
20
|
+
- pattern: log in in
|
|
21
|
+
- pattern: logged in in
|
|
22
|
+
- pattern: opt in in
|
|
23
|
+
- pattern: opted in in
|
|
24
|
+
- pattern: check in in
|
|
25
|
+
- pattern: checked in in
|
|
26
|
+
- pattern: log on on
|
|
27
|
+
- pattern: logged on on
|
|
28
|
+
- pattern: turn on on
|
|
29
|
+
- pattern: turned on on
|
|
30
|
+
- pattern: up up to
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
# Phrases that hand the reader over to the list or table below. A sentence with one is not counted by heading-echo,
|
|
2
|
+
# even when it repeats the heading's words. "the following" is left out: "the following day" hands over nothing.
|
|
3
|
+
id: lead-in
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: as follows
|
|
7
|
+
- pattern: listed below
|
|
8
|
+
- pattern: set out below
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Pronouns that are both an object and a possessive: "give her the book", "her book". The tagger tags every "her" as
|
|
2
|
+
# a possessive (PRP$), so the adapter does not mark these Poss=Yes, and "sent her our report" is not read as two
|
|
3
|
+
# determiners in a row.
|
|
4
|
+
id: object-or-possessive
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: her
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Past participles that, after be, describe a state rather than an action someone did: "We are delighted to offer",
|
|
2
|
+
# "The report is based on a survey", "you are entitled to". There is no hidden actor to name, so agentless-passive
|
|
3
|
+
# leaves them alone. A pattern of several words matches only that phrase: "logged in to" and "located in" are states,
|
|
4
|
+
# "Errors are logged in the console" and "The missing file was located" are passives. Words that are passives in
|
|
5
|
+
# business text as often as states are left out: committed ("an offence was committed"), linked ("documents are
|
|
6
|
+
# linked throughout"), required ("Receipts are required"). "interested" needs no entry: the tagger already reads it
|
|
7
|
+
# as an adjective.
|
|
8
|
+
id: stative-participle
|
|
9
|
+
language: en
|
|
10
|
+
entries:
|
|
11
|
+
- pattern: delighted
|
|
12
|
+
- pattern: excited
|
|
13
|
+
- pattern: surprised
|
|
14
|
+
- pattern: dissatisfied
|
|
15
|
+
- pattern: concerned
|
|
16
|
+
- pattern: nuanced
|
|
17
|
+
- pattern: gone
|
|
18
|
+
- pattern: involved
|
|
19
|
+
- pattern: based
|
|
20
|
+
- pattern: located in
|
|
21
|
+
- pattern: entitled
|
|
22
|
+
- pattern: related
|
|
23
|
+
- pattern: associated
|
|
24
|
+
- pattern: focused on
|
|
25
|
+
- pattern: logged in to
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Words that say what a superlative is the most of. position is the side of the superlative they stand on.
|
|
2
|
+
# English puts them after the superlative's noun phrase, with a noun phrase after them: "the best pizza in Chicago",
|
|
3
|
+
# "the most famous of the sculptures".
|
|
4
|
+
id: superlative-scope
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: in
|
|
8
|
+
position: after
|
|
9
|
+
- pattern: of
|
|
10
|
+
position: after
|
package/package.json
CHANGED
package/src/lexicons.ts
CHANGED
|
@@ -8,6 +8,8 @@ const DIR = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons");
|
|
|
8
8
|
|
|
9
9
|
const isRecord = (value: unknown): value is Record<string, unknown> => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
10
10
|
|
|
11
|
+
const POSITIONS = ["before", "after"] as const;
|
|
12
|
+
|
|
11
13
|
const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
12
14
|
if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
|
|
13
15
|
const weight = raw["weight"];
|
|
@@ -16,6 +18,7 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
|
16
18
|
pattern: raw["pattern"],
|
|
17
19
|
weight: typeof weight === "number" ? weight : undefined,
|
|
18
20
|
instead_of: typeof instead === "string" ? instead : undefined,
|
|
21
|
+
position: POSITIONS.find((position) => position === raw["position"]),
|
|
19
22
|
};
|
|
20
23
|
};
|
|
21
24
|
|
package/src/pos.ts
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
import { createRequire } from "node:module";
|
|
2
2
|
import type { Token } from "chaffjs/plugin";
|
|
3
|
+
import { loadLexicons } from "./lexicons.ts";
|
|
4
|
+
import { properNounChecked } from "./proper-noun.ts";
|
|
5
|
+
import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
|
|
3
6
|
|
|
4
7
|
const require = createRequire(import.meta.url);
|
|
5
8
|
|
|
@@ -127,15 +130,38 @@ const inRelativeClause = (tagged: readonly Tagged[], be: number): boolean => {
|
|
|
127
130
|
return antecedent !== undefined && NOMINAL_TAG.has(antecedent.pos);
|
|
128
131
|
};
|
|
129
132
|
|
|
133
|
+
const LEXICONS = loadLexicons();
|
|
134
|
+
|
|
135
|
+
const STATIVE = stativeVocabulary(LEXICONS);
|
|
136
|
+
|
|
130
137
|
const isPassive = (tagged: readonly Tagged[], at: number): boolean => {
|
|
131
138
|
if (tagged[at]?.pos !== "VBN") return false;
|
|
132
139
|
const be = beBefore(tagged, at);
|
|
133
|
-
return be !== -1 && !inRelativeClause(tagged, be);
|
|
140
|
+
return be !== -1 && !inRelativeClause(tagged, be) && !isStativeParticiple(tagged, at, STATIVE);
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
const wordsOf = (name: string): ReadonlySet<string> => new Set((LEXICONS[name] ?? []).map((entry) => entry.pattern.toLowerCase()));
|
|
144
|
+
|
|
145
|
+
const ARTICLES = wordsOf("article");
|
|
146
|
+
|
|
147
|
+
/** 目的語にも所有にもなる語(give her the book / her book)。解析器はいつも PRP$ と付けるので、所有とは言えない。 */
|
|
148
|
+
const OBJECT_OR_POSSESSIVE = wordsOf("object-or-possessive");
|
|
149
|
+
|
|
150
|
+
const POSSESSIVE_TAG = new Set(["PRP$", "WP$"]);
|
|
151
|
+
|
|
152
|
+
type Features = { features?: Readonly<Record<string, string>> };
|
|
153
|
+
|
|
154
|
+
/** 冠詞(PronType=Art)と所有の語(Poss=Yes)は、名詞の前に一つしか立たない。二つ並ぶ our the platform は書き損じ。 */
|
|
155
|
+
const determinerFeatures = (entry: Tagged): Features => {
|
|
156
|
+
if (POSSESSIVE_TAG.has(entry.pos)) return OBJECT_OR_POSSESSIVE.has(entry.value.toLowerCase()) ? {} : { features: { Poss: "Yes" } };
|
|
157
|
+
return entry.pos === "DT" && ARTICLES.has(entry.value.toLowerCase()) ? { features: { PronType: "Art" } } : {};
|
|
134
158
|
};
|
|
135
159
|
|
|
136
160
|
/** 過去分詞は VerbForm=Part。Based on the review, のような分詞の導入句を、命令形の並び(fix the parser, ship it)と見分ける。 */
|
|
137
|
-
const featuresOf = (tagged: readonly Tagged[], at: number):
|
|
138
|
-
|
|
161
|
+
const featuresOf = (tagged: readonly Tagged[], at: number): Features => {
|
|
162
|
+
const entry = tagged[at];
|
|
163
|
+
if (entry === undefined) return {};
|
|
164
|
+
if (entry.pos !== "VBN") return determinerFeatures(entry);
|
|
139
165
|
return { features: isPassive(tagged, at) ? { VerbForm: "Part", Voice: "Pass" } : { VerbForm: "Part" } };
|
|
140
166
|
};
|
|
141
167
|
|
|
@@ -160,7 +186,7 @@ const locate = (text: string, tagged: readonly Tagged[]): Token[] =>
|
|
|
160
186
|
const token = {
|
|
161
187
|
span: { start, end },
|
|
162
188
|
surface: entry.value,
|
|
163
|
-
pos: upos(entry.pos),
|
|
189
|
+
pos: properNounChecked(entry.value, upos(entry.pos)),
|
|
164
190
|
...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
|
|
165
191
|
...featuresOf(tagged, at),
|
|
166
192
|
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
const LETTER = /\p{L}/u;
|
|
2
|
+
const CAPITAL = /\p{Lu}/u;
|
|
3
|
+
const SMALL = /\p{Ll}/u;
|
|
4
|
+
const DIGIT = /\p{N}/u;
|
|
5
|
+
const SYMBOL = /^[\p{Sc}\p{Sm}\p{So}]+$/u;
|
|
6
|
+
|
|
7
|
+
/** 文字の無い語の品詞。数字があれば数、通貨や数学の記号は記号、ほかは句読点(— や …)。 */
|
|
8
|
+
const nonWord = (surface: string): string => {
|
|
9
|
+
if (DIGIT.test(surface)) return "NUM";
|
|
10
|
+
return SYMBOL.test(surface) ? "SYM" : "PUNCT";
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* 解析器が固有名詞とした語を、表記で確かめる。wink は知らない語を固有名詞にするが、英語の固有名詞は大文字で書く。
|
|
15
|
+
* 小文字だけで書いた語(linters、json、e.g. の e)は普通名詞、文字の無い語(—、$)は記号。大文字の無い文字(東京)の語は確かめようがないので触れない。
|
|
16
|
+
*/
|
|
17
|
+
export const properNounChecked = (surface: string, pos: string): string => {
|
|
18
|
+
if (pos !== "PROPN") return pos;
|
|
19
|
+
if (!LETTER.test(surface)) return nonWord(surface);
|
|
20
|
+
return SMALL.test(surface) && !CAPITAL.test(surface) ? "NOUN" : "PROPN";
|
|
21
|
+
};
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { Lexicon } from "chaffjs/plugin";
|
|
2
|
+
|
|
3
|
+
/** 解析器が返す 1 語。pos は Penn Treebank。 */
|
|
4
|
+
type TaggedWord = { readonly value: string; readonly pos: string };
|
|
5
|
+
|
|
6
|
+
/** 状態を表す過去分詞の句(語の並び)と、程度の副詞。どちらも語彙表から引く。 */
|
|
7
|
+
export type StativeVocabulary = {
|
|
8
|
+
readonly phrases: readonly (readonly string[])[];
|
|
9
|
+
readonly degreeAdverbs: ReadonlySet<string>;
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
const wordsOf = (lexicon: Lexicon | undefined): string[][] =>
|
|
13
|
+
(lexicon ?? [])
|
|
14
|
+
.map((entry) =>
|
|
15
|
+
entry.pattern
|
|
16
|
+
.toLowerCase()
|
|
17
|
+
.split(" ")
|
|
18
|
+
.filter((word) => word !== ""),
|
|
19
|
+
)
|
|
20
|
+
.filter((words) => words.length > 0);
|
|
21
|
+
|
|
22
|
+
export const stativeVocabulary = (lexicons: Readonly<Record<string, Lexicon>>): StativeVocabulary => ({
|
|
23
|
+
phrases: wordsOf(lexicons["stative-participle"]),
|
|
24
|
+
degreeAdverbs: new Set(wordsOf(lexicons["degree-adverb"]).flat()),
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
const wordAt = (tagged: readonly TaggedWord[], at: number): string | undefined => tagged[at]?.value.toLowerCase();
|
|
28
|
+
|
|
29
|
+
const startsPhrase = (tagged: readonly TaggedWord[], at: number, phrase: readonly string[]): boolean =>
|
|
30
|
+
phrase.every((word, offset) => wordAt(tagged, at + offset) === word);
|
|
31
|
+
|
|
32
|
+
/** very は段階のある形容詞にしか付かない。very surprised は言えても very approved は言えない。 */
|
|
33
|
+
const afterDegreeAdverb = (tagged: readonly TaggedWord[], at: number, vocabulary: StativeVocabulary): boolean => {
|
|
34
|
+
const before = wordAt(tagged, at - 1);
|
|
35
|
+
return before !== undefined && vocabulary.degreeAdverbs.has(before);
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* be の後の過去分詞が、誰かのした動作ではなく状態を表しているか。
|
|
40
|
+
* We are delighted to offer / is based on / are entitled to には名指すべき動作主がいない。
|
|
41
|
+
*/
|
|
42
|
+
export const isStativeParticiple = (tagged: readonly TaggedWord[], at: number, vocabulary: StativeVocabulary): boolean =>
|
|
43
|
+
vocabulary.phrases.some((phrase) => startsPhrase(tagged, at, phrase)) || afterDegreeAdverb(tagged, at, vocabulary);
|