@chaffjs/lang-en 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser/wink-modules.d.ts +4 -0
- package/dist/browser/wink-modules.d.ts.map +1 -1
- package/dist/browser/wink-modules.js +5 -2
- package/dist/browser/wink-modules.js.map +1 -1
- package/dist/currency-amounts.d.ts +14 -0
- package/dist/currency-amounts.d.ts.map +1 -0
- package/dist/currency-amounts.js +38 -0
- package/dist/currency-amounts.js.map +1 -0
- package/dist/pos.d.ts.map +1 -1
- package/dist/pos.js +21 -5
- package/dist/pos.js.map +1 -1
- package/dist/proper-noun.d.ts +11 -0
- package/dist/proper-noun.d.ts.map +1 -1
- package/dist/proper-noun.js +31 -0
- package/dist/proper-noun.js.map +1 -1
- package/dist/structure.d.ts.map +1 -1
- package/dist/structure.js +26 -14
- package/dist/structure.js.map +1 -1
- package/dist/unit-case.d.ts +4 -0
- package/dist/unit-case.d.ts.map +1 -0
- package/dist/unit-case.js +11 -0
- package/dist/unit-case.js.map +1 -0
- package/dist/wink-modules.d.ts +5 -0
- package/dist/wink-modules.d.ts.map +1 -1
- package/dist/wink-modules.js +5 -0
- package/dist/wink-modules.js.map +1 -1
- package/lexicons/age-reference-label.yaml +11 -0
- package/lexicons/age-time-mark.yaml +8 -0
- package/lexicons/annual-holidays-label.yaml +19 -0
- package/lexicons/annual-holidays-word.yaml +28 -0
- package/lexicons/annual-pay-label.yaml +24 -0
- package/lexicons/annual-pay-word.yaml +52 -0
- package/lexicons/any-year-marker.yaml +21 -0
- package/lexicons/balance-column.yaml +27 -0
- package/lexicons/benefit-limit-word.yaml +16 -0
- package/lexicons/birth-date-label.yaml +8 -0
- package/lexicons/bmi-term.yaml +12 -0
- package/lexicons/bmi-unit.yaml +6 -0
- package/lexicons/change-column.yaml +28 -0
- package/lexicons/closing-balance-label.yaml +35 -0
- package/lexicons/cumulative-table.yaml +35 -0
- package/lexicons/currency-column.yaml +13 -0
- package/lexicons/date-deadline-mark.yaml +11 -0
- package/lexicons/dated-column-label.yaml +17 -0
- package/lexicons/dated-pair-label.yaml +210 -0
- package/lexicons/dated-span-label.yaml +12 -0
- package/lexicons/deductible-basis.yaml +26 -0
- package/lexicons/deductible-word.yaml +11 -0
- package/lexicons/discount-rate-label.yaml +35 -0
- package/lexicons/email-sentence-end.yaml +8 -0
- package/lexicons/event-bill-word.yaml +9 -0
- package/lexicons/event-due-word.yaml +7 -0
- package/lexicons/event-payment-deadline-word.yaml +12 -0
- package/lexicons/event-payment-word.yaml +7 -0
- package/lexicons/event-start-label.yaml +10 -0
- package/lexicons/event-ticket-word.yaml +8 -0
- package/lexicons/event-trip-word.yaml +13 -0
- package/lexicons/fact-age-joiner.yaml +8 -0
- package/lexicons/fact-age-limit.yaml +9 -0
- package/lexicons/fact-age-mark.yaml +13 -0
- package/lexicons/fact-amount-condition.yaml +15 -0
- package/lexicons/fact-count-column.yaml +10 -0
- package/lexicons/fact-count-counter.yaml +11 -0
- package/lexicons/fact-count-not-whole.yaml +19 -0
- package/lexicons/fact-count-restating.yaml +8 -0
- package/lexicons/fact-count-separate-rows.yaml +7 -0
- package/lexicons/fact-count-whole.yaml +11 -0
- package/lexicons/fact-document-term.yaml +39 -0
- package/lexicons/fact-label-qualifier.yaml +10 -0
- package/lexicons/fact-quantity-period.yaml +7 -0
- package/lexicons/fact-quantity-self.yaml +9 -0
- package/lexicons/fact-quantity-verb.yaml +8 -0
- package/lexicons/fact-rate-base.yaml +15 -0
- package/lexicons/fact-rate-joiner.yaml +16 -0
- package/lexicons/fact-rate-kind.yaml +9 -0
- package/lexicons/fact-rate-period.yaml +14 -0
- package/lexicons/fact-reading-clause.yaml +10 -0
- package/lexicons/fact-reading-gauge.yaml +13 -0
- package/lexicons/fact-reading-verb.yaml +9 -0
- package/lexicons/fact-term-self.yaml +12 -0
- package/lexicons/fact-term-start.yaml +7 -0
- package/lexicons/fact-term-verb.yaml +9 -0
- package/lexicons/gpa-label.yaml +19 -0
- package/lexicons/grade-point.yaml +40 -0
- package/lexicons/heading-number-label.yaml +2 -6
- package/lexicons/installment-label.yaml +33 -0
- package/lexicons/installment-word.yaml +45 -0
- package/lexicons/interest-label.yaml +33 -0
- package/lexicons/item-name-column.yaml +10 -0
- package/lexicons/lab-result-column.yaml +23 -0
- package/lexicons/label-product-word.yaml +36 -0
- package/lexicons/line-amount-column.yaml +4 -0
- package/lexicons/meter-event.yaml +22 -0
- package/lexicons/meter-reading-label.yaml +26 -0
- package/lexicons/meter-unit.yaml +16 -0
- package/lexicons/net-amount-label.yaml +66 -0
- package/lexicons/net-amount-period.yaml +19 -0
- package/lexicons/net-gross-weight-column.yaml +23 -0
- package/lexicons/number-name-word.yaml +49 -0
- package/lexicons/order-quantity-column.yaml +33 -0
- package/lexicons/overnight-work-mark.yaml +13 -0
- package/lexicons/pack-count-column.yaml +13 -0
- package/lexicons/pack-price-column.yaml +12 -0
- package/lexicons/per-unit-mark.yaml +6 -0
- package/lexicons/period-label.yaml +7 -0
- package/lexicons/period-multiple-label.yaml +16 -0
- package/lexicons/period-multiple-word.yaml +28 -0
- package/lexicons/period-part-after.yaml +11 -0
- package/lexicons/period-part-length-verb.yaml +9 -0
- package/lexicons/period-part-marker.yaml +11 -0
- package/lexicons/period-part-range.yaml +9 -0
- package/lexicons/period-part-unnamed.yaml +16 -0
- package/lexicons/period-part-word.yaml +7 -0
- package/lexicons/person-label.yaml +26 -0
- package/lexicons/phone-label.yaml +10 -0
- package/lexicons/points-earned-column.yaml +18 -0
- package/lexicons/points-rate-joiner.yaml +8 -0
- package/lexicons/points-unit.yaml +10 -0
- package/lexicons/quantity-column.yaml +27 -0
- package/lexicons/quantity-range-word.yaml +1 -1
- package/lexicons/range-label.yaml +10 -0
- package/lexicons/rate-unit.yaml +19 -0
- package/lexicons/ratio-label.yaml +14 -0
- package/lexicons/ratio-term.yaml +18 -0
- package/lexicons/reference-bound-mark.yaml +12 -0
- package/lexicons/reference-column-heading.yaml +10 -0
- package/lexicons/reference-column.yaml +18 -0
- package/lexicons/reference-flag.yaml +20 -0
- package/lexicons/reference-label.yaml +151 -0
- package/lexicons/reference-unit-column.yaml +8 -0
- package/lexicons/reference-unit.yaml +81 -0
- package/lexicons/reporting-comparison.yaml +29 -0
- package/lexicons/reporting-current.yaml +14 -0
- package/lexicons/reporting-forecast.yaml +25 -0
- package/lexicons/reporting-period-frame.yaml +27 -0
- package/lexicons/reporting-period-line.yaml +11 -0
- package/lexicons/reporting-results-word.yaml +9 -0
- package/lexicons/reporting-year-frame.yaml +10 -0
- package/lexicons/request-offer.yaml +12 -0
- package/lexicons/revision-column.yaml +17 -0
- package/lexicons/revision-direction.yaml +29 -0
- package/lexicons/serial-label.yaml +13 -0
- package/lexicons/service-charge-label.yaml +6 -0
- package/lexicons/share-amount-label.yaml +20 -0
- package/lexicons/share-column-rate-header.yaml +21 -0
- package/lexicons/shift-column.yaml +23 -0
- package/lexicons/summed-counter.yaml +45 -0
- package/lexicons/table-scale-unit.yaml +17 -0
- package/lexicons/table-unit-caption.yaml +7 -0
- package/lexicons/tax-price-column.yaml +28 -0
- package/lexicons/tier-label.yaml +31 -0
- package/lexicons/tier-unit.yaml +9 -0
- package/lexicons/timed-pair-label.yaml +54 -0
- package/lexicons/total-label-qualifier.yaml +12 -5
- package/lexicons/total-label.yaml +4 -2
- package/lexicons/total-noun-listed.yaml +9 -0
- package/lexicons/total-noun-qualifier.yaml +10 -0
- package/lexicons/total-noun-unsummed.yaml +9 -0
- package/lexicons/total-period-qualifier.yaml +17 -0
- package/lexicons/transcript-column.yaml +23 -0
- package/lexicons/unit-area.yaml +10 -0
- package/lexicons/unit-concentration.yaml +71 -0
- package/lexicons/unit-price-column.yaml +33 -0
- package/lexicons/unit-volume.yaml +4 -0
- package/lexicons/use-amount-word.yaml +11 -0
- package/lexicons/variance-column.yaml +22 -0
- package/lexicons/weekly-days-off.yaml +23 -0
- package/lexicons/year-span-joiner.yaml +12 -0
- package/package.json +1 -1
- package/src/browser/wink-modules.ts +6 -2
- package/src/browser/wink.d.ts +11 -1
- package/src/currency-amounts.ts +51 -0
- package/src/pos.ts +27 -5
- package/src/proper-noun.ts +45 -0
- package/src/structure.ts +33 -15
- package/src/unit-case.ts +14 -0
- package/src/wink-modules.ts +6 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
import winkPosTaggerModule from "wink-pos-tagger";
|
|
2
2
|
import winkLexicon from "wink-lexicon/src/lexicon.js";
|
|
3
|
+
import winkWordNetWords from "wink-lexicon/src/wn-words.js";
|
|
4
|
+
import winkWordNetSenses from "wink-lexicon/src/wn-word-senses.js";
|
|
3
5
|
|
|
4
|
-
// wink-modules.ts in a bundle: the tagger and its vocabulary bundled. The vocabulary
|
|
5
|
-
// imports, so the bundle holds
|
|
6
|
+
// wink-modules.ts in a bundle: the tagger and its vocabulary bundled. The vocabulary and WordNet are the modules the
|
|
7
|
+
// tagger itself imports, so the bundle holds them once.
|
|
6
8
|
|
|
7
9
|
export const winkPosTagger = (): unknown => winkPosTaggerModule;
|
|
8
10
|
|
|
9
11
|
export const winkLexiconWords = (): unknown => winkLexicon;
|
|
12
|
+
|
|
13
|
+
export const winkWordNet = (): { readonly words: unknown; readonly senses: unknown } => ({ words: winkWordNetWords, senses: winkWordNetSenses });
|
package/src/browser/wink.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// wink ships no types. The browser build imports these
|
|
1
|
+
// wink ships no types. The browser build imports these CommonJS modules and reads them as unknown.
|
|
2
2
|
declare module "wink-pos-tagger" {
|
|
3
3
|
const factory: unknown;
|
|
4
4
|
export default factory;
|
|
@@ -8,3 +8,13 @@ declare module "wink-lexicon/src/lexicon.js" {
|
|
|
8
8
|
const words: unknown;
|
|
9
9
|
export default words;
|
|
10
10
|
}
|
|
11
|
+
|
|
12
|
+
declare module "wink-lexicon/src/wn-words.js" {
|
|
13
|
+
const words: unknown;
|
|
14
|
+
export default words;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
declare module "wink-lexicon/src/wn-word-senses.js" {
|
|
18
|
+
const senses: unknown;
|
|
19
|
+
export default senses;
|
|
20
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Which currency an amount is written in, read from the marks of lexicon currency-notation: before the number ($1,320,
|
|
2
|
+
// JPY 1,320) or after it (1,320 yen, 1,320 million dollars, 1,320 USD). The tree's quantity is the number alone; a word of
|
|
3
|
+
// magnitude stays outside it, as it does for "$12 million".
|
|
4
|
+
|
|
5
|
+
import { startsWithUnit } from "./unit-case.ts";
|
|
6
|
+
|
|
7
|
+
export type CurrencyMarks = {
|
|
8
|
+
readonly before: readonly string[];
|
|
9
|
+
readonly after: readonly string[];
|
|
10
|
+
/** Words of magnitude that may stand between the number and a mark after it ("million" in "1,320 million yen"). */
|
|
11
|
+
readonly multipliers: readonly string[];
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const LATIN = /[A-Za-z]/u;
|
|
15
|
+
const WORD_CHAR = /[\p{L}\p{N}_]/u;
|
|
16
|
+
/** A year written alone. "in 2020 dollars" is a year's money, not an amount of 2,020. */
|
|
17
|
+
const YEAR = /^(?:1[89]|2[01])\d{2}$/u;
|
|
18
|
+
|
|
19
|
+
const isGap = (char: string | undefined): boolean => char === " " || char === "\t";
|
|
20
|
+
|
|
21
|
+
/** A Latin mark is a whole word: "XUSD 5" is not USD, "1,320 USDT" is not USD. */
|
|
22
|
+
const standsAlone = (mark: string, before: string, after: string): boolean => !LATIN.test(mark) || (!WORD_CHAR.test(before) && !WORD_CHAR.test(after));
|
|
23
|
+
|
|
24
|
+
/** The mark that touches the number, allowing one space: "US$1,320" is read with "$", as it always was. */
|
|
25
|
+
export const currencyBefore = (text: string, at: number, marks: CurrencyMarks): string | undefined => {
|
|
26
|
+
const end = isGap(text[at - 1]) ? at - 1 : at;
|
|
27
|
+
return marks.before
|
|
28
|
+
.filter((mark) => text.startsWith(mark, end - mark.length) && standsAlone(mark, text.charAt(end - mark.length - 1), ""))
|
|
29
|
+
.toSorted((left, right) => left.length - right.length)[0];
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
const MULTIPLIER_WORD = /^[ \t]([A-Za-z]+)(?=[ \t])/u;
|
|
33
|
+
|
|
34
|
+
/** "1,320 million yen": the space and the word of magnitude before the mark, or nothing. */
|
|
35
|
+
const multiplierLength = (rest: string, multipliers: readonly string[]): number => {
|
|
36
|
+
const word = MULTIPLIER_WORD.exec(rest)?.[1] ?? "";
|
|
37
|
+
return multipliers.includes(word.toLowerCase()) ? word.length + 1 : 0;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The mark after the number: "1,320 yen", "1,320 million dollars", "1,320USD". A year before it ("2020 dollars") is not an
|
|
42
|
+
* amount, nor is the second half of a time or a ratio (the 30 of "10:30").
|
|
43
|
+
*/
|
|
44
|
+
export const currencyAfter = (text: string, start: number, end: number, marks: CurrencyMarks): string | undefined => {
|
|
45
|
+
const scaled = multiplierLength(text.slice(end), marks.multipliers);
|
|
46
|
+
const from = end + scaled + (isGap(text[end + scaled]) ? 1 : 0);
|
|
47
|
+
if (text[start - 1] === ":" || (from > end && scaled === 0 && YEAR.test(text.slice(start, end)))) return undefined;
|
|
48
|
+
return marks.after
|
|
49
|
+
.filter((mark) => startsWithUnit(text, from, mark) && standsAlone(mark, "", text.charAt(from + mark.length)))
|
|
50
|
+
.toSorted((left, right) => right.length - left.length)[0];
|
|
51
|
+
};
|
package/src/pos.ts
CHANGED
|
@@ -2,11 +2,11 @@ import type { Token } from "chaffjs/plugin";
|
|
|
2
2
|
import { straightApostrophes } from "./apostrophe.ts";
|
|
3
3
|
import { loadLexicons } from "./lexicons.ts";
|
|
4
4
|
import { blankLongRuns } from "./long-runs.ts";
|
|
5
|
-
import { lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
|
|
5
|
+
import { capitalisedGuessedVerbsAsNames, isOrdinaryWord, lowercasedAt, properNounChecked, rereadAt, sentenceInitialCommonWord } from "./proper-noun.ts";
|
|
6
6
|
import { isStativeParticiple, stativeVocabulary } from "./stative-participle.ts";
|
|
7
7
|
import { isEmphasisedAdverb, isEmphasisedWord, standsAlone } from "./emphasis.ts";
|
|
8
8
|
import { gerundFeatures } from "./gerund.ts";
|
|
9
|
-
import { winkLexiconWords, winkPosTagger } from "./wink-modules.ts";
|
|
9
|
+
import { winkLexiconWords, winkPosTagger, winkWordNet } from "./wink-modules.ts";
|
|
10
10
|
|
|
11
11
|
/** wink は CommonJS で、辞書を同期に持つ。初期化は 130 ms ほど。 */
|
|
12
12
|
type Tagged = { readonly value: string; readonly pos: string; readonly lemma?: string };
|
|
@@ -224,12 +224,28 @@ const buildVocabulary = (): Vocabulary => {
|
|
|
224
224
|
};
|
|
225
225
|
};
|
|
226
226
|
|
|
227
|
-
|
|
227
|
+
/** WordNet の語の語義の分類(lexicographer file の番号)。WordNet に無い語は空。 */
|
|
228
|
+
type Senses = (word: string) => readonly number[];
|
|
229
|
+
|
|
230
|
+
const isSenseList = (value: unknown): value is readonly number[] => Array.isArray(value) && value.every((sense) => typeof sense === "number");
|
|
231
|
+
|
|
232
|
+
const buildSenses = (): Senses => {
|
|
233
|
+
const { words, senses } = winkWordNet();
|
|
234
|
+
if (!isRecord(words) || !Array.isArray(senses)) throw new Error("wink-lexicon の WordNet が object と配列ではありません");
|
|
235
|
+
return (word) => {
|
|
236
|
+
const index = Object.hasOwn(words, word) ? words[word] : undefined;
|
|
237
|
+
const found: unknown = typeof index === "number" ? senses[index] : undefined;
|
|
238
|
+
return isSenseList(found) ? found : [];
|
|
239
|
+
};
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
const state: { ready: Tagger | undefined; vocabulary: Vocabulary; senses: Senses } = { ready: undefined, vocabulary: () => undefined, senses: () => [] };
|
|
228
243
|
|
|
229
244
|
export const prepare = (): void => {
|
|
230
245
|
if (state.ready !== undefined) return;
|
|
231
246
|
state.ready = build();
|
|
232
247
|
state.vocabulary = buildVocabulary();
|
|
248
|
+
state.senses = buildSenses();
|
|
233
249
|
};
|
|
234
250
|
|
|
235
251
|
export const isReady = (): boolean => state.ready !== undefined;
|
|
@@ -244,6 +260,12 @@ const withEmphasis = (token: Token, text: string, tagged: readonly Tagged[], at:
|
|
|
244
260
|
return { ...token, lemma: (token.lemma ?? token.surface).toLowerCase(), features: { ...token.features, Emph: "Yes" } };
|
|
245
261
|
};
|
|
246
262
|
|
|
263
|
+
/** 名前と読んだ語が、小文字で書けばふつうの語(Green、Course)なら OrdinaryWord=Yes。名前の書き分けが、別の二つの語を比べないため。 */
|
|
264
|
+
const withOrdinaryWord = (token: Token): Token =>
|
|
265
|
+
token.pos === "PROPN" && isOrdinaryWord(token.surface, state.vocabulary, state.senses)
|
|
266
|
+
? { ...token, features: { ...token.features, OrdinaryWord: "Yes" } }
|
|
267
|
+
: token;
|
|
268
|
+
|
|
247
269
|
/**
|
|
248
270
|
* wink は位置を返さないので、表層を順に照合して復元する。
|
|
249
271
|
* 見つからないものは飛ばし、カーソルは進めない。位置の当てずっぽうを下流に流さない。
|
|
@@ -263,7 +285,7 @@ const locate = (text: string, tagged: readonly Tagged[]): Token[] => {
|
|
|
263
285
|
...(entry.lemma === undefined ? {} : { lemma: entry.lemma }),
|
|
264
286
|
...featuresOf(tagged, at),
|
|
265
287
|
};
|
|
266
|
-
tokens.push(withEmphasis(token, text, tagged, at));
|
|
288
|
+
tokens.push(withOrdinaryWord(withEmphasis(token, text, tagged, at)));
|
|
267
289
|
cursor = end;
|
|
268
290
|
});
|
|
269
291
|
return tokens;
|
|
@@ -286,5 +308,5 @@ export const tokenize = (text: string): Token[] | undefined => {
|
|
|
286
308
|
if (tagger === undefined) return undefined;
|
|
287
309
|
// 語は解析させた字(don't)で返す。語彙表の語と同じ字で比べられる。span は本文を指したまま。
|
|
288
310
|
const words = straightApostrophes(blankLongRuns(text, RUN_LIMIT));
|
|
289
|
-
return locate(words, withSentenceInitialCase(tagger, words, tagged(tagger, words)));
|
|
311
|
+
return locate(words, capitalisedGuessedVerbsAsNames(withSentenceInitialCase(tagger, words, tagged(tagger, words)), state.vocabulary, state.senses));
|
|
290
312
|
};
|
package/src/proper-noun.ts
CHANGED
|
@@ -62,3 +62,48 @@ export const rereadAt = (entries: readonly TaggedWord[], at: number, again: read
|
|
|
62
62
|
const word: TaggedWord = { value: entry.value, pos: reread.pos, ...(reread.lemma === undefined ? {} : { lemma: reread.lemma }) };
|
|
63
63
|
return entries.map((original, index) => (index === at ? word : original));
|
|
64
64
|
};
|
|
65
|
+
|
|
66
|
+
/** 解析器が、語彙に無い語の品詞を決めかねて前後から当てる原形の動詞。 */
|
|
67
|
+
const GUESSED_VERB_TAG = new Set(["VB", "VBP"]);
|
|
68
|
+
|
|
69
|
+
/** WordNet の動詞の語義の分類(lexicographer file)の番号。29 verb.body から 43 verb.weather まで。 */
|
|
70
|
+
const FIRST_VERB_SENSE = 29;
|
|
71
|
+
const LAST_VERB_SENSE = 43;
|
|
72
|
+
|
|
73
|
+
const hasVerbSense = (senses: readonly number[]): boolean => senses.some((sense) => sense >= FIRST_VERB_SENSE && sense <= LAST_VERB_SENSE);
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* 文の途中で頭だけ大文字の語を、解析器が語彙に無いまま原形の動詞と当てたもので、WordNet も動詞として知らないものは、名前として読み直す。
|
|
77
|
+
* wink は WordNet にだけ名詞としてある語(Helen、Sarah)を前の to に合わせて動詞にするが(returned to Helen Marsh)、動詞の語義の無い語は動詞でない。
|
|
78
|
+
* 語彙に載っている語(how to Change の Change)、WordNet が動詞と知る語(to Detach)、文頭の語は解析器の読みのまま。
|
|
79
|
+
*/
|
|
80
|
+
export const capitalisedGuessedVerbsAsNames = (
|
|
81
|
+
tagged: readonly TaggedWord[],
|
|
82
|
+
tagsOf: (word: string) => readonly string[] | undefined,
|
|
83
|
+
sensesOf: (word: string) => readonly number[],
|
|
84
|
+
): readonly TaggedWord[] => {
|
|
85
|
+
const first = tagged.findIndex((entry) => LETTER.test(entry.value));
|
|
86
|
+
const isUnknownNonVerb = (word: string): boolean => tagsOf(word) === undefined && !hasVerbSense(sensesOf(word));
|
|
87
|
+
const isGuessedName = (entry: TaggedWord, at: number): boolean =>
|
|
88
|
+
at > first && GUESSED_VERB_TAG.has(entry.pos) && CAPITALISED.test(entry.value) && isUnknownNonVerb(entry.value.toLowerCase());
|
|
89
|
+
return tagged.map((entry, at) => (isGuessedName(entry, at) ? { ...entry, pos: "NNP" } : entry));
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
/** WordNet の語義の分類のうち、名前の載る所。15 noun.location(Paris)と 18 noun.person(Helen)。 */
|
|
93
|
+
const NOUN_LOCATION = 15;
|
|
94
|
+
const NOUN_PERSON = 18;
|
|
95
|
+
const NAME_SENSES: ReadonlySet<number> = new Set([NOUN_LOCATION, NOUN_PERSON]);
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* 頭だけ大文字の語が、小文字で書けばふつうの語か。解析器の語彙が普通の語として知るか(course)、WordNet が人や場所でない語義を
|
|
99
|
+
* 知る(green、Greek の言語)。人や場所の語義しか無い語(Helen、Smith)と、どちらにも無い語(Okafor)は名前の語。
|
|
100
|
+
*/
|
|
101
|
+
export const isOrdinaryWord = (
|
|
102
|
+
surface: string,
|
|
103
|
+
tagsOf: (word: string) => readonly string[] | undefined,
|
|
104
|
+
sensesOf: (word: string) => readonly number[],
|
|
105
|
+
): boolean => {
|
|
106
|
+
if (!CAPITALISED.test(surface)) return false;
|
|
107
|
+
const word = surface.toLowerCase();
|
|
108
|
+
return isCommonWord(tagsOf(word)) || sensesOf(word).some((sense) => !NAME_SENSES.has(sense));
|
|
109
|
+
};
|
package/src/structure.ts
CHANGED
|
@@ -7,6 +7,8 @@ import { parseRoman } from "./roman.ts";
|
|
|
7
7
|
import { dates } from "./dates.ts";
|
|
8
8
|
import { definitionScopeDepth, definitions, opensDefinitionScope } from "./definitions.ts";
|
|
9
9
|
import { CHAPTER_DEPTH, PART_DEPTH } from "./depth.ts";
|
|
10
|
+
import { currencyAfter, currencyBefore, type CurrencyMarks } from "./currency-amounts.ts";
|
|
11
|
+
import { isUnitWord, startsWithUnit } from "./unit-case.ts";
|
|
10
12
|
import { continuesAddress, continuesOpen, depthFor, INSERTED, ordinalOf, styleOf } from "./item-style.ts";
|
|
11
13
|
|
|
12
14
|
// Contracts, specifications and statutes in English. core nests what this reads; it does not know
|
|
@@ -229,6 +231,7 @@ const obligations = (text: string): Mention[] => {
|
|
|
229
231
|
|
|
230
232
|
/** Find the number first, then look at what is right before (a currency) and right after (a unit). */
|
|
231
233
|
const NUMBER_RUN = /\d[\d,.]{0,15}/gu;
|
|
234
|
+
const PERCENT_UNITS = (LEXICONS["percent-unit"] ?? []).map((entry) => entry.pattern);
|
|
232
235
|
const UNITS = [
|
|
233
236
|
"business days",
|
|
234
237
|
"business day",
|
|
@@ -246,11 +249,18 @@ const UNITS = [
|
|
|
246
249
|
"hour",
|
|
247
250
|
"minutes",
|
|
248
251
|
"minute",
|
|
249
|
-
"percent",
|
|
250
252
|
"times",
|
|
251
|
-
|
|
253
|
+
...PERCENT_UNITS,
|
|
252
254
|
];
|
|
253
|
-
|
|
255
|
+
|
|
256
|
+
const notation = (position: "before" | "after"): string[] =>
|
|
257
|
+
(LEXICONS["currency-notation"] ?? []).filter((entry) => entry.position === position).map((entry) => entry.pattern);
|
|
258
|
+
|
|
259
|
+
const CURRENCY_MARKS: CurrencyMarks = {
|
|
260
|
+
before: notation("before"),
|
|
261
|
+
after: notation("after"),
|
|
262
|
+
multipliers: (LEXICONS["amount-multiplier"] ?? []).map((entry) => entry.pattern),
|
|
263
|
+
};
|
|
254
264
|
|
|
255
265
|
/** "30." の終わりの点は文の終わり。数の一部にしない。 */
|
|
256
266
|
const withoutTrailingPunctuation = (run: string): string => {
|
|
@@ -262,16 +272,17 @@ const withoutTrailingPunctuation = (run: string): string => {
|
|
|
262
272
|
/** One space or tab may sit between a number and its unit or currency. */
|
|
263
273
|
const isGap = (char: string | undefined): boolean => char === " " || char === "\t";
|
|
264
274
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
return unit;
|
|
269
|
-
};
|
|
275
|
+
type UnitMatch = (text: string, at: number, unit: string) => boolean;
|
|
276
|
+
|
|
277
|
+
const asWritten: UnitMatch = (text, at, unit) => text.startsWith(unit, at);
|
|
270
278
|
|
|
271
|
-
/**
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
|
|
279
|
+
/** A lexicon unit word is read in any case ("8.5 Percent"). The period words above stay as written until they move to a lexicon. */
|
|
280
|
+
const LEXICON_WORDS: ReadonlySet<string> = new Set(PERCENT_UNITS.filter(isUnitWord));
|
|
281
|
+
const lexiconWordsInAnyCase: UnitMatch = (text, at, unit) => (LEXICON_WORDS.has(unit) ? startsWithUnit(text, at, unit) : asWritten(text, at, unit));
|
|
282
|
+
|
|
283
|
+
const unitAfter = (text: string, from: number, matches: UnitMatch = asWritten): string | undefined => {
|
|
284
|
+
const gap = isGap(text[from]) ? 1 : 0;
|
|
285
|
+
return UNITS.find((candidate) => matches(text, from + gap, candidate) && !isWordChar(text[from + gap + candidate.length]));
|
|
275
286
|
};
|
|
276
287
|
|
|
277
288
|
/** A count of days or months is written "six (6)" or "one thousand (1,000)": never "zero (0)", never "two million (2,000,000)". */
|
|
@@ -295,15 +306,22 @@ const followsNumberWord = (text: string, start: number): boolean => {
|
|
|
295
306
|
|
|
296
307
|
/** "six (6) months": a figure in brackets after a number in words, with the unit after the bracket. */
|
|
297
308
|
const unitAfterBracket = (text: string, start: number, end: number): string | undefined =>
|
|
298
|
-
text[end] === ")" && followsNumberWord(text, start) ? unitAfter(text, end + 1) : undefined;
|
|
309
|
+
text[end] === ")" && followsNumberWord(text, start) ? unitAfter(text, end + 1, lexiconWordsInAnyCase) : undefined;
|
|
299
310
|
|
|
300
311
|
const quantities = (text: string): Mention[] =>
|
|
301
312
|
[...text.matchAll(NUMBER_RUN)].flatMap((match) => {
|
|
302
313
|
const digits = withoutTrailingPunctuation(match[0]);
|
|
303
314
|
const value = Number(digits.replace(/,/gu, ""));
|
|
304
315
|
const end = match.index + digits.length;
|
|
305
|
-
const
|
|
306
|
-
|
|
316
|
+
const before = currencyBefore(text, match.index, CURRENCY_MARKS);
|
|
317
|
+
// A number glued to a letter is part of a word ("A4", "v2"), unless that letter ends a currency code ("USD96").
|
|
318
|
+
const unit = isWordChar(text[match.index - 1])
|
|
319
|
+
? before
|
|
320
|
+
: (unitAfter(text, end, lexiconWordsInAnyCase) ??
|
|
321
|
+
unitAfterBracket(text, match.index, end) ??
|
|
322
|
+
before ??
|
|
323
|
+
currencyAfter(text, match.index, end, CURRENCY_MARKS));
|
|
324
|
+
return unit === undefined || Number.isNaN(value) ? [] : [{ start: match.index, end, attrs: { value, unit } }];
|
|
307
325
|
});
|
|
308
326
|
|
|
309
327
|
const MEASURE_UNITS = (LEXICONS["measure-unit"] ?? []).map((entry) => entry.pattern);
|
package/src/unit-case.ts
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
// Whether a lexicon unit is a word, read in any case, or a symbol, read as written.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* A word is three or more lowercase letters, possibly in several words ("percent", "per cent", "million", "yen"). Anything
|
|
5
|
+
* else is a symbol whose case carries meaning: a capital the lexicon wrote (USD, kDa, mL vs ML), or a lowercase symbol of
|
|
6
|
+
* one or two letters (m vs M, g vs G, mg vs Mg).
|
|
7
|
+
*/
|
|
8
|
+
const UNIT_WORD = /^\p{Ll}{3,}(?: \p{Ll}+)*$/u;
|
|
9
|
+
|
|
10
|
+
export const isUnitWord = (unit: string): boolean => UNIT_WORD.test(unit);
|
|
11
|
+
|
|
12
|
+
/** "8.5 Percent", "198 Million Yen": a unit word starts at `at` in any case; a symbol only as the lexicon wrote it. */
|
|
13
|
+
export const startsWithUnit = (text: string, at: number, unit: string): boolean =>
|
|
14
|
+
isUnitWord(unit) ? text.slice(at, at + unit.length).toLowerCase() === unit : text.startsWith(unit, at);
|
package/src/wink-modules.ts
CHANGED
|
@@ -9,3 +9,9 @@ export const winkPosTagger = (): unknown => require("wink-pos-tagger");
|
|
|
9
9
|
|
|
10
10
|
/** The vocabulary the tagger reads (word → Penn Treebank tags): the one wink-pos-tagger loads from its own dependency. */
|
|
11
11
|
export const winkLexiconWords = (): unknown => createRequire(require.resolve("wink-pos-tagger"))("wink-lexicon/src/lexicon.js");
|
|
12
|
+
|
|
13
|
+
/** WordNet's words (word → index) and each index's senses (lexicographer file numbers): the two the tagger reads for words its vocabulary lacks. */
|
|
14
|
+
export const winkWordNet = (): { readonly words: unknown; readonly senses: unknown } => {
|
|
15
|
+
const fromTagger = createRequire(require.resolve("wink-pos-tagger"));
|
|
16
|
+
return { words: fromTagger("wink-lexicon/src/wn-words.js"), senses: fromTagger("wink-lexicon/src/wn-word-senses.js") };
|
|
17
|
+
};
|