@chaffjs/lang-en 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/definitions.d.ts.map +1 -1
- package/dist/definitions.js +4 -2
- package/dist/definitions.js.map +1 -1
- package/dist/dictionary.d.ts +2 -0
- package/dist/dictionary.d.ts.map +1 -0
- package/dist/dictionary.js +20 -0
- package/dist/dictionary.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/lexicons.d.ts.map +1 -1
- package/dist/lexicons.js +4 -0
- package/dist/lexicons.js.map +1 -1
- package/lexicons/absolute-word.yaml +14 -0
- package/lexicons/abstract-heading.yaml +7 -0
- package/lexicons/age-marker.yaml +10 -0
- package/lexicons/ai-tell.yaml +12 -0
- package/lexicons/approximate-marker.yaml +34 -0
- package/lexicons/article-sound.yaml +83 -0
- package/lexicons/birth-origin.yaml +5 -0
- package/lexicons/bookend-heading.yaml +30 -0
- package/lexicons/chat-citation-marker.yaml +11 -0
- package/lexicons/citation-author-joiner.yaml +11 -0
- package/lexicons/citation-not-author.yaml +40 -0
- package/lexicons/count-counter.yaml +50 -0
- package/lexicons/cushion-phrase.yaml +6 -0
- package/lexicons/date-change-word.yaml +27 -0
- package/lexicons/dictionary.txt +87150 -0
- package/lexicons/double-negative.yaml +25 -0
- package/lexicons/duration-day.yaml +6 -0
- package/lexicons/duration-month.yaml +6 -0
- package/lexicons/duration-week.yaml +6 -0
- package/lexicons/duration-year.yaml +6 -0
- package/lexicons/elapsed-link.yaml +11 -0
- package/lexicons/elapsed-origin.yaml +8 -0
- package/lexicons/elapsed-unit.yaml +6 -0
- package/lexicons/enumeration-frame.yaml +37 -0
- package/lexicons/enumeration-joiner.yaml +9 -0
- package/lexicons/enumeration-negation.yaml +13 -0
- package/lexicons/exception-marker.yaml +10 -0
- package/lexicons/extra-word.yaml +185 -0
- package/lexicons/fact-attribute.yaml +6 -0
- package/lexicons/fact-label-drop.yaml +13 -0
- package/lexicons/fact-label-vague.yaml +16 -0
- package/lexicons/fact-separator.yaml +11 -0
- package/lexicons/fact-value-end.yaml +8 -0
- package/lexicons/figure-in-text.yaml +9 -0
- package/lexicons/figure-noun.yaml +31 -0
- package/lexicons/heading-form.yaml +63 -0
- package/lexicons/heading-number-label.yaml +29 -0
- package/lexicons/homophone-slip.yaml +37 -0
- package/lexicons/known-misspelling.yaml +177 -0
- package/lexicons/label-affix.yaml +7 -0
- package/lexicons/light-verb.yaml +9 -0
- package/lexicons/month-name.yaml +29 -0
- package/lexicons/nominalization-phrase.yaml +27 -0
- package/lexicons/orthographic-variant-distinct.yaml +24 -0
- package/lexicons/paper-section-heading.yaml +18 -0
- package/lexicons/person-form.yaml +9 -0
- package/lexicons/person-not-after.yaml +54 -0
- package/lexicons/person-word.yaml +38 -0
- package/lexicons/place-region.yaml +247 -0
- package/lexicons/placeholder-word.yaml +10 -0
- package/lexicons/pro-con-heading.yaml +15 -0
- package/lexicons/range-frame.yaml +9 -0
- package/lexicons/redundant-expression.yaml +83 -0
- package/lexicons/reference-aside.yaml +21 -0
- package/lexicons/reference-list-heading.yaml +11 -0
- package/lexicons/reference-topic-link.yaml +5 -0
- package/lexicons/reference-topic-vague.yaml +32 -0
- package/lexicons/relative-after.yaml +5 -0
- package/lexicons/relative-before.yaml +5 -0
- package/lexicons/relative-link.yaml +7 -0
- package/lexicons/requirement-either.yaml +5 -0
- package/lexicons/requirement-loophole.yaml +17 -0
- package/lexicons/requirement-marker.yaml +9 -0
- package/lexicons/requirement-modal-must.yaml +7 -0
- package/lexicons/requirement-open-end.yaml +9 -0
- package/lexicons/restating-opener.yaml +22 -0
- package/lexicons/so-opener.yaml +6 -0
- package/lexicons/spelling-ize.yaml +273 -0
- package/lexicons/spelling-variant.yaml +260 -0
- package/lexicons/summary-heading.yaml +13 -0
- package/lexicons/unit-data.yaml +28 -0
- package/lexicons/unit-length.yaml +44 -0
- package/lexicons/unit-mass.yaml +30 -0
- package/lexicons/unit-time.yaml +32 -0
- package/lexicons/unit-volume.yaml +26 -0
- package/lexicons/unnumbered-heading.yaml +27 -0
- package/lexicons/vague-clause-pointer.yaml +10 -0
- package/lexicons/vague-figure-pointer.yaml +14 -0
- package/lexicons/vague-link-text.yaml +20 -0
- package/lexicons/vague-word.yaml +31 -0
- package/lexicons/value-unit.yaml +58 -0
- package/lexicons/very-exception.yaml +17 -0
- package/lexicons/very-intensifier.yaml +6 -0
- package/lexicons/weasel-word.yaml +30 -0
- package/lexicons/word-prefix.yaml +31 -0
- package/lexicons/word-suffix.yaml +81 -0
- package/lexicons/wordy-phrase.yaml +50 -0
- package/package.json +1 -1
- package/src/definitions.ts +4 -2
- package/src/dictionary.ts +22 -0
- package/src/index.ts +2 -0
- package/src/lexicons.ts +4 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Endings that make an inflected or derived form of a dictionary word: unknown-word strips the ending (pattern) and
|
|
2
|
+
# puts the replacement (instead_of, empty when nothing goes back) before looking the stem up. "carried" is "carry",
|
|
3
|
+
# "stopped" is "stop" (a doubled last consonant is undone too), "baking" is "bake". The dictionary itself is
|
|
4
|
+
# dictionary.txt; this list only saves listing every inflected form there.
|
|
5
|
+
id: word-suffix
|
|
6
|
+
language: en
|
|
7
|
+
entries:
|
|
8
|
+
- pattern: s
|
|
9
|
+
instead_of: ""
|
|
10
|
+
- pattern: es
|
|
11
|
+
instead_of: ""
|
|
12
|
+
- pattern: ies
|
|
13
|
+
instead_of: "y"
|
|
14
|
+
- pattern: ied
|
|
15
|
+
instead_of: "y"
|
|
16
|
+
- pattern: ed
|
|
17
|
+
instead_of: ""
|
|
18
|
+
- pattern: ed
|
|
19
|
+
instead_of: "e"
|
|
20
|
+
- pattern: ing
|
|
21
|
+
instead_of: ""
|
|
22
|
+
- pattern: ing
|
|
23
|
+
instead_of: "e"
|
|
24
|
+
- pattern: er
|
|
25
|
+
instead_of: ""
|
|
26
|
+
- pattern: er
|
|
27
|
+
instead_of: "e"
|
|
28
|
+
- pattern: ers
|
|
29
|
+
instead_of: ""
|
|
30
|
+
- pattern: ers
|
|
31
|
+
instead_of: "e"
|
|
32
|
+
- pattern: ier
|
|
33
|
+
instead_of: "y"
|
|
34
|
+
- pattern: iest
|
|
35
|
+
instead_of: "y"
|
|
36
|
+
- pattern: est
|
|
37
|
+
instead_of: ""
|
|
38
|
+
- pattern: est
|
|
39
|
+
instead_of: "e"
|
|
40
|
+
- pattern: ly
|
|
41
|
+
instead_of: ""
|
|
42
|
+
- pattern: ily
|
|
43
|
+
instead_of: "y"
|
|
44
|
+
- pattern: ness
|
|
45
|
+
instead_of: ""
|
|
46
|
+
- pattern: iness
|
|
47
|
+
instead_of: "y"
|
|
48
|
+
- pattern: ment
|
|
49
|
+
instead_of: ""
|
|
50
|
+
- pattern: ments
|
|
51
|
+
instead_of: ""
|
|
52
|
+
- pattern: able
|
|
53
|
+
instead_of: ""
|
|
54
|
+
- pattern: able
|
|
55
|
+
instead_of: "e"
|
|
56
|
+
- pattern: ability
|
|
57
|
+
instead_of: "able"
|
|
58
|
+
- pattern: ity
|
|
59
|
+
instead_of: ""
|
|
60
|
+
- pattern: ful
|
|
61
|
+
instead_of: ""
|
|
62
|
+
- pattern: less
|
|
63
|
+
instead_of: ""
|
|
64
|
+
- pattern: ize
|
|
65
|
+
instead_of: ""
|
|
66
|
+
- pattern: ization
|
|
67
|
+
instead_of: "ize"
|
|
68
|
+
- pattern: ise
|
|
69
|
+
instead_of: ""
|
|
70
|
+
- pattern: isation
|
|
71
|
+
instead_of: "ise"
|
|
72
|
+
- pattern: ation
|
|
73
|
+
instead_of: "e"
|
|
74
|
+
- pattern: ation
|
|
75
|
+
instead_of: ""
|
|
76
|
+
- pattern: ably
|
|
77
|
+
instead_of: "able"
|
|
78
|
+
- pattern: ibly
|
|
79
|
+
instead_of: "ible"
|
|
80
|
+
- pattern: men
|
|
81
|
+
instead_of: "man"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Wordy phrases: several words where one or two say the same. Read by wordy-phrase; rewrite is the short form.
|
|
2
|
+
# Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
|
|
3
|
+
# are left out: contracts and statutes use them on purpose. Matched as whole words, case aside.
|
|
4
|
+
id: wordy-phrase
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: in order to
|
|
8
|
+
rewrite: to
|
|
9
|
+
- pattern: in order for
|
|
10
|
+
rewrite: for
|
|
11
|
+
- pattern: due to the fact that
|
|
12
|
+
rewrite: because
|
|
13
|
+
- pattern: owing to the fact that
|
|
14
|
+
rewrite: because
|
|
15
|
+
- pattern: in view of the fact that
|
|
16
|
+
rewrite: because
|
|
17
|
+
- pattern: in spite of the fact that
|
|
18
|
+
rewrite: although
|
|
19
|
+
- pattern: despite the fact that
|
|
20
|
+
rewrite: although
|
|
21
|
+
- pattern: at this point in time
|
|
22
|
+
rewrite: now
|
|
23
|
+
- pattern: at the present time
|
|
24
|
+
rewrite: now
|
|
25
|
+
- pattern: at this moment in time
|
|
26
|
+
rewrite: now
|
|
27
|
+
- pattern: in the event that
|
|
28
|
+
rewrite: if
|
|
29
|
+
- pattern: in the near future
|
|
30
|
+
rewrite: soon
|
|
31
|
+
- pattern: for the purpose of
|
|
32
|
+
rewrite: to, or for
|
|
33
|
+
- pattern: on a daily basis
|
|
34
|
+
rewrite: daily
|
|
35
|
+
- pattern: on a weekly basis
|
|
36
|
+
rewrite: weekly
|
|
37
|
+
- pattern: on a monthly basis
|
|
38
|
+
rewrite: monthly
|
|
39
|
+
- pattern: on a regular basis
|
|
40
|
+
rewrite: regularly
|
|
41
|
+
- pattern: a large number of
|
|
42
|
+
rewrite: many
|
|
43
|
+
- pattern: has the ability to
|
|
44
|
+
rewrite: can
|
|
45
|
+
- pattern: have the ability to
|
|
46
|
+
rewrite: can
|
|
47
|
+
- pattern: is in a position to
|
|
48
|
+
rewrite: can
|
|
49
|
+
- pattern: until such time as
|
|
50
|
+
rewrite: until
|
package/package.json
CHANGED
package/src/definitions.ts
CHANGED
|
@@ -37,7 +37,9 @@ export const definitions = (text: string): Mention[] =>
|
|
|
37
37
|
const term = match.groups?.["term"];
|
|
38
38
|
if (term === undefined) return [];
|
|
39
39
|
const end = match.index + match[0].length;
|
|
40
|
-
const
|
|
41
|
-
|
|
40
|
+
const inline = match[0].startsWith("(");
|
|
41
|
+
const namesAParty = inline && APPLIES.test(text);
|
|
42
|
+
const scope = isPointer(text, end) || namesAParty ? { scope: "local" } : {};
|
|
43
|
+
return [{ start: match.index, end, attrs: { term, ...scope, ...(inline ? { placement: "inline" } : {}) } }];
|
|
42
44
|
}),
|
|
43
45
|
);
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
|
|
5
|
+
// The English word list (lexicons/dictionary.txt, built by scripts/en-dictionary.ts). It is large, so it is read only
|
|
6
|
+
// when a rule asks for it, and once.
|
|
7
|
+
|
|
8
|
+
const FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons", "dictionary.txt");
|
|
9
|
+
|
|
10
|
+
const loaded: { words: ReadonlySet<string> | undefined } = { words: undefined };
|
|
11
|
+
|
|
12
|
+
const wordsIn = (text: string): ReadonlySet<string> => new Set(text.split("\n").filter((line) => line !== "" && !line.startsWith("#")));
|
|
13
|
+
|
|
14
|
+
export const dictionary = (): ReadonlySet<string> => {
|
|
15
|
+
if (loaded.words !== undefined) return loaded.words;
|
|
16
|
+
try {
|
|
17
|
+
loaded.words = wordsIn(readFileSync(FILE, "utf8"));
|
|
18
|
+
} catch (error) {
|
|
19
|
+
throw new Error(`${FILE}: cannot read the English word list`, { cause: error });
|
|
20
|
+
}
|
|
21
|
+
return loaded.words;
|
|
22
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { sentenceSpans } from "./sentence-split.ts";
|
|
|
5
5
|
import { splitAtQuotedStops } from "./quoted-stop.ts";
|
|
6
6
|
import { reattachClosingQuotes } from "./closing-quote.ts";
|
|
7
7
|
import { structure } from "./structure.ts";
|
|
8
|
+
import { dictionary } from "./dictionary.ts";
|
|
8
9
|
import { isReady, prepare, tokenize } from "./pos.ts";
|
|
9
10
|
import { isJapaneseRun } from "./japanese-run.ts";
|
|
10
11
|
import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
|
|
@@ -62,6 +63,7 @@ export const adapter: LanguageAdapter = {
|
|
|
62
63
|
},
|
|
63
64
|
lexicons: LEXICONS,
|
|
64
65
|
structure,
|
|
66
|
+
dictionary,
|
|
65
67
|
segment: (text: string): Segmentation => {
|
|
66
68
|
const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
|
|
67
69
|
const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
|
package/src/lexicons.ts
CHANGED
|
@@ -14,11 +14,15 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
|
|
|
14
14
|
if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
|
|
15
15
|
const weight = raw["weight"];
|
|
16
16
|
const instead = raw["instead_of"];
|
|
17
|
+
const rewrite = raw["rewrite"];
|
|
18
|
+
const group = raw["group"];
|
|
17
19
|
return {
|
|
18
20
|
pattern: raw["pattern"],
|
|
19
21
|
weight: typeof weight === "number" ? weight : undefined,
|
|
20
22
|
instead_of: typeof instead === "string" ? instead : undefined,
|
|
21
23
|
position: POSITIONS.find((position) => position === raw["position"]),
|
|
24
|
+
rewrite: typeof rewrite === "string" ? rewrite : undefined,
|
|
25
|
+
...(typeof group === "string" ? { group } : {}),
|
|
22
26
|
};
|
|
23
27
|
};
|
|
24
28
|
|