@chaffjs/lang-en 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dictionary.d.ts +2 -0
- package/dist/dictionary.d.ts.map +1 -0
- package/dist/dictionary.js +20 -0
- package/dist/dictionary.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/lexicons/absolute-word.yaml +14 -0
- package/lexicons/abstract-heading.yaml +7 -0
- package/lexicons/age-marker.yaml +10 -0
- package/lexicons/approximate-marker.yaml +34 -0
- package/lexicons/article-sound.yaml +83 -0
- package/lexicons/birth-origin.yaml +5 -0
- package/lexicons/bookend-heading.yaml +30 -0
- package/lexicons/chat-citation-marker.yaml +11 -0
- package/lexicons/citation-author-joiner.yaml +11 -0
- package/lexicons/citation-not-author.yaml +40 -0
- package/lexicons/dictionary.txt +87150 -0
- package/lexicons/duration-day.yaml +6 -0
- package/lexicons/duration-month.yaml +6 -0
- package/lexicons/duration-week.yaml +6 -0
- package/lexicons/duration-year.yaml +6 -0
- package/lexicons/elapsed-link.yaml +11 -0
- package/lexicons/elapsed-origin.yaml +8 -0
- package/lexicons/elapsed-unit.yaml +6 -0
- package/lexicons/exception-marker.yaml +10 -0
- package/lexicons/extra-word.yaml +185 -0
- package/lexicons/fact-attribute.yaml +6 -0
- package/lexicons/fact-label-drop.yaml +13 -0
- package/lexicons/fact-label-vague.yaml +16 -0
- package/lexicons/fact-separator.yaml +11 -0
- package/lexicons/fact-value-end.yaml +8 -0
- package/lexicons/figure-in-text.yaml +9 -0
- package/lexicons/figure-noun.yaml +31 -0
- package/lexicons/heading-form.yaml +63 -0
- package/lexicons/heading-number-label.yaml +29 -0
- package/lexicons/homophone-slip.yaml +37 -0
- package/lexicons/known-misspelling.yaml +177 -0
- package/lexicons/label-affix.yaml +7 -0
- package/lexicons/light-verb.yaml +9 -0
- package/lexicons/month-name.yaml +29 -0
- package/lexicons/orthographic-variant-distinct.yaml +24 -0
- package/lexicons/paper-section-heading.yaml +18 -0
- package/lexicons/person-form.yaml +9 -0
- package/lexicons/person-not-after.yaml +54 -0
- package/lexicons/person-word.yaml +38 -0
- package/lexicons/placeholder-word.yaml +10 -0
- package/lexicons/pro-con-heading.yaml +15 -0
- package/lexicons/redundant-expression.yaml +83 -0
- package/lexicons/reference-aside.yaml +21 -0
- package/lexicons/reference-list-heading.yaml +11 -0
- package/lexicons/reference-topic-link.yaml +5 -0
- package/lexicons/reference-topic-vague.yaml +32 -0
- package/lexicons/relative-after.yaml +5 -0
- package/lexicons/relative-before.yaml +5 -0
- package/lexicons/relative-link.yaml +7 -0
- package/lexicons/restating-opener.yaml +22 -0
- package/lexicons/so-opener.yaml +6 -0
- package/lexicons/summary-heading.yaml +13 -0
- package/lexicons/unit-data.yaml +28 -0
- package/lexicons/unit-length.yaml +44 -0
- package/lexicons/unit-mass.yaml +30 -0
- package/lexicons/unit-time.yaml +32 -0
- package/lexicons/unit-volume.yaml +26 -0
- package/lexicons/unnumbered-heading.yaml +27 -0
- package/lexicons/vague-link-text.yaml +20 -0
- package/lexicons/vague-word.yaml +31 -0
- package/lexicons/value-unit.yaml +58 -0
- package/lexicons/very-exception.yaml +17 -0
- package/lexicons/very-intensifier.yaml +6 -0
- package/lexicons/weasel-word.yaml +30 -0
- package/lexicons/word-prefix.yaml +31 -0
- package/lexicons/word-suffix.yaml +81 -0
- package/lexicons/wordy-phrase.yaml +50 -0
- package/package.json +1 -1
- package/src/dictionary.ts +22 -0
- package/src/index.ts +2 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# Units and counted nouns that follow a number. number-without-unit reads a word after a number as a unit only if it is
|
|
2
|
+
# here or in measure-unit, percent-unit or currency-code; a number followed by any other word ("3 fixed") is not a value.
|
|
3
|
+
id: value-unit
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: dollars
|
|
7
|
+
- pattern: yen
|
|
8
|
+
- pattern: euros
|
|
9
|
+
- pattern: usd
|
|
10
|
+
- pattern: day
|
|
11
|
+
- pattern: days
|
|
12
|
+
- pattern: week
|
|
13
|
+
- pattern: weeks
|
|
14
|
+
- pattern: month
|
|
15
|
+
- pattern: months
|
|
16
|
+
- pattern: year
|
|
17
|
+
- pattern: years
|
|
18
|
+
- pattern: hour
|
|
19
|
+
- pattern: hours
|
|
20
|
+
- pattern: hr
|
|
21
|
+
- pattern: hrs
|
|
22
|
+
- pattern: h
|
|
23
|
+
- pattern: min
|
|
24
|
+
- pattern: mins
|
|
25
|
+
- pattern: minute
|
|
26
|
+
- pattern: minutes
|
|
27
|
+
- pattern: sec
|
|
28
|
+
- pattern: secs
|
|
29
|
+
- pattern: second
|
|
30
|
+
- pattern: seconds
|
|
31
|
+
- pattern: ms
|
|
32
|
+
- pattern: kg
|
|
33
|
+
- pattern: g
|
|
34
|
+
- pattern: mg
|
|
35
|
+
- pattern: lb
|
|
36
|
+
- pattern: lbs
|
|
37
|
+
- pattern: oz
|
|
38
|
+
- pattern: km
|
|
39
|
+
- pattern: m
|
|
40
|
+
- pattern: cm
|
|
41
|
+
- pattern: mm
|
|
42
|
+
- pattern: mi
|
|
43
|
+
- pattern: ft
|
|
44
|
+
- pattern: in
|
|
45
|
+
- pattern: people
|
|
46
|
+
- pattern: persons
|
|
47
|
+
- pattern: users
|
|
48
|
+
- pattern: items
|
|
49
|
+
- pattern: pages
|
|
50
|
+
- pattern: words
|
|
51
|
+
- pattern: times
|
|
52
|
+
- pattern: x
|
|
53
|
+
- pattern: gb
|
|
54
|
+
- pattern: mb
|
|
55
|
+
- pattern: kb
|
|
56
|
+
- pattern: tb
|
|
57
|
+
- pattern: px
|
|
58
|
+
- pattern: pt
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# Words after "very" that are not a weak adjective propped up: "the very first", "the very same", "very few", "its very
|
|
2
|
+
# own". very-adjective does not count them.
|
|
3
|
+
id: very-exception
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: first
|
|
7
|
+
- pattern: last
|
|
8
|
+
- pattern: next
|
|
9
|
+
- pattern: same
|
|
10
|
+
- pattern: few
|
|
11
|
+
- pattern: many
|
|
12
|
+
- pattern: little
|
|
13
|
+
- pattern: much
|
|
14
|
+
- pattern: own
|
|
15
|
+
- pattern: best
|
|
16
|
+
- pattern: least
|
|
17
|
+
- pattern: most
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# Weasel words: a claim credited to an unnamed crowd, or to research no one names. Read by weasel-word.
|
|
2
|
+
# "studies show" followed by a citation is still reported: the sentence can name the study instead of leaning on it.
|
|
3
|
+
id: weasel-word
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: many experts say
|
|
7
|
+
- pattern: many experts believe
|
|
8
|
+
- pattern: experts agree
|
|
9
|
+
- pattern: experts say
|
|
10
|
+
- pattern: experts believe
|
|
11
|
+
- pattern: it is widely believed
|
|
12
|
+
- pattern: it is widely known
|
|
13
|
+
- pattern: it is widely accepted
|
|
14
|
+
- pattern: it is generally believed
|
|
15
|
+
- pattern: it is generally accepted
|
|
16
|
+
- pattern: it is commonly believed
|
|
17
|
+
- pattern: it is often said
|
|
18
|
+
- pattern: it is said that
|
|
19
|
+
- pattern: it has been said
|
|
20
|
+
- pattern: some people say
|
|
21
|
+
- pattern: some people believe
|
|
22
|
+
- pattern: many people believe
|
|
23
|
+
- pattern: many people say
|
|
24
|
+
- pattern: most people agree
|
|
25
|
+
- pattern: studies show
|
|
26
|
+
- pattern: studies have shown
|
|
27
|
+
- pattern: research shows
|
|
28
|
+
- pattern: research has shown
|
|
29
|
+
- pattern: critics say
|
|
30
|
+
- pattern: according to some
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# Prefixes that make a word from a dictionary word ("unshipped", "reupload", "multiline"). unknown-word strips one and
|
|
2
|
+
# looks the rest up, with its endings. Only prefixes that attach freely in current English are listed.
|
|
3
|
+
id: word-prefix
|
|
4
|
+
language: en
|
|
5
|
+
entries:
|
|
6
|
+
- pattern: un
|
|
7
|
+
- pattern: re
|
|
8
|
+
- pattern: non
|
|
9
|
+
- pattern: pre
|
|
10
|
+
- pattern: post
|
|
11
|
+
- pattern: over
|
|
12
|
+
- pattern: under
|
|
13
|
+
- pattern: sub
|
|
14
|
+
- pattern: super
|
|
15
|
+
- pattern: co
|
|
16
|
+
- pattern: multi
|
|
17
|
+
- pattern: inter
|
|
18
|
+
- pattern: de
|
|
19
|
+
- pattern: dis
|
|
20
|
+
- pattern: mis
|
|
21
|
+
- pattern: out
|
|
22
|
+
- pattern: auto
|
|
23
|
+
- pattern: anti
|
|
24
|
+
- pattern: semi
|
|
25
|
+
- pattern: micro
|
|
26
|
+
- pattern: mega
|
|
27
|
+
- pattern: cross
|
|
28
|
+
- pattern: self
|
|
29
|
+
- pattern: meta
|
|
30
|
+
- pattern: hyper
|
|
31
|
+
- pattern: mini
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Endings that make an inflected or derived form of a dictionary word: unknown-word strips the ending (pattern) and
|
|
2
|
+
# puts the replacement (instead_of, empty when nothing goes back) before looking the stem up. "carried" is "carry",
|
|
3
|
+
# "stopped" is "stop" (a doubled last consonant is undone too), "baking" is "bake". The dictionary itself is
|
|
4
|
+
# dictionary.txt; this list only saves listing every inflected form there.
|
|
5
|
+
id: word-suffix
|
|
6
|
+
language: en
|
|
7
|
+
entries:
|
|
8
|
+
- pattern: s
|
|
9
|
+
instead_of: ""
|
|
10
|
+
- pattern: es
|
|
11
|
+
instead_of: ""
|
|
12
|
+
- pattern: ies
|
|
13
|
+
instead_of: "y"
|
|
14
|
+
- pattern: ied
|
|
15
|
+
instead_of: "y"
|
|
16
|
+
- pattern: ed
|
|
17
|
+
instead_of: ""
|
|
18
|
+
- pattern: ed
|
|
19
|
+
instead_of: "e"
|
|
20
|
+
- pattern: ing
|
|
21
|
+
instead_of: ""
|
|
22
|
+
- pattern: ing
|
|
23
|
+
instead_of: "e"
|
|
24
|
+
- pattern: er
|
|
25
|
+
instead_of: ""
|
|
26
|
+
- pattern: er
|
|
27
|
+
instead_of: "e"
|
|
28
|
+
- pattern: ers
|
|
29
|
+
instead_of: ""
|
|
30
|
+
- pattern: ers
|
|
31
|
+
instead_of: "e"
|
|
32
|
+
- pattern: ier
|
|
33
|
+
instead_of: "y"
|
|
34
|
+
- pattern: iest
|
|
35
|
+
instead_of: "y"
|
|
36
|
+
- pattern: est
|
|
37
|
+
instead_of: ""
|
|
38
|
+
- pattern: est
|
|
39
|
+
instead_of: "e"
|
|
40
|
+
- pattern: ly
|
|
41
|
+
instead_of: ""
|
|
42
|
+
- pattern: ily
|
|
43
|
+
instead_of: "y"
|
|
44
|
+
- pattern: ness
|
|
45
|
+
instead_of: ""
|
|
46
|
+
- pattern: iness
|
|
47
|
+
instead_of: "y"
|
|
48
|
+
- pattern: ment
|
|
49
|
+
instead_of: ""
|
|
50
|
+
- pattern: ments
|
|
51
|
+
instead_of: ""
|
|
52
|
+
- pattern: able
|
|
53
|
+
instead_of: ""
|
|
54
|
+
- pattern: able
|
|
55
|
+
instead_of: "e"
|
|
56
|
+
- pattern: ability
|
|
57
|
+
instead_of: "able"
|
|
58
|
+
- pattern: ity
|
|
59
|
+
instead_of: ""
|
|
60
|
+
- pattern: ful
|
|
61
|
+
instead_of: ""
|
|
62
|
+
- pattern: less
|
|
63
|
+
instead_of: ""
|
|
64
|
+
- pattern: ize
|
|
65
|
+
instead_of: ""
|
|
66
|
+
- pattern: ization
|
|
67
|
+
instead_of: "ize"
|
|
68
|
+
- pattern: ise
|
|
69
|
+
instead_of: ""
|
|
70
|
+
- pattern: isation
|
|
71
|
+
instead_of: "ise"
|
|
72
|
+
- pattern: ation
|
|
73
|
+
instead_of: "e"
|
|
74
|
+
- pattern: ation
|
|
75
|
+
instead_of: ""
|
|
76
|
+
- pattern: ably
|
|
77
|
+
instead_of: "able"
|
|
78
|
+
- pattern: ibly
|
|
79
|
+
instead_of: "ible"
|
|
80
|
+
- pattern: men
|
|
81
|
+
instead_of: "man"
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Wordy phrases: several words where one or two say the same. Read by wordy-phrase; rewrite is the short form.
|
|
2
|
+
# Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
|
|
3
|
+
# are left out: contracts and statutes use them on purpose. Matched as whole words, case aside.
|
|
4
|
+
id: wordy-phrase
|
|
5
|
+
language: en
|
|
6
|
+
entries:
|
|
7
|
+
- pattern: in order to
|
|
8
|
+
rewrite: to
|
|
9
|
+
- pattern: in order for
|
|
10
|
+
rewrite: for
|
|
11
|
+
- pattern: due to the fact that
|
|
12
|
+
rewrite: because
|
|
13
|
+
- pattern: owing to the fact that
|
|
14
|
+
rewrite: because
|
|
15
|
+
- pattern: in view of the fact that
|
|
16
|
+
rewrite: because
|
|
17
|
+
- pattern: in spite of the fact that
|
|
18
|
+
rewrite: although
|
|
19
|
+
- pattern: despite the fact that
|
|
20
|
+
rewrite: although
|
|
21
|
+
- pattern: at this point in time
|
|
22
|
+
rewrite: now
|
|
23
|
+
- pattern: at the present time
|
|
24
|
+
rewrite: now
|
|
25
|
+
- pattern: at this moment in time
|
|
26
|
+
rewrite: now
|
|
27
|
+
- pattern: in the event that
|
|
28
|
+
rewrite: if
|
|
29
|
+
- pattern: in the near future
|
|
30
|
+
rewrite: soon
|
|
31
|
+
- pattern: for the purpose of
|
|
32
|
+
rewrite: to, or for
|
|
33
|
+
- pattern: on a daily basis
|
|
34
|
+
rewrite: daily
|
|
35
|
+
- pattern: on a weekly basis
|
|
36
|
+
rewrite: weekly
|
|
37
|
+
- pattern: on a monthly basis
|
|
38
|
+
rewrite: monthly
|
|
39
|
+
- pattern: on a regular basis
|
|
40
|
+
rewrite: regularly
|
|
41
|
+
- pattern: a large number of
|
|
42
|
+
rewrite: many
|
|
43
|
+
- pattern: has the ability to
|
|
44
|
+
rewrite: can
|
|
45
|
+
- pattern: have the ability to
|
|
46
|
+
rewrite: can
|
|
47
|
+
- pattern: is in a position to
|
|
48
|
+
rewrite: can
|
|
49
|
+
- pattern: until such time as
|
|
50
|
+
rewrite: until
|
package/package.json
CHANGED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { fileURLToPath } from "node:url";
|
|
3
|
+
import { dirname, join } from "node:path";
|
|
4
|
+
|
|
5
|
+
// The English word list (lexicons/dictionary.txt, built by scripts/en-dictionary.ts). It is large, so it is read only
|
|
6
|
+
// when a rule asks for it, and once.
|
|
7
|
+
|
|
8
|
+
const FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons", "dictionary.txt");
|
|
9
|
+
|
|
10
|
+
const loaded: { words: ReadonlySet<string> | undefined } = { words: undefined };
|
|
11
|
+
|
|
12
|
+
const wordsIn = (text: string): ReadonlySet<string> => new Set(text.split("\n").filter((line) => line !== "" && !line.startsWith("#")));
|
|
13
|
+
|
|
14
|
+
export const dictionary = (): ReadonlySet<string> => {
|
|
15
|
+
if (loaded.words !== undefined) return loaded.words;
|
|
16
|
+
try {
|
|
17
|
+
loaded.words = wordsIn(readFileSync(FILE, "utf8"));
|
|
18
|
+
} catch (error) {
|
|
19
|
+
throw new Error(`${FILE}: cannot read the English word list`, { cause: error });
|
|
20
|
+
}
|
|
21
|
+
return loaded.words;
|
|
22
|
+
};
|
package/src/index.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { sentenceSpans } from "./sentence-split.ts";
|
|
|
5
5
|
import { splitAtQuotedStops } from "./quoted-stop.ts";
|
|
6
6
|
import { reattachClosingQuotes } from "./closing-quote.ts";
|
|
7
7
|
import { structure } from "./structure.ts";
|
|
8
|
+
import { dictionary } from "./dictionary.ts";
|
|
8
9
|
import { isReady, prepare, tokenize } from "./pos.ts";
|
|
9
10
|
import { isJapaneseRun } from "./japanese-run.ts";
|
|
10
11
|
import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
|
|
@@ -62,6 +63,7 @@ export const adapter: LanguageAdapter = {
|
|
|
62
63
|
},
|
|
63
64
|
lexicons: LEXICONS,
|
|
64
65
|
structure,
|
|
66
|
+
dictionary,
|
|
65
67
|
segment: (text: string): Segmentation => {
|
|
66
68
|
const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
|
|
67
69
|
const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
|