@chaffjs/lang-en 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/dist/dictionary.d.ts +2 -0
  2. package/dist/dictionary.d.ts.map +1 -0
  3. package/dist/dictionary.js +20 -0
  4. package/dist/dictionary.js.map +1 -0
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +2 -0
  7. package/dist/index.js.map +1 -1
  8. package/lexicons/absolute-word.yaml +14 -0
  9. package/lexicons/abstract-heading.yaml +7 -0
  10. package/lexicons/age-marker.yaml +10 -0
  11. package/lexicons/approximate-marker.yaml +34 -0
  12. package/lexicons/article-sound.yaml +83 -0
  13. package/lexicons/birth-origin.yaml +5 -0
  14. package/lexicons/bookend-heading.yaml +30 -0
  15. package/lexicons/chat-citation-marker.yaml +11 -0
  16. package/lexicons/citation-author-joiner.yaml +11 -0
  17. package/lexicons/citation-not-author.yaml +40 -0
  18. package/lexicons/dictionary.txt +87150 -0
  19. package/lexicons/duration-day.yaml +6 -0
  20. package/lexicons/duration-month.yaml +6 -0
  21. package/lexicons/duration-week.yaml +6 -0
  22. package/lexicons/duration-year.yaml +6 -0
  23. package/lexicons/elapsed-link.yaml +11 -0
  24. package/lexicons/elapsed-origin.yaml +8 -0
  25. package/lexicons/elapsed-unit.yaml +6 -0
  26. package/lexicons/exception-marker.yaml +10 -0
  27. package/lexicons/extra-word.yaml +185 -0
  28. package/lexicons/fact-attribute.yaml +6 -0
  29. package/lexicons/fact-label-drop.yaml +13 -0
  30. package/lexicons/fact-label-vague.yaml +16 -0
  31. package/lexicons/fact-separator.yaml +11 -0
  32. package/lexicons/fact-value-end.yaml +8 -0
  33. package/lexicons/figure-in-text.yaml +9 -0
  34. package/lexicons/figure-noun.yaml +31 -0
  35. package/lexicons/heading-form.yaml +63 -0
  36. package/lexicons/heading-number-label.yaml +29 -0
  37. package/lexicons/homophone-slip.yaml +37 -0
  38. package/lexicons/known-misspelling.yaml +177 -0
  39. package/lexicons/label-affix.yaml +7 -0
  40. package/lexicons/light-verb.yaml +9 -0
  41. package/lexicons/month-name.yaml +29 -0
  42. package/lexicons/orthographic-variant-distinct.yaml +24 -0
  43. package/lexicons/paper-section-heading.yaml +18 -0
  44. package/lexicons/person-form.yaml +9 -0
  45. package/lexicons/person-not-after.yaml +54 -0
  46. package/lexicons/person-word.yaml +38 -0
  47. package/lexicons/placeholder-word.yaml +10 -0
  48. package/lexicons/pro-con-heading.yaml +15 -0
  49. package/lexicons/redundant-expression.yaml +83 -0
  50. package/lexicons/reference-aside.yaml +21 -0
  51. package/lexicons/reference-list-heading.yaml +11 -0
  52. package/lexicons/reference-topic-link.yaml +5 -0
  53. package/lexicons/reference-topic-vague.yaml +32 -0
  54. package/lexicons/relative-after.yaml +5 -0
  55. package/lexicons/relative-before.yaml +5 -0
  56. package/lexicons/relative-link.yaml +7 -0
  57. package/lexicons/restating-opener.yaml +22 -0
  58. package/lexicons/so-opener.yaml +6 -0
  59. package/lexicons/summary-heading.yaml +13 -0
  60. package/lexicons/unit-data.yaml +28 -0
  61. package/lexicons/unit-length.yaml +44 -0
  62. package/lexicons/unit-mass.yaml +30 -0
  63. package/lexicons/unit-time.yaml +32 -0
  64. package/lexicons/unit-volume.yaml +26 -0
  65. package/lexicons/unnumbered-heading.yaml +27 -0
  66. package/lexicons/vague-link-text.yaml +20 -0
  67. package/lexicons/vague-word.yaml +31 -0
  68. package/lexicons/value-unit.yaml +58 -0
  69. package/lexicons/very-exception.yaml +17 -0
  70. package/lexicons/very-intensifier.yaml +6 -0
  71. package/lexicons/weasel-word.yaml +30 -0
  72. package/lexicons/word-prefix.yaml +31 -0
  73. package/lexicons/word-suffix.yaml +81 -0
  74. package/lexicons/wordy-phrase.yaml +50 -0
  75. package/package.json +1 -1
  76. package/src/dictionary.ts +22 -0
  77. package/src/index.ts +2 -0
@@ -0,0 +1,58 @@
1
+ # Units and counted nouns that follow a number. number-without-unit reads a word after a number as a unit only if it is
2
+ # here or in measure-unit, percent-unit or currency-code; a number followed by any other word ("3 fixed") is not a value.
3
+ id: value-unit
4
+ language: en
5
+ entries:
6
+ - pattern: dollars
7
+ - pattern: yen
8
+ - pattern: euros
9
+ - pattern: usd
10
+ - pattern: day
11
+ - pattern: days
12
+ - pattern: week
13
+ - pattern: weeks
14
+ - pattern: month
15
+ - pattern: months
16
+ - pattern: year
17
+ - pattern: years
18
+ - pattern: hour
19
+ - pattern: hours
20
+ - pattern: hr
21
+ - pattern: hrs
22
+ - pattern: h
23
+ - pattern: min
24
+ - pattern: mins
25
+ - pattern: minute
26
+ - pattern: minutes
27
+ - pattern: sec
28
+ - pattern: secs
29
+ - pattern: second
30
+ - pattern: seconds
31
+ - pattern: ms
32
+ - pattern: kg
33
+ - pattern: g
34
+ - pattern: mg
35
+ - pattern: lb
36
+ - pattern: lbs
37
+ - pattern: oz
38
+ - pattern: km
39
+ - pattern: m
40
+ - pattern: cm
41
+ - pattern: mm
42
+ - pattern: mi
43
+ - pattern: ft
44
+ - pattern: in
45
+ - pattern: people
46
+ - pattern: persons
47
+ - pattern: users
48
+ - pattern: items
49
+ - pattern: pages
50
+ - pattern: words
51
+ - pattern: times
52
+ - pattern: x
53
+ - pattern: gb
54
+ - pattern: mb
55
+ - pattern: kb
56
+ - pattern: tb
57
+ - pattern: px
58
+ - pattern: pt
@@ -0,0 +1,17 @@
1
+ # Words after "very" that are not a weak adjective propped up: "the very first", "the very same", "very few", "its very
2
+ # own". very-adjective does not count them.
3
+ id: very-exception
4
+ language: en
5
+ entries:
6
+ - pattern: first
7
+ - pattern: last
8
+ - pattern: next
9
+ - pattern: same
10
+ - pattern: few
11
+ - pattern: many
12
+ - pattern: little
13
+ - pattern: much
14
+ - pattern: own
15
+ - pattern: best
16
+ - pattern: least
17
+ - pattern: most
@@ -0,0 +1,6 @@
1
+ # Intensifiers counted by very-adjective when an adjective follows ("very important"). Adverbs ("very well") and the
2
+ # words in very-exception ("the very first") are not counted.
3
+ id: very-intensifier
4
+ language: en
5
+ entries:
6
+ - pattern: very
@@ -0,0 +1,30 @@
1
+ # Weasel words: a claim credited to an unnamed crowd, or to research no one names. Read by weasel-word.
2
+ # "studies show" followed by a citation is still reported: the sentence can name the study instead of leaning on it.
3
+ id: weasel-word
4
+ language: en
5
+ entries:
6
+ - pattern: many experts say
7
+ - pattern: many experts believe
8
+ - pattern: experts agree
9
+ - pattern: experts say
10
+ - pattern: experts believe
11
+ - pattern: it is widely believed
12
+ - pattern: it is widely known
13
+ - pattern: it is widely accepted
14
+ - pattern: it is generally believed
15
+ - pattern: it is generally accepted
16
+ - pattern: it is commonly believed
17
+ - pattern: it is often said
18
+ - pattern: it is said that
19
+ - pattern: it has been said
20
+ - pattern: some people say
21
+ - pattern: some people believe
22
+ - pattern: many people believe
23
+ - pattern: many people say
24
+ - pattern: most people agree
25
+ - pattern: studies show
26
+ - pattern: studies have shown
27
+ - pattern: research shows
28
+ - pattern: research has shown
29
+ - pattern: critics say
30
+ - pattern: according to some
@@ -0,0 +1,31 @@
1
+ # Prefixes that make a word from a dictionary word ("unshipped", "reupload", "multiline"). unknown-word strips one and
2
+ # looks the rest up, with its endings. Only prefixes that attach freely in current English are listed.
3
+ id: word-prefix
4
+ language: en
5
+ entries:
6
+ - pattern: un
7
+ - pattern: re
8
+ - pattern: non
9
+ - pattern: pre
10
+ - pattern: post
11
+ - pattern: over
12
+ - pattern: under
13
+ - pattern: sub
14
+ - pattern: super
15
+ - pattern: co
16
+ - pattern: multi
17
+ - pattern: inter
18
+ - pattern: de
19
+ - pattern: dis
20
+ - pattern: mis
21
+ - pattern: out
22
+ - pattern: auto
23
+ - pattern: anti
24
+ - pattern: semi
25
+ - pattern: micro
26
+ - pattern: mega
27
+ - pattern: cross
28
+ - pattern: self
29
+ - pattern: meta
30
+ - pattern: hyper
31
+ - pattern: mini
@@ -0,0 +1,81 @@
1
+ # Endings that make an inflected or derived form of a dictionary word: unknown-word strips the ending (pattern) and
2
+ # puts the replacement (instead_of, empty when nothing goes back) before looking the stem up. "carried" is "carry",
3
+ # "stopped" is "stop" (a doubled last consonant is undone too), "baking" is "bake". The dictionary itself is
4
+ # dictionary.txt; this list only saves listing every inflected form there.
5
+ id: word-suffix
6
+ language: en
7
+ entries:
8
+ - pattern: s
9
+ instead_of: ""
10
+ - pattern: es
11
+ instead_of: ""
12
+ - pattern: ies
13
+ instead_of: "y"
14
+ - pattern: ied
15
+ instead_of: "y"
16
+ - pattern: ed
17
+ instead_of: ""
18
+ - pattern: ed
19
+ instead_of: "e"
20
+ - pattern: ing
21
+ instead_of: ""
22
+ - pattern: ing
23
+ instead_of: "e"
24
+ - pattern: er
25
+ instead_of: ""
26
+ - pattern: er
27
+ instead_of: "e"
28
+ - pattern: ers
29
+ instead_of: ""
30
+ - pattern: ers
31
+ instead_of: "e"
32
+ - pattern: ier
33
+ instead_of: "y"
34
+ - pattern: iest
35
+ instead_of: "y"
36
+ - pattern: est
37
+ instead_of: ""
38
+ - pattern: est
39
+ instead_of: "e"
40
+ - pattern: ly
41
+ instead_of: ""
42
+ - pattern: ily
43
+ instead_of: "y"
44
+ - pattern: ness
45
+ instead_of: ""
46
+ - pattern: iness
47
+ instead_of: "y"
48
+ - pattern: ment
49
+ instead_of: ""
50
+ - pattern: ments
51
+ instead_of: ""
52
+ - pattern: able
53
+ instead_of: ""
54
+ - pattern: able
55
+ instead_of: "e"
56
+ - pattern: ability
57
+ instead_of: "able"
58
+ - pattern: ity
59
+ instead_of: ""
60
+ - pattern: ful
61
+ instead_of: ""
62
+ - pattern: less
63
+ instead_of: ""
64
+ - pattern: ize
65
+ instead_of: ""
66
+ - pattern: ization
67
+ instead_of: "ize"
68
+ - pattern: ise
69
+ instead_of: ""
70
+ - pattern: isation
71
+ instead_of: "ise"
72
+ - pattern: ation
73
+ instead_of: "e"
74
+ - pattern: ation
75
+ instead_of: ""
76
+ - pattern: ably
77
+ instead_of: "able"
78
+ - pattern: ibly
79
+ instead_of: "ible"
80
+ - pattern: men
81
+ instead_of: "man"
@@ -0,0 +1,50 @@
1
+ # Wordy phrases: several words where one or two say the same. Read by wordy-phrase; rewrite is the short form.
2
+ # Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
3
+ # are left out: contracts and statutes use them on purpose. Matched as whole words, case aside.
4
+ id: wordy-phrase
5
+ language: en
6
+ entries:
7
+ - pattern: in order to
8
+ rewrite: to
9
+ - pattern: in order for
10
+ rewrite: for
11
+ - pattern: due to the fact that
12
+ rewrite: because
13
+ - pattern: owing to the fact that
14
+ rewrite: because
15
+ - pattern: in view of the fact that
16
+ rewrite: because
17
+ - pattern: in spite of the fact that
18
+ rewrite: although
19
+ - pattern: despite the fact that
20
+ rewrite: although
21
+ - pattern: at this point in time
22
+ rewrite: now
23
+ - pattern: at the present time
24
+ rewrite: now
25
+ - pattern: at this moment in time
26
+ rewrite: now
27
+ - pattern: in the event that
28
+ rewrite: if
29
+ - pattern: in the near future
30
+ rewrite: soon
31
+ - pattern: for the purpose of
32
+ rewrite: to, or for
33
+ - pattern: on a daily basis
34
+ rewrite: daily
35
+ - pattern: on a weekly basis
36
+ rewrite: weekly
37
+ - pattern: on a monthly basis
38
+ rewrite: monthly
39
+ - pattern: on a regular basis
40
+ rewrite: regularly
41
+ - pattern: a large number of
42
+ rewrite: many
43
+ - pattern: has the ability to
44
+ rewrite: can
45
+ - pattern: have the ability to
46
+ rewrite: can
47
+ - pattern: is in a position to
48
+ rewrite: can
49
+ - pattern: until such time as
50
+ rewrite: until
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chaffjs/lang-en",
3
- "version": "0.17.0",
3
+ "version": "0.18.0",
4
4
  "description": "English language adapter for chaff",
5
5
  "license": "MIT",
6
6
  "author": "isamu",
@@ -0,0 +1,22 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { fileURLToPath } from "node:url";
3
+ import { dirname, join } from "node:path";
4
+
5
+ // The English word list (lexicons/dictionary.txt, built by scripts/en-dictionary.ts). It is large, so it is read only
6
+ // when a rule asks for it, and once.
7
+
8
+ const FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons", "dictionary.txt");
9
+
10
+ const loaded: { words: ReadonlySet<string> | undefined } = { words: undefined };
11
+
12
+ const wordsIn = (text: string): ReadonlySet<string> => new Set(text.split("\n").filter((line) => line !== "" && !line.startsWith("#")));
13
+
14
+ export const dictionary = (): ReadonlySet<string> => {
15
+ if (loaded.words !== undefined) return loaded.words;
16
+ try {
17
+ loaded.words = wordsIn(readFileSync(FILE, "utf8"));
18
+ } catch (error) {
19
+ throw new Error(`${FILE}: cannot read the English word list`, { cause: error });
20
+ }
21
+ return loaded.words;
22
+ };
package/src/index.ts CHANGED
@@ -5,6 +5,7 @@ import { sentenceSpans } from "./sentence-split.ts";
5
5
  import { splitAtQuotedStops } from "./quoted-stop.ts";
6
6
  import { reattachClosingQuotes } from "./closing-quote.ts";
7
7
  import { structure } from "./structure.ts";
8
+ import { dictionary } from "./dictionary.ts";
8
9
  import { isReady, prepare, tokenize } from "./pos.ts";
9
10
  import { isJapaneseRun } from "./japanese-run.ts";
10
11
  import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
@@ -62,6 +63,7 @@ export const adapter: LanguageAdapter = {
62
63
  },
63
64
  lexicons: LEXICONS,
64
65
  structure,
66
+ dictionary,
65
67
  segment: (text: string): Segmentation => {
66
68
  const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
67
69
  const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));