@chaffjs/lang-en 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/dist/definitions.d.ts.map +1 -1
  2. package/dist/definitions.js +4 -2
  3. package/dist/definitions.js.map +1 -1
  4. package/dist/dictionary.d.ts +2 -0
  5. package/dist/dictionary.d.ts.map +1 -0
  6. package/dist/dictionary.js +20 -0
  7. package/dist/dictionary.js.map +1 -0
  8. package/dist/index.d.ts.map +1 -1
  9. package/dist/index.js +2 -0
  10. package/dist/index.js.map +1 -1
  11. package/dist/lexicons.d.ts.map +1 -1
  12. package/dist/lexicons.js +4 -0
  13. package/dist/lexicons.js.map +1 -1
  14. package/lexicons/absolute-word.yaml +14 -0
  15. package/lexicons/abstract-heading.yaml +7 -0
  16. package/lexicons/age-marker.yaml +10 -0
  17. package/lexicons/ai-tell.yaml +12 -0
  18. package/lexicons/approximate-marker.yaml +34 -0
  19. package/lexicons/article-sound.yaml +83 -0
  20. package/lexicons/birth-origin.yaml +5 -0
  21. package/lexicons/bookend-heading.yaml +30 -0
  22. package/lexicons/chat-citation-marker.yaml +11 -0
  23. package/lexicons/citation-author-joiner.yaml +11 -0
  24. package/lexicons/citation-not-author.yaml +40 -0
  25. package/lexicons/count-counter.yaml +50 -0
  26. package/lexicons/cushion-phrase.yaml +6 -0
  27. package/lexicons/date-change-word.yaml +27 -0
  28. package/lexicons/dictionary.txt +87150 -0
  29. package/lexicons/double-negative.yaml +25 -0
  30. package/lexicons/duration-day.yaml +6 -0
  31. package/lexicons/duration-month.yaml +6 -0
  32. package/lexicons/duration-week.yaml +6 -0
  33. package/lexicons/duration-year.yaml +6 -0
  34. package/lexicons/elapsed-link.yaml +11 -0
  35. package/lexicons/elapsed-origin.yaml +8 -0
  36. package/lexicons/elapsed-unit.yaml +6 -0
  37. package/lexicons/enumeration-frame.yaml +37 -0
  38. package/lexicons/enumeration-joiner.yaml +9 -0
  39. package/lexicons/enumeration-negation.yaml +13 -0
  40. package/lexicons/exception-marker.yaml +10 -0
  41. package/lexicons/extra-word.yaml +185 -0
  42. package/lexicons/fact-attribute.yaml +6 -0
  43. package/lexicons/fact-label-drop.yaml +13 -0
  44. package/lexicons/fact-label-vague.yaml +16 -0
  45. package/lexicons/fact-separator.yaml +11 -0
  46. package/lexicons/fact-value-end.yaml +8 -0
  47. package/lexicons/figure-in-text.yaml +9 -0
  48. package/lexicons/figure-noun.yaml +31 -0
  49. package/lexicons/heading-form.yaml +63 -0
  50. package/lexicons/heading-number-label.yaml +29 -0
  51. package/lexicons/homophone-slip.yaml +37 -0
  52. package/lexicons/known-misspelling.yaml +177 -0
  53. package/lexicons/label-affix.yaml +7 -0
  54. package/lexicons/light-verb.yaml +9 -0
  55. package/lexicons/month-name.yaml +29 -0
  56. package/lexicons/nominalization-phrase.yaml +27 -0
  57. package/lexicons/orthographic-variant-distinct.yaml +24 -0
  58. package/lexicons/paper-section-heading.yaml +18 -0
  59. package/lexicons/person-form.yaml +9 -0
  60. package/lexicons/person-not-after.yaml +54 -0
  61. package/lexicons/person-word.yaml +38 -0
  62. package/lexicons/place-region.yaml +247 -0
  63. package/lexicons/placeholder-word.yaml +10 -0
  64. package/lexicons/pro-con-heading.yaml +15 -0
  65. package/lexicons/range-frame.yaml +9 -0
  66. package/lexicons/redundant-expression.yaml +83 -0
  67. package/lexicons/reference-aside.yaml +21 -0
  68. package/lexicons/reference-list-heading.yaml +11 -0
  69. package/lexicons/reference-topic-link.yaml +5 -0
  70. package/lexicons/reference-topic-vague.yaml +32 -0
  71. package/lexicons/relative-after.yaml +5 -0
  72. package/lexicons/relative-before.yaml +5 -0
  73. package/lexicons/relative-link.yaml +7 -0
  74. package/lexicons/requirement-either.yaml +5 -0
  75. package/lexicons/requirement-loophole.yaml +17 -0
  76. package/lexicons/requirement-marker.yaml +9 -0
  77. package/lexicons/requirement-modal-must.yaml +7 -0
  78. package/lexicons/requirement-open-end.yaml +9 -0
  79. package/lexicons/restating-opener.yaml +22 -0
  80. package/lexicons/so-opener.yaml +6 -0
  81. package/lexicons/spelling-ize.yaml +273 -0
  82. package/lexicons/spelling-variant.yaml +260 -0
  83. package/lexicons/summary-heading.yaml +13 -0
  84. package/lexicons/unit-data.yaml +28 -0
  85. package/lexicons/unit-length.yaml +44 -0
  86. package/lexicons/unit-mass.yaml +30 -0
  87. package/lexicons/unit-time.yaml +32 -0
  88. package/lexicons/unit-volume.yaml +26 -0
  89. package/lexicons/unnumbered-heading.yaml +27 -0
  90. package/lexicons/vague-clause-pointer.yaml +10 -0
  91. package/lexicons/vague-figure-pointer.yaml +14 -0
  92. package/lexicons/vague-link-text.yaml +20 -0
  93. package/lexicons/vague-word.yaml +31 -0
  94. package/lexicons/value-unit.yaml +58 -0
  95. package/lexicons/very-exception.yaml +17 -0
  96. package/lexicons/very-intensifier.yaml +6 -0
  97. package/lexicons/weasel-word.yaml +30 -0
  98. package/lexicons/word-prefix.yaml +31 -0
  99. package/lexicons/word-suffix.yaml +81 -0
  100. package/lexicons/wordy-phrase.yaml +50 -0
  101. package/package.json +1 -1
  102. package/src/definitions.ts +4 -2
  103. package/src/dictionary.ts +22 -0
  104. package/src/index.ts +2 -0
  105. package/src/lexicons.ts +4 -0
@@ -0,0 +1,81 @@
1
+ # Endings that make an inflected or derived form of a dictionary word: unknown-word strips the ending (pattern) and
2
+ # puts the replacement (instead_of, empty when nothing goes back) before looking the stem up. "carried" is "carry",
3
+ # "stopped" is "stop" (a doubled last consonant is undone too), "baking" is "bake". The dictionary itself is
4
+ # dictionary.txt; this list only saves listing every inflected form there.
5
+ id: word-suffix
6
+ language: en
7
+ entries:
8
+ - pattern: s
9
+ instead_of: ""
10
+ - pattern: es
11
+ instead_of: ""
12
+ - pattern: ies
13
+ instead_of: "y"
14
+ - pattern: ied
15
+ instead_of: "y"
16
+ - pattern: ed
17
+ instead_of: ""
18
+ - pattern: ed
19
+ instead_of: "e"
20
+ - pattern: ing
21
+ instead_of: ""
22
+ - pattern: ing
23
+ instead_of: "e"
24
+ - pattern: er
25
+ instead_of: ""
26
+ - pattern: er
27
+ instead_of: "e"
28
+ - pattern: ers
29
+ instead_of: ""
30
+ - pattern: ers
31
+ instead_of: "e"
32
+ - pattern: ier
33
+ instead_of: "y"
34
+ - pattern: iest
35
+ instead_of: "y"
36
+ - pattern: est
37
+ instead_of: ""
38
+ - pattern: est
39
+ instead_of: "e"
40
+ - pattern: ly
41
+ instead_of: ""
42
+ - pattern: ily
43
+ instead_of: "y"
44
+ - pattern: ness
45
+ instead_of: ""
46
+ - pattern: iness
47
+ instead_of: "y"
48
+ - pattern: ment
49
+ instead_of: ""
50
+ - pattern: ments
51
+ instead_of: ""
52
+ - pattern: able
53
+ instead_of: ""
54
+ - pattern: able
55
+ instead_of: "e"
56
+ - pattern: ability
57
+ instead_of: "able"
58
+ - pattern: ity
59
+ instead_of: ""
60
+ - pattern: ful
61
+ instead_of: ""
62
+ - pattern: less
63
+ instead_of: ""
64
+ - pattern: ize
65
+ instead_of: ""
66
+ - pattern: ization
67
+ instead_of: "ize"
68
+ - pattern: ise
69
+ instead_of: ""
70
+ - pattern: isation
71
+ instead_of: "ise"
72
+ - pattern: ation
73
+ instead_of: "e"
74
+ - pattern: ation
75
+ instead_of: ""
76
+ - pattern: ably
77
+ instead_of: "able"
78
+ - pattern: ibly
79
+ instead_of: "ible"
80
+ - pattern: men
81
+ instead_of: "man"
@@ -0,0 +1,50 @@
1
+ # Wordy phrases: several words where one or two say the same. Read by wordy-phrase; rewrite is the short form.
2
+ # Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
3
+ # are left out: contracts and statutes use them on purpose. Matched as whole words, case aside.
4
+ id: wordy-phrase
5
+ language: en
6
+ entries:
7
+ - pattern: in order to
8
+ rewrite: to
9
+ - pattern: in order for
10
+ rewrite: for
11
+ - pattern: due to the fact that
12
+ rewrite: because
13
+ - pattern: owing to the fact that
14
+ rewrite: because
15
+ - pattern: in view of the fact that
16
+ rewrite: because
17
+ - pattern: in spite of the fact that
18
+ rewrite: although
19
+ - pattern: despite the fact that
20
+ rewrite: although
21
+ - pattern: at this point in time
22
+ rewrite: now
23
+ - pattern: at the present time
24
+ rewrite: now
25
+ - pattern: at this moment in time
26
+ rewrite: now
27
+ - pattern: in the event that
28
+ rewrite: if
29
+ - pattern: in the near future
30
+ rewrite: soon
31
+ - pattern: for the purpose of
32
+ rewrite: to, or for
33
+ - pattern: on a daily basis
34
+ rewrite: daily
35
+ - pattern: on a weekly basis
36
+ rewrite: weekly
37
+ - pattern: on a monthly basis
38
+ rewrite: monthly
39
+ - pattern: on a regular basis
40
+ rewrite: regularly
41
+ - pattern: a large number of
42
+ rewrite: many
43
+ - pattern: has the ability to
44
+ rewrite: can
45
+ - pattern: have the ability to
46
+ rewrite: can
47
+ - pattern: is in a position to
48
+ rewrite: can
49
+ - pattern: until such time as
50
+ rewrite: until
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chaffjs/lang-en",
3
- "version": "0.16.0",
3
+ "version": "0.18.0",
4
4
  "description": "English language adapter for chaff",
5
5
  "license": "MIT",
6
6
  "author": "isamu",
@@ -37,7 +37,9 @@ export const definitions = (text: string): Mention[] =>
37
37
  const term = match.groups?.["term"];
38
38
  if (term === undefined) return [];
39
39
  const end = match.index + match[0].length;
40
- const namesAParty = match[0].startsWith("(") && APPLIES.test(text);
41
- return [{ start: match.index, end, attrs: { term, ...(isPointer(text, end) || namesAParty ? { scope: "local" } : {}) } }];
40
+ const inline = match[0].startsWith("(");
41
+ const namesAParty = inline && APPLIES.test(text);
42
+ const scope = isPointer(text, end) || namesAParty ? { scope: "local" } : {};
43
+ return [{ start: match.index, end, attrs: { term, ...scope, ...(inline ? { placement: "inline" } : {}) } }];
42
44
  }),
43
45
  );
@@ -0,0 +1,22 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { fileURLToPath } from "node:url";
3
+ import { dirname, join } from "node:path";
4
+
5
+ // The English word list (lexicons/dictionary.txt, built by scripts/en-dictionary.ts). It is large, so it is read only
6
+ // when a rule asks for it, and once.
7
+
8
+ const FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons", "dictionary.txt");
9
+
10
+ const loaded: { words: ReadonlySet<string> | undefined } = { words: undefined };
11
+
12
+ const wordsIn = (text: string): ReadonlySet<string> => new Set(text.split("\n").filter((line) => line !== "" && !line.startsWith("#")));
13
+
14
+ export const dictionary = (): ReadonlySet<string> => {
15
+ if (loaded.words !== undefined) return loaded.words;
16
+ try {
17
+ loaded.words = wordsIn(readFileSync(FILE, "utf8"));
18
+ } catch (error) {
19
+ throw new Error(`${FILE}: cannot read the English word list`, { cause: error });
20
+ }
21
+ return loaded.words;
22
+ };
package/src/index.ts CHANGED
@@ -5,6 +5,7 @@ import { sentenceSpans } from "./sentence-split.ts";
5
5
  import { splitAtQuotedStops } from "./quoted-stop.ts";
6
6
  import { reattachClosingQuotes } from "./closing-quote.ts";
7
7
  import { structure } from "./structure.ts";
8
+ import { dictionary } from "./dictionary.ts";
8
9
  import { isReady, prepare, tokenize } from "./pos.ts";
9
10
  import { isJapaneseRun } from "./japanese-run.ts";
10
11
  import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
@@ -62,6 +63,7 @@ export const adapter: LanguageAdapter = {
62
63
  },
63
64
  lexicons: LEXICONS,
64
65
  structure,
66
+ dictionary,
65
67
  segment: (text: string): Segmentation => {
66
68
  const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
67
69
  const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));
package/src/lexicons.ts CHANGED
@@ -14,11 +14,15 @@ const toEntry = (raw: unknown): LexiconEntry | undefined => {
14
14
  if (!isRecord(raw) || typeof raw["pattern"] !== "string") return undefined;
15
15
  const weight = raw["weight"];
16
16
  const instead = raw["instead_of"];
17
+ const rewrite = raw["rewrite"];
18
+ const group = raw["group"];
17
19
  return {
18
20
  pattern: raw["pattern"],
19
21
  weight: typeof weight === "number" ? weight : undefined,
20
22
  instead_of: typeof instead === "string" ? instead : undefined,
21
23
  position: POSITIONS.find((position) => position === raw["position"]),
24
+ rewrite: typeof rewrite === "string" ? rewrite : undefined,
25
+ ...(typeof group === "string" ? { group } : {}),
22
26
  };
23
27
  };
24
28