@chaffjs/lang-en 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/dist/dictionary.d.ts +2 -0
  2. package/dist/dictionary.d.ts.map +1 -0
  3. package/dist/dictionary.js +20 -0
  4. package/dist/dictionary.js.map +1 -0
  5. package/dist/index.d.ts.map +1 -1
  6. package/dist/index.js +2 -0
  7. package/dist/index.js.map +1 -1
  8. package/lexicons/absolute-word.yaml +14 -0
  9. package/lexicons/abstract-heading.yaml +7 -0
  10. package/lexicons/age-marker.yaml +10 -0
  11. package/lexicons/approximate-marker.yaml +34 -0
  12. package/lexicons/article-sound.yaml +83 -0
  13. package/lexicons/attachment-file-extension.yaml +22 -0
  14. package/lexicons/attachment-line.yaml +14 -0
  15. package/lexicons/attachment-mention.yaml +8 -0
  16. package/lexicons/attachment-not-mention.yaml +12 -0
  17. package/lexicons/birth-origin.yaml +5 -0
  18. package/lexicons/bookend-heading.yaml +30 -0
  19. package/lexicons/caps-convention.yaml +12 -0
  20. package/lexicons/chat-citation-marker.yaml +11 -0
  21. package/lexicons/citation-author-joiner.yaml +11 -0
  22. package/lexicons/citation-not-author.yaml +40 -0
  23. package/lexicons/clock-meridiem.yaml +13 -0
  24. package/lexicons/definition-statement.yaml +10 -0
  25. package/lexicons/dictionary.txt +87150 -0
  26. package/lexicons/duration-day.yaml +6 -0
  27. package/lexicons/duration-month.yaml +6 -0
  28. package/lexicons/duration-week.yaml +6 -0
  29. package/lexicons/duration-year.yaml +6 -0
  30. package/lexicons/elapsed-link.yaml +11 -0
  31. package/lexicons/elapsed-origin.yaml +8 -0
  32. package/lexicons/elapsed-unit.yaml +6 -0
  33. package/lexicons/email-closing.yaml +28 -0
  34. package/lexicons/email-greeting.yaml +17 -0
  35. package/lexicons/email-reply-mark.yaml +8 -0
  36. package/lexicons/email-subject-field.yaml +5 -0
  37. package/lexicons/exception-marker.yaml +10 -0
  38. package/lexicons/extra-word.yaml +185 -0
  39. package/lexicons/fact-attribute.yaml +6 -0
  40. package/lexicons/fact-label-drop.yaml +13 -0
  41. package/lexicons/fact-label-vague.yaml +16 -0
  42. package/lexicons/fact-separator.yaml +11 -0
  43. package/lexicons/fact-value-end.yaml +8 -0
  44. package/lexicons/figure-in-text.yaml +9 -0
  45. package/lexicons/figure-noun.yaml +31 -0
  46. package/lexicons/heading-form.yaml +63 -0
  47. package/lexicons/heading-number-label.yaml +29 -0
  48. package/lexicons/homophone-slip.yaml +37 -0
  49. package/lexicons/image-file-name.yaml +16 -0
  50. package/lexicons/known-misspelling.yaml +177 -0
  51. package/lexicons/label-affix.yaml +7 -0
  52. package/lexicons/latin-abbreviation.yaml +10 -0
  53. package/lexicons/light-verb.yaml +9 -0
  54. package/lexicons/month-name.yaml +29 -0
  55. package/lexicons/orthographic-variant-distinct.yaml +24 -0
  56. package/lexicons/paper-section-heading.yaml +18 -0
  57. package/lexicons/person-form.yaml +9 -0
  58. package/lexicons/person-not-after.yaml +54 -0
  59. package/lexicons/person-word.yaml +38 -0
  60. package/lexicons/placeholder-word.yaml +10 -0
  61. package/lexicons/pro-con-heading.yaml +15 -0
  62. package/lexicons/quote-style.yaml +17 -0
  63. package/lexicons/redundant-expression.yaml +83 -0
  64. package/lexicons/reference-aside.yaml +21 -0
  65. package/lexicons/reference-list-heading.yaml +11 -0
  66. package/lexicons/reference-topic-link.yaml +5 -0
  67. package/lexicons/reference-topic-vague.yaml +32 -0
  68. package/lexicons/relative-after.yaml +5 -0
  69. package/lexicons/relative-before.yaml +5 -0
  70. package/lexicons/relative-link.yaml +7 -0
  71. package/lexicons/request-action.yaml +32 -0
  72. package/lexicons/request-condition.yaml +15 -0
  73. package/lexicons/request-deadline.yaml +67 -0
  74. package/lexicons/request-ending.yaml +11 -0
  75. package/lexicons/request-owner.yaml +30 -0
  76. package/lexicons/request-pointer.yaml +13 -0
  77. package/lexicons/restating-opener.yaml +22 -0
  78. package/lexicons/shell-fence-language.yaml +21 -0
  79. package/lexicons/shell-prompt.yaml +9 -0
  80. package/lexicons/so-opener.yaml +6 -0
  81. package/lexicons/summary-heading.yaml +13 -0
  82. package/lexicons/unit-data.yaml +28 -0
  83. package/lexicons/unit-length.yaml +44 -0
  84. package/lexicons/unit-mass.yaml +30 -0
  85. package/lexicons/unit-time.yaml +32 -0
  86. package/lexicons/unit-volume.yaml +26 -0
  87. package/lexicons/unnumbered-heading.yaml +27 -0
  88. package/lexicons/vague-link-text.yaml +20 -0
  89. package/lexicons/vague-word.yaml +31 -0
  90. package/lexicons/value-unit.yaml +58 -0
  91. package/lexicons/very-exception.yaml +17 -0
  92. package/lexicons/very-intensifier.yaml +6 -0
  93. package/lexicons/weasel-word.yaml +30 -0
  94. package/lexicons/word-prefix.yaml +31 -0
  95. package/lexicons/word-suffix.yaml +81 -0
  96. package/lexicons/wordy-phrase.yaml +50 -0
  97. package/package.json +1 -1
  98. package/src/dictionary.ts +22 -0
  99. package/src/index.ts +2 -0
@@ -0,0 +1,50 @@
1
+ # Wordy phrases: several words where one or two say the same. Read by wordy-phrase; rewrite is the short form.
2
+ # Only phrases whose short form says the same in every context. "with regard to", "prior to" and "in accordance with"
3
+ # are left out: contracts and statutes use them on purpose. Matched as whole words, case aside.
4
+ id: wordy-phrase
5
+ language: en
6
+ entries:
7
+ - pattern: in order to
8
+ rewrite: to
9
+ - pattern: in order for
10
+ rewrite: for
11
+ - pattern: due to the fact that
12
+ rewrite: because
13
+ - pattern: owing to the fact that
14
+ rewrite: because
15
+ - pattern: in view of the fact that
16
+ rewrite: because
17
+ - pattern: in spite of the fact that
18
+ rewrite: although
19
+ - pattern: despite the fact that
20
+ rewrite: although
21
+ - pattern: at this point in time
22
+ rewrite: now
23
+ - pattern: at the present time
24
+ rewrite: now
25
+ - pattern: at this moment in time
26
+ rewrite: now
27
+ - pattern: in the event that
28
+ rewrite: if
29
+ - pattern: in the near future
30
+ rewrite: soon
31
+ - pattern: for the purpose of
32
+ rewrite: to, or for
33
+ - pattern: on a daily basis
34
+ rewrite: daily
35
+ - pattern: on a weekly basis
36
+ rewrite: weekly
37
+ - pattern: on a monthly basis
38
+ rewrite: monthly
39
+ - pattern: on a regular basis
40
+ rewrite: regularly
41
+ - pattern: a large number of
42
+ rewrite: many
43
+ - pattern: has the ability to
44
+ rewrite: can
45
+ - pattern: have the ability to
46
+ rewrite: can
47
+ - pattern: is in a position to
48
+ rewrite: can
49
+ - pattern: until such time as
50
+ rewrite: until
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@chaffjs/lang-en",
3
- "version": "0.17.0",
3
+ "version": "0.19.0",
4
4
  "description": "English language adapter for chaff",
5
5
  "license": "MIT",
6
6
  "author": "isamu",
@@ -0,0 +1,22 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { fileURLToPath } from "node:url";
3
+ import { dirname, join } from "node:path";
4
+
5
+ // The English word list (lexicons/dictionary.txt, built by scripts/en-dictionary.ts). It is large, so it is read only
6
+ // when a rule asks for it, and once.
7
+
8
+ const FILE = join(dirname(fileURLToPath(import.meta.url)), "..", "lexicons", "dictionary.txt");
9
+
10
+ const loaded: { words: ReadonlySet<string> | undefined } = { words: undefined };
11
+
12
+ const wordsIn = (text: string): ReadonlySet<string> => new Set(text.split("\n").filter((line) => line !== "" && !line.startsWith("#")));
13
+
14
+ export const dictionary = (): ReadonlySet<string> => {
15
+ if (loaded.words !== undefined) return loaded.words;
16
+ try {
17
+ loaded.words = wordsIn(readFileSync(FILE, "utf8"));
18
+ } catch (error) {
19
+ throw new Error(`${FILE}: cannot read the English word list`, { cause: error });
20
+ }
21
+ return loaded.words;
22
+ };
package/src/index.ts CHANGED
@@ -5,6 +5,7 @@ import { sentenceSpans } from "./sentence-split.ts";
5
5
  import { splitAtQuotedStops } from "./quoted-stop.ts";
6
6
  import { reattachClosingQuotes } from "./closing-quote.ts";
7
7
  import { structure } from "./structure.ts";
8
+ import { dictionary } from "./dictionary.ts";
8
9
  import { isReady, prepare, tokenize } from "./pos.ts";
9
10
  import { isJapaneseRun } from "./japanese-run.ts";
10
11
  import type { AdapterNeeds, EmbeddedLanguage, LanguageAdapter, Segmentation, Sentence, Span } from "chaffjs/plugin";
@@ -62,6 +63,7 @@ export const adapter: LanguageAdapter = {
62
63
  },
63
64
  lexicons: LEXICONS,
64
65
  structure,
66
+ dictionary,
65
67
  segment: (text: string): Segmentation => {
66
68
  const quotedStops = sentenceSpans(unmarkNumberStops(unmarkLabelStops(text, LABEL_STOPS))).flatMap((span) => splitAtQuotedStops(text, span));
67
69
  const sentences: Sentence[] = reattachClosingQuotes(text, quotedStops).map((span) => withLanguage(text.slice(span.start, span.end), span));