@mark1russell7/ste-lint 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. package/README.md +225 -2
  2. package/dist/.tsbuildinfo +1 -0
  3. package/dist/cli.d.ts +3 -0
  4. package/dist/cli.d.ts.map +1 -0
  5. package/dist/cli.js +9 -0
  6. package/dist/cli.js.map +1 -0
  7. package/dist/config.d.ts +103 -0
  8. package/dist/config.d.ts.map +1 -0
  9. package/dist/config.js +251 -0
  10. package/dist/config.js.map +1 -0
  11. package/dist/dictionary.d.ts +74 -0
  12. package/dist/dictionary.d.ts.map +1 -0
  13. package/dist/dictionary.js +169 -0
  14. package/dist/dictionary.js.map +1 -0
  15. package/dist/files.d.ts +35 -0
  16. package/dist/files.d.ts.map +1 -0
  17. package/dist/files.js +181 -0
  18. package/dist/files.js.map +1 -0
  19. package/dist/format.d.ts +21 -0
  20. package/dist/format.d.ts.map +1 -0
  21. package/dist/format.js +61 -0
  22. package/dist/format.js.map +1 -0
  23. package/dist/index.d.ts +17 -0
  24. package/dist/index.d.ts.map +1 -0
  25. package/dist/index.js +17 -0
  26. package/dist/index.js.map +1 -0
  27. package/dist/lexicon.d.ts +33 -0
  28. package/dist/lexicon.d.ts.map +1 -0
  29. package/dist/lexicon.js +71 -0
  30. package/dist/lexicon.js.map +1 -0
  31. package/dist/lint.d.ts +28 -0
  32. package/dist/lint.d.ts.map +1 -0
  33. package/dist/lint.js +132 -0
  34. package/dist/lint.js.map +1 -0
  35. package/dist/rules/about-quantity.d.ts +3 -0
  36. package/dist/rules/about-quantity.d.ts.map +1 -0
  37. package/dist/rules/about-quantity.js +34 -0
  38. package/dist/rules/about-quantity.js.map +1 -0
  39. package/dist/rules/index.d.ts +8 -0
  40. package/dist/rules/index.d.ts.map +1 -0
  41. package/dist/rules/index.js +35 -0
  42. package/dist/rules/index.js.map +1 -0
  43. package/dist/rules/ing-verb.d.ts +9 -0
  44. package/dist/rules/ing-verb.d.ts.map +1 -0
  45. package/dist/rules/ing-verb.js +60 -0
  46. package/dist/rules/ing-verb.js.map +1 -0
  47. package/dist/rules/latin-abbreviation.d.ts +3 -0
  48. package/dist/rules/latin-abbreviation.d.ts.map +1 -0
  49. package/dist/rules/latin-abbreviation.js +48 -0
  50. package/dist/rules/latin-abbreviation.js.map +1 -0
  51. package/dist/rules/missing-subject.d.ts +16 -0
  52. package/dist/rules/missing-subject.d.ts.map +1 -0
  53. package/dist/rules/missing-subject.js +138 -0
  54. package/dist/rules/missing-subject.js.map +1 -0
  55. package/dist/rules/modal-verb.d.ts +3 -0
  56. package/dist/rules/modal-verb.d.ts.map +1 -0
  57. package/dist/rules/modal-verb.js +72 -0
  58. package/dist/rules/modal-verb.js.map +1 -0
  59. package/dist/rules/noun-cluster.d.ts +17 -0
  60. package/dist/rules/noun-cluster.d.ts.map +1 -0
  61. package/dist/rules/noun-cluster.js +83 -0
  62. package/dist/rules/noun-cluster.js.map +1 -0
  63. package/dist/rules/paragraph-length.d.ts +3 -0
  64. package/dist/rules/paragraph-length.d.ts.map +1 -0
  65. package/dist/rules/paragraph-length.js +19 -0
  66. package/dist/rules/paragraph-length.js.map +1 -0
  67. package/dist/rules/passive-instruction.d.ts +15 -0
  68. package/dist/rules/passive-instruction.d.ts.map +1 -0
  69. package/dist/rules/passive-instruction.js +38 -0
  70. package/dist/rules/passive-instruction.js.map +1 -0
  71. package/dist/rules/pronouns.d.ts +4 -0
  72. package/dist/rules/pronouns.d.ts.map +1 -0
  73. package/dist/rules/pronouns.js +57 -0
  74. package/dist/rules/pronouns.js.map +1 -0
  75. package/dist/rules/rule.d.ts +46 -0
  76. package/dist/rules/rule.d.ts.map +1 -0
  77. package/dist/rules/rule.js +31 -0
  78. package/dist/rules/rule.js.map +1 -0
  79. package/dist/rules/semicolon.d.ts +3 -0
  80. package/dist/rules/semicolon.d.ts.map +1 -0
  81. package/dist/rules/semicolon.js +20 -0
  82. package/dist/rules/semicolon.js.map +1 -0
  83. package/dist/rules/sentence-length.d.ts +3 -0
  84. package/dist/rules/sentence-length.d.ts.map +1 -0
  85. package/dist/rules/sentence-length.js +31 -0
  86. package/dist/rules/sentence-length.js.map +1 -0
  87. package/dist/rules/unknown-word.d.ts +9 -0
  88. package/dist/rules/unknown-word.d.ts.map +1 -0
  89. package/dist/rules/unknown-word.js +49 -0
  90. package/dist/rules/unknown-word.js.map +1 -0
  91. package/dist/rules/word-list-matcher.d.ts +25 -0
  92. package/dist/rules/word-list-matcher.d.ts.map +1 -0
  93. package/dist/rules/word-list-matcher.js +60 -0
  94. package/dist/rules/word-list-matcher.js.map +1 -0
  95. package/dist/rules/word-list.d.ts +3 -0
  96. package/dist/rules/word-list.d.ts.map +1 -0
  97. package/dist/rules/word-list.js +26 -0
  98. package/dist/rules/word-list.js.map +1 -0
  99. package/dist/run-cli.d.ts +27 -0
  100. package/dist/run-cli.d.ts.map +1 -0
  101. package/dist/run-cli.js +176 -0
  102. package/dist/run-cli.js.map +1 -0
  103. package/dist/source/blocks.d.ts +55 -0
  104. package/dist/source/blocks.d.ts.map +1 -0
  105. package/dist/source/blocks.js +464 -0
  106. package/dist/source/blocks.js.map +1 -0
  107. package/dist/source/inline.d.ts +42 -0
  108. package/dist/source/inline.d.ts.map +1 -0
  109. package/dist/source/inline.js +396 -0
  110. package/dist/source/inline.js.map +1 -0
  111. package/dist/source/jsx.d.ts +18 -0
  112. package/dist/source/jsx.d.ts.map +1 -0
  113. package/dist/source/jsx.js +137 -0
  114. package/dist/source/jsx.js.map +1 -0
  115. package/dist/source/lines.d.ts +34 -0
  116. package/dist/source/lines.d.ts.map +1 -0
  117. package/dist/source/lines.js +83 -0
  118. package/dist/source/lines.js.map +1 -0
  119. package/dist/source/markdown.d.ts +23 -0
  120. package/dist/source/markdown.d.ts.map +1 -0
  121. package/dist/source/markdown.js +42 -0
  122. package/dist/source/markdown.js.map +1 -0
  123. package/dist/source/suppressions.d.ts +38 -0
  124. package/dist/source/suppressions.d.ts.map +1 -0
  125. package/dist/source/suppressions.js +69 -0
  126. package/dist/source/suppressions.js.map +1 -0
  127. package/dist/source/tsdoc.d.ts +40 -0
  128. package/dist/source/tsdoc.d.ts.map +1 -0
  129. package/dist/source/tsdoc.js +176 -0
  130. package/dist/source/tsdoc.js.map +1 -0
  131. package/dist/text/grammar.d.ts +91 -0
  132. package/dist/text/grammar.d.ts.map +1 -0
  133. package/dist/text/grammar.js +289 -0
  134. package/dist/text/grammar.js.map +1 -0
  135. package/dist/text/phrases.d.ts +30 -0
  136. package/dist/text/phrases.d.ts.map +1 -0
  137. package/dist/text/phrases.js +65 -0
  138. package/dist/text/phrases.js.map +1 -0
  139. package/dist/text/sentences.d.ts +45 -0
  140. package/dist/text/sentences.d.ts.map +1 -0
  141. package/dist/text/sentences.js +141 -0
  142. package/dist/text/sentences.js.map +1 -0
  143. package/dist/text/tokens.d.ts +35 -0
  144. package/dist/text/tokens.d.ts.map +1 -0
  145. package/dist/text/tokens.js +113 -0
  146. package/dist/text/tokens.js.map +1 -0
  147. package/dist/text/words.d.ts +58 -0
  148. package/dist/text/words.d.ts.map +1 -0
  149. package/dist/text/words.js +159 -0
  150. package/dist/text/words.js.map +1 -0
  151. package/dist/types.d.ts +18 -0
  152. package/dist/types.d.ts.map +1 -0
  153. package/dist/types.js +20 -0
  154. package/dist/types.js.map +1 -0
  155. package/dist/units.d.ts +47 -0
  156. package/dist/units.d.ts.map +1 -0
  157. package/dist/units.js +42 -0
  158. package/dist/units.js.map +1 -0
  159. package/dist/word-list.d.ts +34 -0
  160. package/dist/word-list.d.ts.map +1 -0
  161. package/dist/word-list.js +90 -0
  162. package/dist/word-list.js.map +1 -0
  163. package/package.json +46 -3
  164. package/src/cli.ts +9 -0
  165. package/src/config.test.ts +188 -0
  166. package/src/config.ts +318 -0
  167. package/src/dictionary.test.ts +82 -0
  168. package/src/dictionary.ts +177 -0
  169. package/src/files.test.ts +107 -0
  170. package/src/files.ts +191 -0
  171. package/src/format.ts +78 -0
  172. package/src/index.test.ts +28 -0
  173. package/src/index.ts +34 -0
  174. package/src/lexicon.ts +92 -0
  175. package/src/lint.test.ts +182 -0
  176. package/src/lint.ts +135 -0
  177. package/src/rules/about-quantity.ts +33 -0
  178. package/src/rules/index.ts +40 -0
  179. package/src/rules/ing-verb.test.ts +48 -0
  180. package/src/rules/ing-verb.ts +54 -0
  181. package/src/rules/latin-abbreviation.ts +46 -0
  182. package/src/rules/missing-subject.test.ts +53 -0
  183. package/src/rules/missing-subject.ts +129 -0
  184. package/src/rules/modal-verb.test.ts +45 -0
  185. package/src/rules/modal-verb.ts +72 -0
  186. package/src/rules/noun-cluster.test.ts +49 -0
  187. package/src/rules/noun-cluster.ts +82 -0
  188. package/src/rules/paragraph-length.test.ts +30 -0
  189. package/src/rules/paragraph-length.ts +19 -0
  190. package/src/rules/passive-instruction.test.ts +38 -0
  191. package/src/rules/passive-instruction.ts +37 -0
  192. package/src/rules/pronouns.test.ts +38 -0
  193. package/src/rules/pronouns.ts +54 -0
  194. package/src/rules/punctuation.test.ts +59 -0
  195. package/src/rules/rule.ts +73 -0
  196. package/src/rules/semicolon.ts +19 -0
  197. package/src/rules/sentence-length.test.ts +52 -0
  198. package/src/rules/sentence-length.ts +32 -0
  199. package/src/rules/unknown-word.test.ts +81 -0
  200. package/src/rules/unknown-word.ts +43 -0
  201. package/src/rules/word-list-matcher.ts +68 -0
  202. package/src/rules/word-list.test.ts +110 -0
  203. package/src/rules/word-list.ts +25 -0
  204. package/src/run-cli.test.ts +189 -0
  205. package/src/run-cli.ts +193 -0
  206. package/src/source/blocks.ts +534 -0
  207. package/src/source/inline.test.ts +92 -0
  208. package/src/source/inline.ts +416 -0
  209. package/src/source/jsx.test.ts +46 -0
  210. package/src/source/jsx.ts +147 -0
  211. package/src/source/lines.test.ts +51 -0
  212. package/src/source/lines.ts +91 -0
  213. package/src/source/markdown.test.ts +210 -0
  214. package/src/source/markdown.ts +49 -0
  215. package/src/source/suppressions.ts +79 -0
  216. package/src/source/tsdoc.test.ts +137 -0
  217. package/src/source/tsdoc.ts +189 -0
  218. package/src/text/grammar.ts +304 -0
  219. package/src/text/phrases.ts +72 -0
  220. package/src/text/sentences.ts +148 -0
  221. package/src/text/text.test.ts +262 -0
  222. package/src/text/tokens.ts +135 -0
  223. package/src/text/words.ts +183 -0
  224. package/src/types.ts +39 -0
  225. package/src/units.ts +86 -0
  226. package/src/word-list.ts +128 -0
  227. package/ste.config.schema.json +87 -0
@@ -0,0 +1,148 @@
1
+ /**
2
+ * Sentence division and STE word counting (STE Rules 8.4 to 8.7).
3
+ */
4
+
5
+ import type { PhraseMatcher } from "./phrases.js";
6
+ import { isCapitalized, isCounted, isNumber, isWord, type Token } from "./tokens.js";
7
+ import { NO_BREAK_ABBREVIATIONS, NUMBER_ABBREVIATIONS, UNITS } from "./words.js";
8
+
9
+ /** A range of tokens: `from` is the first token, `to` is the token after the last one. */
10
+ export type TokenRange = {
11
+ readonly from : number;
12
+ readonly to : number;
13
+ };
14
+
15
+ const TERMINALS = new Set([".", "!", "?", "...", "…"]);
16
+ const CLOSERS = new Set([")", "]", "}", "\"", "”", "’", "'"]);
17
+ const OPENERS = new Set(["(", "[", "\"", "“", "‘", "'"]);
18
+
19
+ function startsSentence(token : Token) : boolean {
20
+ if (token.kind === "code") return true;
21
+ if (token.kind === "punct") return OPENERS.has(token.text);
22
+ return /^[\p{Lu}\p{N}]/u.test(token.text);
23
+ }
24
+
25
+ /** True when the period at `index` comes after an abbreviation, so it does not end the sentence. */
26
+ function isAbbreviationPeriod(tokens : readonly Token[], index : number) : boolean {
27
+ const previous = tokens[index - 1];
28
+ const token = tokens[index]!;
29
+ if (token.text !== "." || !isWord(previous) || token.spaceBefore) return false;
30
+ if (NO_BREAK_ABBREVIATIONS.has(previous.lower)) return true;
31
+ // "Ms." is a title. In technical text, "ms" is the unit millisecond, and it can end a sentence.
32
+ if (previous.text === "Ms") return true;
33
+ if (NUMBER_ABBREVIATIONS.has(previous.lower) && isNumber(tokens[index + 1])) return true;
34
+ // An initial, for example "J. Smith".
35
+ return /^\p{Lu}$/u.test(previous.text) && isCapitalized(tokens[index + 1]);
36
+ }
37
+
38
+ /**
39
+ * This function divides tokens into sentences. A sentence stops at `.`,
40
+ * `!`, `?` or an ellipsis, when white space comes next. After the white
41
+ * space, there must be a capital letter, a digit, a placeholder, or an
42
+ * opening bracket or quote. Closing brackets and quotes after the stop stay
43
+ * in the sentence. A period after an abbreviation, for example `e.g.`, does
44
+ * not stop a sentence.
45
+ */
46
+ export function splitSentences(tokens : readonly Token[]) : TokenRange[] {
47
+ const ranges : TokenRange[] = [];
48
+ let start = 0;
49
+ for (let i = 0; i < tokens.length; i++) {
50
+ const token = tokens[i]!;
51
+ if (token.kind !== "punct" || !TERMINALS.has(token.text)) continue;
52
+ if (isAbbreviationPeriod(tokens, i)) continue;
53
+ let end = i + 1;
54
+ while (end < tokens.length) {
55
+ const next = tokens[end]!;
56
+ if (next.kind === "punct" && !next.spaceBefore && (CLOSERS.has(next.text) || TERMINALS.has(next.text))) end++;
57
+ else break;
58
+ }
59
+ const next = tokens[end];
60
+ if (next === undefined || (next.spaceBefore && startsSentence(next))) {
61
+ ranges.push({ from : start, to : end });
62
+ start = end;
63
+ i = end - 1;
64
+ }
65
+ }
66
+ if (start < tokens.length) ranges.push({ from : start, to : tokens.length });
67
+ return ranges.filter((range) => tokens.slice(range.from, range.to).some(isCounted));
68
+ }
69
+
70
+ /** The pairs of matching parentheses that no other pair contains. A parenthesis without a partner is ignored. */
71
+ export function topLevelParentheses(tokens : readonly Token[]) : TokenRange[] {
72
+ const stack : number[] = [];
73
+ const pairs : TokenRange[] = [];
74
+ tokens.forEach((token, index) => {
75
+ if (token.kind !== "punct") return;
76
+ if (token.text === "(") {
77
+ stack.push(index);
78
+ } else if (token.text === ")" && stack.length > 0) {
79
+ const open = stack.pop()!;
80
+ if (stack.length === 0) pairs.push({ from : open, to : index + 1 });
81
+ }
82
+ });
83
+ return pairs;
84
+ }
85
+
86
+ export type WordCount = {
87
+ /** The STE word count of the sentence. */
88
+ readonly words : number;
89
+ /** The tokens in each top-level parenthesis that has words. Each one is a separate sentence (STE Rule 8.5). */
90
+ readonly parentheticals : readonly (readonly Token[])[];
91
+ };
92
+
93
+ export type CountOptions = {
94
+ /** Multi-word proper nouns and technical nouns that count as one word. */
95
+ readonly properNouns : PhraseMatcher;
96
+ };
97
+
98
+ /**
99
+ * This function counts the words of a sentence as STE Rules 8.5 to 8.7
100
+ * tell:
101
+ *
102
+ * - A placeholder (code span, link tag, URL, quoted text) is one word.
103
+ * - A hyphenated word is one word.
104
+ * - A number and its unit, for example `20 ms`, are one word.
105
+ * - A multi-word proper noun from the glossary is one word.
106
+ * - Two or more words with capital letters after the start of the
107
+ * sentence are a name or a title. They count as one word.
108
+ * - Text in parentheses is one word. Its words make a separate sentence.
109
+ */
110
+ export function countWords(tokens : readonly Token[], options : CountOptions) : WordCount {
111
+ const parentheses = topLevelParentheses(tokens);
112
+ const parentheticals : Token[][] = [];
113
+ let words = 0;
114
+ let first = true;
115
+ let i = 0;
116
+ while (i < tokens.length) {
117
+ const group = parentheses.find((pair) => pair.from === i);
118
+ if (group !== undefined) {
119
+ const inner = tokens.slice(group.from + 1, group.to - 1);
120
+ if (inner.some(isCounted)) {
121
+ words++;
122
+ parentheticals.push(inner);
123
+ first = false;
124
+ }
125
+ i = group.to;
126
+ continue;
127
+ }
128
+ const token = tokens[i]!;
129
+ if (!isCounted(token)) {
130
+ i++;
131
+ continue;
132
+ }
133
+ words++;
134
+ const phrase = options.properNouns.longestMatch(tokens, i);
135
+ if (phrase > 1) {
136
+ i += phrase;
137
+ } else if (isNumber(token) && isWord(tokens[i + 1]) && UNITS.has(tokens[i + 1]!.lower)) {
138
+ i += 2;
139
+ } else if (!first && isCapitalized(token)) {
140
+ i++;
141
+ while (isCapitalized(tokens[i])) i++;
142
+ } else {
143
+ i++;
144
+ }
145
+ first = false;
146
+ }
147
+ return { words, parentheticals };
148
+ }
@@ -0,0 +1,262 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { normalizeInline } from "../source/inline.js";
3
+ import { splitLines } from "../source/lines.js";
4
+ import { findPassives, hasLemmaIn, isInstruction, isParticiple, isVerbUse, lemmas, nextCounted, startsObject } from "./grammar.js";
5
+ import { PhraseMatcher } from "./phrases.js";
6
+ import { countWords, splitSentences, topLevelParentheses } from "./sentences.js";
7
+ import { isAbbreviation, isCapitalized, isCounted, isIdentifierLike, isNumber, normalizeWord, tokenize, type Token } from "./tokens.js";
8
+ import { COMMON_VERBS } from "./words.js";
9
+
10
+ const tokens = (text : string, tsdoc : boolean = false) : Token[] => tokenize(normalizeInline(splitLines(text), { mdx : false, tsdoc }));
11
+ const texts = (text : string) : string[] => tokens(text).map((token) => token.text);
12
+ const sentences = (text : string) : string[] => {
13
+ const all = tokens(text);
14
+ return splitSentences(all).map((range) => all.slice(range.from, range.to).map((token) => token.text).join(" "));
15
+ };
16
+ const noPhrases = new PhraseMatcher();
17
+ const words = (text : string, properNouns : PhraseMatcher = noPhrases) : number => countWords(tokens(text), { properNouns }).words;
18
+ const verbs = { verbs : COMMON_VERBS };
19
+
20
+ describe("tokenize", () => {
21
+ it("keeps hyphenated words, identifiers, paths, versions and flags as one token", () => {
22
+ expect(texts("main-thread I/O e.g. v1.2.3 snake_case don't packages/lag --max-warnings @lag/core #12 $HOME foo() 5–20")).toEqual([
23
+ "main-thread", "I/O", "e.g", ".", "v1.2.3", "snake_case", "don't", "packages/lag", "--max-warnings", "@lag/core", "#12", "$HOME", "foo()", "5–20",
24
+ ]);
25
+ });
26
+
27
+ it("gives punctuation as separate tokens and an ellipsis as one token", () => {
28
+ expect(texts("Wait... then (go); end.")).toEqual(["Wait", "...", "then", "(", "go", ")", ";", "end", "."]);
29
+ });
30
+
31
+ it("gives a code token for each placeholder, with the file range of the code span", () => {
32
+ const all = tokens("Run `pnpm test` now.");
33
+ const code = all.find((token) => token.kind === "code")!;
34
+ expect(code.text).toBe("pnpm test");
35
+ expect([code.start, code.end]).toEqual([4, 15]);
36
+ expect(all.map((token) => token.kind)).toEqual(["word", "code", "word", "punct"]);
37
+ });
38
+
39
+ it("records white space before each token", () => {
40
+ expect(tokens("a, b").map((token) => token.spaceBefore)).toEqual([true, false, true]);
41
+ });
42
+
43
+ it("classifies tokens", () => {
44
+ const [number, numberWord, abbreviation, plural, word, camel] = tokens("1,000 twelve API CPUs Word camelCase");
45
+ expect(isNumber(number)).toBe(true);
46
+ expect(isNumber(numberWord)).toBe(true);
47
+ expect(isAbbreviation(abbreviation)).toBe(true);
48
+ expect(isAbbreviation(plural)).toBe(true);
49
+ expect(isAbbreviation(word)).toBe(false);
50
+ expect(isCapitalized(word)).toBe(true);
51
+ expect(isIdentifierLike(camel)).toBe(true);
52
+ expect(isIdentifierLike(word)).toBe(false);
53
+ expect(isIdentifierLike(tokens("e.g")[0])).toBe(false);
54
+ expect(isCounted(undefined)).toBe(false);
55
+ expect(normalizeWord("It’s")).toBe("it's");
56
+ });
57
+ });
58
+
59
+ describe("splitSentences", () => {
60
+ it("divides at a period, a question mark or an exclamation mark before a capital letter", () => {
61
+ expect(sentences("First one. Second one? Third one! Fourth")).toEqual(["First one .", "Second one ?", "Third one !", "Fourth"]);
62
+ });
63
+
64
+ it("does not divide after an abbreviation, an initial, a decimal or before a lower-case word", () => {
65
+ expect(sentences("Use e.g. A or i.e. B here. J. Smith wrote 1.5 lines. The approx. value is fine. Fig. 3 shows it.")).toEqual([
66
+ "Use e.g . A or i.e . B here .",
67
+ "J . Smith wrote 1.5 lines .",
68
+ "The approx . value is fine .",
69
+ "Fig . 3 shows it .",
70
+ ]);
71
+ expect(sentences("A short ending etc. then more.")).toHaveLength(1);
72
+ });
73
+
74
+ it("divides after a unit at the end of a sentence, for example milliseconds", () => {
75
+ expect(sentences("The handler blocks for 120 ms. The monitor gives 5 s. Ms. Lee wrote it.")).toEqual([
76
+ "The handler blocks for 120 ms .",
77
+ "The monitor gives 5 s .",
78
+ "Ms . Lee wrote it .",
79
+ ]);
80
+ });
81
+
82
+ it("keeps closing brackets and quotes in the sentence", () => {
83
+ expect(sentences("(See the guide.) Next. He said \"Go.\" Then")).toEqual(["( See the guide . )", "Next .", "He said Go. .", "Then"]);
84
+ });
85
+
86
+ it("divides before a code span, a digit or an opening bracket", () => {
87
+ expect(sentences("It stops. `stop()` frees it. 3 monitors run. (Optional) text.")).toHaveLength(4);
88
+ });
89
+
90
+ it("ignores a sentence without words", () => {
91
+ expect(sentences("Text. ...")).toEqual(["Text . ..."]);
92
+ });
93
+ });
94
+
95
+ describe("countWords", () => {
96
+ it("counts each code span as one word", () => {
97
+ expect(words("Run `pnpm -r build --filter @lag/core` and `pnpm test` now.")).toBe(5);
98
+ });
99
+
100
+ it("counts each link tag, URL and quoted text as one word", () => {
101
+ expect(countWords(tokens("Refer to {@link Monitor.stop | the stop method}.", true), { properNouns : noPhrases }).words).toBe(3);
102
+ expect(words("Open https://example.com/a/b/c in the browser.")).toBe(5);
103
+ expect(words("Click the \"Save all changes\" button.")).toBe(4);
104
+ });
105
+
106
+ it("counts a number and its unit as one word, and a hyphenated word as one word", () => {
107
+ expect(words("The unit weighs 20 kilograms.")).toBe(4);
108
+ expect(words("Wait 5 ms for the soap-and-water rinse.")).toBe(6);
109
+ });
110
+
111
+ it("counts text in parentheses as one word and gives it as a separate sentence", () => {
112
+ const all = tokens("Make sure that the switch is released (the legend is off).");
113
+ const count = countWords(all, { properNouns : noPhrases });
114
+ expect(count.words).toBe(8);
115
+ expect(count.parentheticals).toHaveLength(1);
116
+ expect(count.parentheticals[0]!.map((token) => token.text)).toEqual(["the", "legend", "is", "off"]);
117
+ expect(words("Remove the safety pin (10).")).toBe(5);
118
+ expect(words("An unmatched ( bracket.")).toBe(3);
119
+ });
120
+
121
+ it("counts a multi-word proper noun from the glossary, and a group of capitalized words, as one word", () => {
122
+ const proper = new PhraseMatcher({ caseSensitive : true });
123
+ proper.add("Web Worker", true);
124
+ expect(words("Web Worker threads run code.", proper)).toBe(4);
125
+ expect(words("A web worker thread runs code.", proper)).toBe(6);
126
+ expect(words("The first president of the United States of America was George Washington.")).toBe(10);
127
+ expect(words("Use the Long Animation Frames API to measure.")).toBe(5);
128
+ });
129
+
130
+ it("finds only the top-level parentheses", () => {
131
+ expect(topLevelParentheses(tokens("a (b (c) d) e (f)"))).toHaveLength(2);
132
+ });
133
+ });
134
+
135
+ describe("lemmas", () => {
136
+ it("gives the possible base forms of a word", () => {
137
+ expect(lemmas("monitors")).toContain("monitor");
138
+ expect(lemmas("batteries")).toContain("battery");
139
+ expect(lemmas("pushes")).toContain("push");
140
+ expect(lemmas("enabled")).toContain("enable");
141
+ expect(lemmas("debugged")).toContain("debug");
142
+ expect(lemmas("applied")).toContain("apply");
143
+ expect(lemmas("running")).toContain("run");
144
+ expect(lemmas("making")).toContain("make");
145
+ expect(lemmas("dying")).toContain("die");
146
+ expect(lemmas("page's")).toContain("page");
147
+ expect(hasLemmaIn("installs", new Set(["install"]))).toBe(true);
148
+ });
149
+
150
+ it("finds participles", () => {
151
+ expect(isParticiple("written")).toBe(true);
152
+ expect(isParticiple("started")).toBe(true);
153
+ expect(isParticiple("speed")).toBe(false);
154
+ expect(isParticiple("need")).toBe(false);
155
+ });
156
+ });
157
+
158
+ describe("isInstruction", () => {
159
+ const instruction = (text : string) : boolean => isInstruction(tokens(text), verbs);
160
+
161
+ it("finds imperative sentences", () => {
162
+ expect(instruction("Run the tests.")).toBe(true);
163
+ expect(instruction("Then run the tests.")).toBe(true);
164
+ expect(instruction("Please open the file.")).toBe(true);
165
+ expect(instruction("Do not stop the worker.")).toBe(true);
166
+ expect(instruction("Don't stop the worker.")).toBe(true);
167
+ expect(instruction("Never stop the worker.")).toBe(true);
168
+ expect(instruction("Make sure that the port is open.")).toBe(true);
169
+ expect(instruction("Be careful with the cache.")).toBe(true);
170
+ expect(instruction("If the tests fail, run the build again.")).toBe(true);
171
+ expect(instruction("To run the tests, use `pnpm test`.")).toBe(true);
172
+ expect(instruction("Update tests before the release.")).toBe(true);
173
+ });
174
+
175
+ it("does not find descriptive sentences", () => {
176
+ expect(instruction("The monitor runs every second.")).toBe(false);
177
+ expect(instruction("Test runs take 5 s.")).toBe(false);
178
+ expect(instruction("Build is fast.")).toBe(false);
179
+ expect(instruction("Use of the cache is optional.")).toBe(false);
180
+ expect(instruction("If the tests fail the build stops.")).toBe(false);
181
+ expect(instruction("When the page is hidden, the monitor stops.")).toBe(false);
182
+ expect(instruction("`stop()` releases the timers.")).toBe(false);
183
+ expect(instruction("Then.")).toBe(false);
184
+ expect(instruction("If, then.")).toBe(false);
185
+ expect(instruction("(")).toBe(false);
186
+ });
187
+ });
188
+
189
+ describe("isVerbUse", () => {
190
+ const verbAt = (text : string, word : string) : boolean => {
191
+ const all = tokens(text);
192
+ return isVerbUse(all, all.findIndex((token) => token.lower === word), verbs);
193
+ };
194
+
195
+ it("finds verbs from the words near them", () => {
196
+ expect(verbAt("Run the tests.", "run")).toBe(true);
197
+ expect(verbAt("You can run the tests.", "run")).toBe(true);
198
+ expect(verbAt("Do not call it.", "call")).toBe(true);
199
+ expect(verbAt("It is called twice.", "called")).toBe(true);
200
+ expect(verbAt("It returned the value.", "returned")).toBe(true);
201
+ expect(verbAt("The monitor runs every second.", "runs")).toBe(true);
202
+ expect(verbAt("This function returns the value.", "returns")).toBe(true);
203
+ expect(verbAt("The worker is running now.", "running")).toBe(true);
204
+ expect(verbAt("Stop it by calling `stop()`.", "calling")).toBe(true);
205
+ expect(verbAt("Calling `stop()` frees it.", "calling")).toBe(true);
206
+ expect(verbAt("To call the API, use a key.", "call")).toBe(true);
207
+ });
208
+
209
+ it("does not find nouns and adjectives", () => {
210
+ expect(verbAt("Each call costs time.", "call")).toBe(false);
211
+ expect(verbAt("A test run takes time.", "run")).toBe(false);
212
+ expect(verbAt("The return value is a number.", "return")).toBe(false);
213
+ expect(verbAt("API calls are slow.", "calls")).toBe(false);
214
+ expect(verbAt("The running process stops.", "running")).toBe(false);
215
+ expect(verbAt("Run time", "run")).toBe(false);
216
+ expect(verbAt("2 runs", "runs")).toBe(false);
217
+ expect(verbAt("The value returned stays.", "returned")).toBe(false);
218
+ expect(verbAt("For running", "running")).toBe(false);
219
+ });
220
+
221
+ it("finds no verb at a punctuation token", () => {
222
+ expect(isVerbUse(tokens(", x"), 0, verbs)).toBe(false);
223
+ });
224
+ });
225
+
226
+ describe("findPassives", () => {
227
+ const passives = (text : string) => {
228
+ const all = tokens(text);
229
+ return findPassives(all).map((passive) => ({
230
+ words : all.slice(passive.auxiliary, passive.participle + 1).map((token) => token.text).join(" "),
231
+ obligation : passive.obligation,
232
+ modal : passive.modal,
233
+ agent : passive.agent,
234
+ inCondition : passive.inCondition,
235
+ }));
236
+ };
237
+
238
+ it("finds a form of 'be' and a participle, with the details", () => {
239
+ expect(passives("The file is created by the build.")).toEqual([{ words : "is created", obligation : false, modal : false, agent : true, inCondition : false }]);
240
+ expect(passives("The token must be set first.")).toEqual([{ words : "be set", obligation : true, modal : true, agent : false, inCondition : false }]);
241
+ expect(passives("The value has to be written.")[0]!.obligation).toBe(true);
242
+ expect(passives("It can be used.")[0]).toMatchObject({ modal : true, obligation : false });
243
+ expect(passives("Make sure that the server is not started.")[0]).toMatchObject({ words : "is not started", inCondition : true });
244
+ expect(passives("The callback gets called.")).toHaveLength(1);
245
+ });
246
+
247
+ it("ignores a form of 'be' without a participle", () => {
248
+ expect(passives("The value is positive.")).toEqual([]);
249
+ expect(passives("The rate is speed.")).toEqual([]);
250
+ });
251
+ });
252
+
253
+ describe("small helpers", () => {
254
+ it("finds the next counted token and object starts", () => {
255
+ const all = tokens(", the value");
256
+ expect(nextCounted(all, 0)).toBe(1);
257
+ expect(nextCounted(all, 5)).toBe(-1);
258
+ expect(startsObject(all[1])).toBe(true);
259
+ expect(startsObject(undefined)).toBe(false);
260
+ expect(startsObject(all[0])).toBe(false);
261
+ });
262
+ });
@@ -0,0 +1,135 @@
1
+ import { PLACEHOLDER, type InlineText, type Placeholder } from "../source/inline.js";
2
+ import { NUMBER_WORDS } from "./words.js";
3
+
4
+ export type TokenKind = "word" | "code" | "punct";
5
+
6
+ export type Token = {
7
+ readonly kind : TokenKind;
8
+ /** The surface text. For a placeholder, the text in the code span, link tag or quote. */
9
+ readonly text : string;
10
+ /** The text in lower case, with typographic apostrophes changed to `'`. */
11
+ readonly lower : string;
12
+ /** The file offset of the first character. */
13
+ readonly start : number;
14
+ /** The file offset after the last character. */
15
+ readonly end : number;
16
+ /** True when white space comes before the token in the clean text. */
17
+ readonly spaceBefore : boolean;
18
+ readonly placeholder : Placeholder | null;
19
+ };
20
+
21
+ /**
22
+ * One word: letters and digits, with hyphens, dots, slashes, underscores,
23
+ * colons and apostrophes inside it. Thus `main-thread`, `I/O`, `e.g`,
24
+ * `v1.2.3`, `snake_case`, `don't`, `1,000` and `packages/lag` are each one word. A
25
+ * word can start with `--`, `-`, `@`, `#` or `$` (flags, scoped packages,
26
+ * issue numbers, variables), and it can end with `()`.
27
+ */
28
+ const WORD = /(?:--?|[@#$])?[\p{L}\p{N}][\p{L}\p{N}\p{M}]*(?:(?:['’](?=\p{L})|[-_./:+@#\\](?=[\p{L}\p{N}])|(?<=\p{N})–(?=\p{N})|(?<=\p{N}),(?=\p{N}{3}(?!\p{N})))[\p{L}\p{N}\p{M}]*)*(?:\(\))?/uy;
29
+
30
+ const CHAR = /./suy;
31
+
32
+ /** This function divides clean text into word, code and punctuation tokens. */
33
+ export function tokenize(inline : InlineText) : Token[] {
34
+ const { clean, offsets, placeholders } = inline;
35
+ const tokens : Token[] = [];
36
+ let i = 0;
37
+ let space = true;
38
+ while (i < clean.length) {
39
+ const char = clean[i]!;
40
+ if (/\s/.test(char)) {
41
+ space = true;
42
+ i++;
43
+ continue;
44
+ }
45
+ if (char === PLACEHOLDER) {
46
+ const placeholder = placeholders.get(i);
47
+ if (placeholder !== undefined) {
48
+ tokens.push({
49
+ kind : "code",
50
+ text : placeholder.text,
51
+ lower : normalizeWord(placeholder.text),
52
+ start : placeholder.start,
53
+ end : placeholder.end,
54
+ spaceBefore : space,
55
+ placeholder,
56
+ });
57
+ space = false;
58
+ i++;
59
+ continue;
60
+ }
61
+ }
62
+ WORD.lastIndex = i;
63
+ const word = WORD.exec(clean);
64
+ let text : string;
65
+ let kind : TokenKind;
66
+ if (word !== null) {
67
+ text = word[0];
68
+ kind = "word";
69
+ } else if (clean.startsWith("...", i)) {
70
+ text = "...";
71
+ kind = "punct";
72
+ } else {
73
+ CHAR.lastIndex = i;
74
+ text = CHAR.exec(clean)![0];
75
+ kind = "punct";
76
+ }
77
+ tokens.push({
78
+ kind,
79
+ text,
80
+ lower : normalizeWord(text),
81
+ start : offsets[i]!,
82
+ end : offsets[i + text.length - 1]! + 1,
83
+ spaceBefore : space,
84
+ placeholder : null,
85
+ });
86
+ space = false;
87
+ i += text.length;
88
+ }
89
+ return tokens;
90
+ }
91
+
92
+ /** The lower-case form of a word, with typographic apostrophes changed to `'`. */
93
+ export function normalizeWord(text : string) : string {
94
+ return text.toLowerCase().replace(/[’‘]/g, "'");
95
+ }
96
+
97
+ export function isWord(token : Token | undefined) : token is Token {
98
+ return token !== undefined && token.kind === "word";
99
+ }
100
+
101
+ /** True for a word or a placeholder. Both count in the STE word count. */
102
+ export function isCounted(token : Token | undefined) : boolean {
103
+ return token !== undefined && token.kind !== "punct";
104
+ }
105
+
106
+ export function isNumber(token : Token | undefined) : boolean {
107
+ return isWord(token) && (/^[-+]?\d[\d,.–-]*%?$/.test(token.text) || NUMBER_WORDS.has(token.lower));
108
+ }
109
+
110
+ /** True for an abbreviation in capital letters, for example `API`, `CPUs` or `HTTP2`. */
111
+ export function isAbbreviation(token : Token | undefined) : boolean {
112
+ return isWord(token) && /^[A-Z][A-Z0-9]+s?$/.test(token.text) && /[A-Z].*[A-Z]|^[A-Z][0-9]/.test(token.text);
113
+ }
114
+
115
+ /** True when the first character is a capital letter. */
116
+ export function isCapitalized(token : Token | undefined) : boolean {
117
+ return isWord(token) && /^\p{Lu}/u.test(token.text);
118
+ }
119
+
120
+ /**
121
+ * True for a word that looks like code although it is not in code font.
122
+ * Examples are a camelCase or snake_case name, a path with dots or slashes,
123
+ * a flag, a word with digits, and a name with `()`.
124
+ */
125
+ export function isIdentifierLike(token : Token | undefined) : boolean {
126
+ if (!isWord(token)) return false;
127
+ const text = token.text;
128
+ if (/^(?:e\.g|i\.e|u\.s|a\.m|p\.m)$/i.test(text)) return false;
129
+ return /\p{Ll}\p{Lu}/u.test(text)
130
+ || /\p{Lu}{2,}\p{Ll}/u.test(text)
131
+ || /[_./\\:@#$]/.test(text)
132
+ || /^--?/.test(text)
133
+ || text.endsWith("()")
134
+ || (/\d/.test(text) && /\p{L}/u.test(text));
135
+ }