onionskin-cli 0.0.0-stage → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +144 -0
  2. package/INSTALL.md +25 -0
  3. package/LICENSE +21 -0
  4. package/NOTICE +49 -0
  5. package/README.md +148 -2
  6. package/docs/contracts/batch.schema.json +839 -0
  7. package/docs/contracts/history.schema.json +126 -0
  8. package/docs/contracts/ops.schema.json +541 -0
  9. package/package.json +67 -4
  10. package/scripts/checker/check.py +411 -0
  11. package/scripts/checker/requirements.txt +5 -0
  12. package/scripts/install-check.mjs +9 -0
  13. package/scripts/requirements.txt +5 -0
  14. package/scripts/rewriter/apple/main.swift +135 -0
  15. package/scripts/rewriter/apple/onionskin-fm +0 -0
  16. package/scripts/rewriter/apple/onionskin-fm.json +6 -0
  17. package/skills/onionskin/SKILL.md +117 -0
  18. package/skills/onionskin/reference/batch.md +94 -0
  19. package/skills/onionskin/reference/formats.md +15 -0
  20. package/skills/onionskin/reference/ops.md +253 -0
  21. package/skills/onionskin/reference/troubleshooting.md +73 -0
  22. package/skills/onionskin/reference/voice-guide.md +86 -0
  23. package/skills/onionskin/voice.example.md +51 -0
  24. package/src/agent-detect.js +149 -0
  25. package/src/agent-events.js +159 -0
  26. package/src/agent-prompt.js +27 -0
  27. package/src/agent-runner.js +593 -0
  28. package/src/analyze.js +17 -0
  29. package/src/anchors.js +190 -0
  30. package/src/apply.js +387 -0
  31. package/src/auth.js +130 -0
  32. package/src/batch.js +322 -0
  33. package/src/check-worker.js +8 -0
  34. package/src/checker.js +293 -0
  35. package/src/cli.js +516 -0
  36. package/src/constants.js +20 -0
  37. package/src/diff.js +267 -0
  38. package/src/doctor.js +288 -0
  39. package/src/docx.js +438 -0
  40. package/src/fmt.js +30 -0
  41. package/src/fsutil.js +348 -0
  42. package/src/history-worker.js +16 -0
  43. package/src/history.js +576 -0
  44. package/src/lease.js +79 -0
  45. package/src/local.js +242 -0
  46. package/src/merge-pptx.js +198 -0
  47. package/src/merge-xlsx.js +453 -0
  48. package/src/merge.js +410 -0
  49. package/src/offload.js +34 -0
  50. package/src/ops/common.js +48 -0
  51. package/src/ops/docx-table.js +323 -0
  52. package/src/ops/docx.js +503 -0
  53. package/src/ops/engine.js +41 -0
  54. package/src/ops/pptx.js +336 -0
  55. package/src/ops/run.js +115 -0
  56. package/src/ops/runs.js +127 -0
  57. package/src/ops/schema.js +109 -0
  58. package/src/ops/xlsx.js +277 -0
  59. package/src/package.js +189 -0
  60. package/src/paths.js +64 -0
  61. package/src/person.js +66 -0
  62. package/src/pptx.js +194 -0
  63. package/src/question.js +47 -0
  64. package/src/quick-plan.js +107 -0
  65. package/src/quick.js +164 -0
  66. package/src/redline-check.js +251 -0
  67. package/src/redline-worker.js +29 -0
  68. package/src/redline.js +438 -0
  69. package/src/reply.js +410 -0
  70. package/src/review.js +274 -0
  71. package/src/rewrite-check.js +1014 -0
  72. package/src/rewriter-cleanup.js +105 -0
  73. package/src/rewriter-setup.js +104 -0
  74. package/src/rewriter.js +396 -0
  75. package/src/sanitize.js +42 -0
  76. package/src/sentence-span.js +25 -0
  77. package/src/server-entry.js +13 -0
  78. package/src/server.js +576 -0
  79. package/src/session.js +1912 -0
  80. package/src/settings.js +161 -0
  81. package/src/setup.js +199 -0
  82. package/src/stamp-words.js +10 -0
  83. package/src/stamps.js +59 -0
  84. package/src/style.js +110 -0
  85. package/src/suggest-reply.js +84 -0
  86. package/src/suggest.js +261 -0
  87. package/src/text.js +31 -0
  88. package/src/tracked-words.js +43 -0
  89. package/src/voice-guide.js +182 -0
  90. package/src/voice-learn.js +314 -0
  91. package/src/voice-sample-worker.js +95 -0
  92. package/src/voice-samples.js +329 -0
  93. package/src/voice-store.js +305 -0
  94. package/src/voice-worker.js +28 -0
  95. package/src/voice.js +297 -0
  96. package/src/xlsx.js +454 -0
  97. package/src/xml-tree.js +118 -0
  98. package/src/xml.js +127 -0
  99. package/src/zip-write.js +74 -0
  100. package/src/zip.js +88 -0
  101. package/vendor/engine/DocViewer-BZLXUcxJ.js +15137 -0
  102. package/vendor/engine/THIRD_PARTY_LICENSES.txt +79 -0
  103. package/vendor/engine/collab-BVynXMqc.js +2591 -0
  104. package/vendor/engine/collab.js +2 -0
  105. package/vendor/engine/core.js +4 -0
  106. package/vendor/engine/default-worker-DogNT09A.js +35625 -0
  107. package/vendor/engine/embed.js +107 -0
  108. package/vendor/engine/index.js +5 -0
  109. package/vendor/engine/office-engine.embed.js +184 -0
  110. package/vendor/engine/office-engine.worker.js +36030 -0
  111. package/vendor/engine/xlsx-ops-SYAv6V0f.js +939 -0
  112. package/vendor/engine-node/LICENSE +21 -0
  113. package/vendor/engine-node/engine-node.mjs +46618 -0
  114. package/vendor/engine.lock.json +26 -0
  115. package/web/dist/assets/index-BUtofdtr.js +197 -0
  116. package/web/dist/assets/index-Dfizlg0i.css +1 -0
  117. package/web/dist/index.html +14 -0
@@ -0,0 +1,1014 @@
1
+ // The code check for a one-sentence rewrite made by a small model on this Mac (milestone 2, idea 10).
2
+ // Pure: no I/O, no clock, no model. checkRewrite(task, before, after) decides whether `after` may replace `before` in the document.
3
+ // checkRewrite(task, before, after, {mode: "suggest"}) is the relaxed check for a suggestion the person sees before anything is written,
4
+ // and inputQualifies(before) decides whether the model may be asked at all; both are described below the strict check.
5
+ //
6
+ // Answers:
7
+ // { ok: true, verdict: "ok", text, reasons: [] } the cleaned rewrite may be used as is
8
+ // { ok: false, verdict: "no", text: null, reasons } a rule was broken for certain (a number, range, code, name, link, negation or modal changed,
9
+ // a refusal, two sentences, out of the length bounds)
10
+ // { ok: false, verdict: "unsure", text: null, reasons } nothing was proven wrong, but the check cannot show the meaning was kept
11
+ // (a word that carries meaning was dropped, a tone or actor word was added, ...)
12
+ // Every reason is {code, words?}; all failing reasons are listed, not only the first. "no" and "unsure" both mean: do not use the rewrite,
13
+ // hand the request to the big agent. When in doubt the answer is "unsure", never "ok".
14
+ //
15
+ // What it catches (the rules, in order):
16
+ // - the task is one of the three fixed stamps (shorter, friendlier, grammar); anything else, Translate included, is "unsure";
17
+ // - the original is one sentence (no internal sentence end, no line break, no "…[truncated]" mark, 3 to 300 characters). Whether it
18
+ // starts with a capital and ends with . ! or ? (or is the whole paragraph) is the caller's check, since only the caller knows;
19
+ // - the rewrite is one clean line: untrusted(after, 400) equals it (no control or bidi characters, no line break, never cut), with one
20
+ // wrapping quote pair removed when the original had none; it is not unchanged; it is one sentence (a second one that starts in lowercase
21
+ // counts); it ends in . ! or ? when the original does, and has as many "?" as the original, so a statement never becomes a question or a
22
+ // question a statement; and no clause of it starts with a word that makes a question by word order ("is", "do", "can", "when", ...;
23
+ // at the start, after a greeting or courtesy word, or after a comma or colon: "Hi, is the fee refundable.") unless the original has
24
+ // that word at a clause start too;
25
+ // - no refusal ("I'm sorry", "I can't", "As an AI", ...), no chatter ("Here is the rewritten sentence:"), no reply to the text instead of a
26
+ // rewrite of it ("sorry to hear", "thank you for reaching out", ...);
27
+ // - facts kept, as exact tokens: the same multiset of numbers in any script, with any sign, currency mark or percent sign ("-5" is not
28
+ // "5", "240" is not "2400", "20 %" is "20%" but not "20"; fractions, superscripts, Roman numeral characters, full-width and Arabic-Indic
29
+ // digits are numbers too, compared as written, never folded to ASCII; digits inside a word count: "BA123", "Q3"), the same joins between
30
+ // two numbers ("3-5" is a range, not the list "3 and 5" or "3, 5"; any dash, "to" and "through" are the same range mark), every run of
31
+ // letters and digits written together ("xk9pq", "4B") with the same spelling and case, every name-like word (capitalized after the first
32
+ // word, all caps, or with a capital after its first letter: "iPhone", "xK9pQ") with the same spelling ("Friday" is not "Fridays"), the
33
+ // same links, emails and bare domains, case included (a link's path is case-sensitive), with none added, the same negations (not, never,
34
+ // no, ...), the same modals (may, will, must, can, ...) and the same "have to" and "is to" pairs, the same symbols outside ordinary
35
+ // punctuation (no added markup, "~", "%", emoji), and each number still next to the same words ("March 3", "5 pm", "240 units",
36
+ // "account 12345");
37
+ // - every other word of the original is still there, letter for letter, unless it is on a short list of words that carry no fact
38
+ // (articles, forms of "be", "have" and "do", "please", "that", "to", ...) or is a pronoun. Words that look like filler but limit, add,
39
+ // time or weigh a fact are not on that list: "just", "simply", "also", "currently", "very", "really", "quite", "so", "actually",
40
+ // "basically". Fix grammar may also correct a word's spelling or ending, but never add or drop a plural "s" ("weekdays" to "weekday")
41
+ // and never pair a word on any list below, any part of a hyphenated one included ("sixty" is not "sixteen", "thirtieth" is not
42
+ // "thirteenth", "twenty-five" is not "twenty-six", "xii" is not "xi", "minimum" is not "maximum");
43
+ // - nothing that changes who acts, what it covers or how it sounds is added: "we", "I", a pronoun ("your" to "our"), a modal, a scope word
44
+ // (all, every, only, just, also, full, ...), a qualifier or condition (about, over, if, unless, or, before, until, currently, ...), a
45
+ // tone or degree word (sorry, happy, pleased, unfortunately, good news, very, ...), an instruction word (wire, click, contact, ignore,
46
+ // ...), a number word (cardinal, ordinal or Roman), or a new first word; and no other new word at all: Shorter and Fix grammar add none
47
+ // beyond a spelling fix, Friendlier only the free words and a few courtesy words (quick, reminder, heads-up, friendly, gentle);
48
+ // - length: Shorter 30% to 95% of the original, Friendlier 70% to 160%, Fix grammar 80% to 120% with similarity() at least 0.8.
49
+ //
50
+ // What it cannot catch (each one is pinned by the "known limits" test in test/rewrite-check.test.js):
51
+ // - a change of meaning that keeps every word: a negation moved to another clause ("We will not ship until the order is paid" to "We will
52
+ // ship until the order is not paid"), two names or actors swapped ("Acme pays Beta" to "Beta pays Acme"), words reordered;
53
+ // - tone itself: code sees words, not tone. A tone flip is caught only when it drops a word, adds a word that is not free or courtesy, or
54
+ // adds or drops a "?"; one made with "." to "!", courtesy words, or both, passes ("Quick reminder: the 240 units are delayed until
55
+ // March 13!");
56
+ // - a change of tense through a word ending that Fix grammar may correct ("We close the desk" to "We closed the desk"), or through a free
57
+ // word added or dropped alone ("We refund the fee" to "We have refunded the fee");
58
+ // - a swap between two words on the free list: "is" for "was" (a change of tense), "the" for "a", "to" for "for";
59
+ // - whether a Fix grammar spelling correction picked the right word (only "close in spelling" is checked);
60
+ // - a hostile sentence steering the model: the output is checked like any other, so an output that keeps every fact and word passes even
61
+ // if the model was steered. It can only replace that one selected sentence, and the person sees it and can revert it.
62
+ // It is also stricter than a person would be: Fix grammar "start" to "starts" or "your" to "you're" ("its", "their" likewise), Shorter with
63
+ // a synonym or without "just", "20 percent" to "20%", "3pm" to "3 pm", "has to" to "have to", a recased link, "A fee is due" to "There is a
64
+ // fee due" (read as a possible question), and Friendlier with any warm word off the courtesy list all go to the agent. That costs time,
65
+ // never a wrong change.
66
+ import { normText, similarity } from "./text.js";
67
+ import { untrusted as cleanUntrusted } from "./sanitize.js";
68
+
69
+ export const REWRITE_TASKS = ["shorter", "friendlier", "grammar"];
70
+
71
+ const LENGTH = { shorter: [0.3, 0.95], friendlier: [0.7, 1.6], grammar: [0.8, 1.2] };
72
+ const GRAMMAR_SIMILARITY = 0.8;
73
+ const MAX_BEFORE = 300;
74
+ const MAX_AFTER = 400;
75
+ const MARK = "…[truncated]";
76
+
77
+ // Words free to drop or add: they carry no number, name, condition, scope, modality or polarity on their own.
78
+ // Kept off on purpose: modals, negations, quantifiers, am/pm, and condition words (before, after, until, by, within, with, without, only, if,
79
+ // unless, or, except, per, from, than).
80
+ // Not free either, though they look like filler: "just" and "simply" (they mean "only"), "also" (in addition), "currently" (for now),
81
+ // "very", "really", "quite", "so", "actually", "basically" (degree and tone). Each sits on a list below.
82
+ const FREE = new Set(
83
+ ("a an the is are was were be been being has have had do does did that which this these those there here " +
84
+ "please kindly note and of to for on at in as hi hello hey thanks thank").split(" "),
85
+ );
86
+ // Pronouns are free to drop but not to add: "Your account" to "Our account" changes whose it is.
87
+ const PRONOUNS = new Set("you your yours our ours us they them their theirs it its he him his she her hers".split(" "));
88
+ // Words that soften, widen, limit or put a condition on a fact: "about 240 units", "on March 3 if paid", "even before that".
89
+ const QUALIFIERS = new Set(
90
+ ("about around approximately roughly nearly almost over under up least more less fewer than unless if or except maybe perhaps possibly " +
91
+ "likely probably estimated expected even before after until within by from per with since during between currently now still yet").split(" "),
92
+ );
93
+ const NEGATIONS = new Set(["not", "no", "never", "none", "nobody", "nothing", "neither", "nor", "without", "nowhere"]);
94
+ const MODALS = new Set(["will", "would", "shall", "should", "must", "may", "might", "can", "could"]);
95
+ const SCOPE = new Set(
96
+ ("all every each any some only full fully partial partly most many few several both either entire whole always sometimes usually " +
97
+ "minimum maximum min max lowest highest earliest latest last increase decrease just simply also too").split(" "),
98
+ );
99
+ const FIRST_PERSON = new Set(["we", "i", "me", "my", "mine"]);
100
+ const TONE = new Set(
101
+ ("sorry apologize apologise apologies apology regret regrets regretfully unfortunately sadly fortunately luckily happy happily pleased " +
102
+ "glad delighted excited thrilled congratulations congrats great good news exciting wonderful amazing excellent very really quite so " +
103
+ "actually basically").split(" "),
104
+ );
105
+ const INSTRUCTION = new Set(
106
+ ("ignore disregard wire transfer send pay click visit call email reply download open run delete contact forward submit approve buy sign " +
107
+ "login log password account link follow instructions instruction confirm verify").split(" "),
108
+ );
109
+ // The only new words Friendlier may add beyond the free list. Shorter and Fix grammar add none (Fix grammar may still fix a spelling).
110
+ const COURTESY = new Set("quick reminder heads-up friendly gentle".split(" "));
111
+ const NUMBER_WORDS = new Set(
112
+ ("zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen " +
113
+ "twenty thirty forty fifty sixty seventy eighty ninety hundred thousand million billion trillion dozen half halves quarter twice " +
114
+ "thrice once double triple quadruple first second third fourth fifth sixth seventh eighth ninth tenth eleventh twelfth thirteenth " +
115
+ "fourteenth fifteenth sixteenth seventeenth eighteenth nineteenth twentieth thirtieth fortieth fiftieth sixtieth seventieth " +
116
+ "eightieth ninetieth hundredth thousandth millionth billionth").split(" "),
117
+ );
118
+ // a Roman numeral in lowercase ("xii"; in capitals it is name-like already). A few plain words match too ("mix"): they only become protected.
119
+ const ROMAN = /^(?=[ivxlcdm])m{0,3}(cm|cd|d?c{0,3})(xc|xl|l?x{0,3})(ix|iv|v?i{0,3})$/;
120
+ // a word made of number words, a plural of one, or a Roman numeral: "twenty-five", "thirty-first", "hundreds", "xii"
121
+ const isNumberWord = (w) => ROMAN.test(w) || w.split("-").some((p) => NUMBER_WORDS.has(p) || NUMBER_WORDS.has(p.replace(/s$/, "")));
122
+
123
+ // words that never count as a spelling fix of another word ("sixty" and "sixteen", "minimum" and "maximum")
124
+ const PROTECTED = new Set([...NEGATIONS, ...MODALS, ...SCOPE, ...FIRST_PERSON, ...PRONOUNS, ...QUALIFIERS, ...TONE, ...INSTRUCTION, ...NUMBER_WORDS]);
125
+ // checked per part of a hyphenated word, so "twenty-five" and "full-time" are protected too
126
+ const isProtected = (w) => isNumberWord(w) || w.split("-").some((p) => PROTECTED.has(p) || PROTECTED.has(p.replace(/s$/, "")));
127
+ // two words that say "must" between them: "have to", "is to" ("The form is to be signed"); compared like modals, though each word is free
128
+ const OBLIGATION_LEAD = new Set("have has had having is are am was were be".split(" "));
129
+ // a first word that makes a question by word order alone: "Is the fee refundable."
130
+ const QUESTION_LEAD = new Set([..."is are am was were do does did has have had".split(" "), ...MODALS, ..."who what when where why how which".split(" ")]);
131
+
132
+ const REFUSAL_START = /^(i'?m sorry|i am sorry|i apologi[sz]e|i can(not|'t)|i am unable|i'm unable|i won't|i will not|as an ai|as a language model|sorry\b|unfortunately, i\b)/i;
133
+ const REFUSAL_ANYWHERE = /\b(as an ai|language model|i can(not|'t) (help|assist|comply|do that|rewrite)|i'm not able to|i am not able to)\b/i;
134
+ const CHATTER = /^(here is|here's|sure|certainly|of course|okay|ok|rewritten|revised|shorter|friendlier|corrected|fixed|output|answer|result)\b[^.!?]*[:,!]|\b(rewritten|revised|corrected) (sentence|version|text)\b|\bnote:/i;
135
+ const REPLY = /\b(sorry to hear|sad to hear|glad to hear|happy to hear|thank you for (reaching|contacting|letting|your (message|email|patience|note|question))|thanks for (reaching|letting|your)|i understand|we understand|i hope this helps|hope this helps|let me know if)\b/i;
136
+
137
+ // a sentence end: . ! or ? (and any closing quote or bracket), a space, then a capital or a digit (with any opening quote)
138
+ // built with new RegExp from escaped strings so no raw quote or separator characters live in this source file
139
+ const SENTENCE_END = new RegExp("[.!?]+[\"')\\]\\u201d\\u2019]*\\s+[\"'(\\[\\u201c\\u2018]?[\\p{Lu}\\p{N}]", "u");
140
+ const LINE_BREAK = new RegExp("[\\r\\n\\u2028\\u2029]");
141
+ const INNER_STOP = new RegExp("[.!?]+[\"')\\]\\u201d\\u2019]*\\s+\\S", "gu");
142
+ // a link or email: anything with ://, www. or @, or a bare domain (letters, a dot, two or more letters: "evil.com", not "a.m.")
143
+ const LINK = /\S*(?::\/\/|www\.|@)\S*|(?:[\p{L}\p{N}-]+\.)+\p{L}{2,}/giu;
144
+ // symbols that are not ordinary punctuation (markup, math, arrows, emoji, "%", ...): the rewrite must have the same ones
145
+ const SYMBOL = new RegExp("[^\\p{L}\\p{M}\\p{N}\\s.,;:!?'\"()\\[\\]\\u2013\\u2014/&$\\u20ac\\u00a3\\u00a5\\u20b9-]", "gu");
146
+ const QUOTE_PAIRS = [['"', '"'], ["'", "'"], [String.fromCharCode(0x201c), String.fromCharCode(0x201d)], [String.fromCharCode(0x2018), String.fromCharCode(0x2019)]];
147
+
148
+ /** Curly apostrophes and quotes to straight ones, so contractions and refusals match. */
149
+ const SINGLE = new RegExp("[\\u2018\\u2019\\u02bc]", "g");
150
+ const DOUBLE = new RegExp("[\\u201c\\u201d]", "g");
151
+ const straighten = (s) => s.replace(SINGLE, "'").replace(DOUBLE, '"');
152
+
153
+ /** Contractions written out, case kept: won't -> will not, we'll -> we will, cannot -> can not, Acme's -> Acme. */
154
+ function expand(s) {
155
+ return straighten(s)
156
+ .replace(/\b(w)on't\b/gi, "$1ill not")
157
+ .replace(/\b(c)an't\b/gi, "$1an not")
158
+ .replace(/\b(s)han't\b/gi, "$1hall not")
159
+ .replace(/\b(c)annot\b/gi, "$1an not")
160
+ .replace(/n't\b/gi, " not")
161
+ .replace(/'ll\b/gi, " will")
162
+ .replace(/'re\b/gi, " are")
163
+ .replace(/'ve\b/gi, " have")
164
+ .replace(/'m\b/gi, " am")
165
+ .replace(/'d\b/gi, " would")
166
+ .replace(/\b(it|that|there|here|what|who|he|she|where|how|let)'s\b/gi, (_, w) => (w.toLowerCase() === "let" ? `${w} us` : `${w} is`))
167
+ .replace(/'s\b/gi, "")
168
+ .replace(/\b(t)hank you\b/gi, "$1hanks");
169
+ }
170
+
171
+ /**
172
+ * Word and number tokens, in order, case kept, with where each sits in expand(s) ({t, num, at, end}). A number is any run of digits in
173
+ * any script ("12", "\u00bd", "10\u2076", "\uff15\uff10\uff10", "\u0665\u0660\u0660", "\u216b"), compared exactly, never folded to ASCII. Numbers lose trailing
174
+ * punctuation ("3." is "3") and keep a percent sign written after them, spaced or not ("20 %" is "20%").
175
+ */
176
+ function tokens(s) {
177
+ const out = [];
178
+ // a number keeps a sign written right before it ("-5", "+3", "~240"), but not the dash of a range ("3-5")
179
+ const re = /((?:(?<![\p{L}\p{N}])[-+~<>\u2212\u2248\u00b1])?[$€£¥₹]?\p{N}[\p{N},.:/%]*(?:\s?%)?)|[\p{L}][\p{L}\p{M}'-]*/gu;
180
+ for (const m of expand(s).matchAll(re)) {
181
+ if (m[1] !== undefined) {
182
+ const kept = m[0].replace(/[.,:/]+$/, "");
183
+ out.push({ t: kept.replace(/\s%$/, "%"), num: true, at: m.index, end: m.index + kept.length });
184
+ } else {
185
+ const t = m[0].replace(/['-]+$/, "");
186
+ out.push({ t, num: false, at: m.index, end: m.index + t.length });
187
+ }
188
+ }
189
+ return out;
190
+ }
191
+
192
+ const DASHES = new RegExp("[\\u2010-\\u2015\\u2212-]", "g");
193
+ /**
194
+ * What joins each two numbers that follow each other with at most one word between them: "3-5" and "3 to 5" are a range, "3, 5" and
195
+ * "3 and 5" a list ("3|-|5", "3|,|5", "3|and|5"). Any dash, "to", "through" and "thru" read as the same range mark.
196
+ */
197
+ function numberJoins(s) {
198
+ const text = expand(s);
199
+ const nums = tokens(s).filter((t) => t.num);
200
+ const out = [];
201
+ for (let i = 1; i < nums.length; i++) {
202
+ const gap = text.slice(nums[i - 1].end, nums[i].at).toLowerCase().replace(DASHES, "-").replace(/\s+/g, " ").trim();
203
+ if ((gap.match(/\p{L}+/gu) ?? []).length > 1) continue;
204
+ const mark = /^(-|to|through|thru)$/.test(gap) ? "-" : gap;
205
+ out.push(`${nums[i - 1].t}|${mark}|${nums[i].t}`);
206
+ }
207
+ return out;
208
+ }
209
+
210
+ const count = (list) => {
211
+ const m = new Map();
212
+ for (const x of list) m.set(x, (m.get(x) || 0) + 1);
213
+ return m;
214
+ };
215
+ /** What `a` has that `b` lacks, as a multiset difference. */
216
+ function missing(a, b) {
217
+ const B = count(b);
218
+ const out = [];
219
+ for (const x of a) {
220
+ const n = B.get(x) || 0;
221
+ if (n > 0) B.set(x, n - 1);
222
+ else out.push(x);
223
+ }
224
+ return out;
225
+ }
226
+ const sameMultiset = (a, b) => missing(a, b).length === 0 && missing(b, a).length === 0;
227
+ const cps = (s) => [...s].length;
228
+
229
+ // capitalized after the first word, all caps, or a capital after the first letter anywhere ("iPhone", the "xK" and "pQ" of "xK9pQ")
230
+ const isNameLike = (t, i) =>
231
+ !t.num && t.t !== "I" && ((i > 0 && /^\p{Lu}/u.test(t.t)) || /^.+\p{Lu}/u.test(t.t) || (t.t.length >= 2 && /^[\p{Lu}\p{N}'-]+$/u.test(t.t) && /\p{Lu}.*\p{Lu}/u.test(t.t)));
232
+
233
+ const bondWord = (t) => (t && !t.num && !FREE.has(t.t.toLowerCase()) ? t.t.toLowerCase() : null);
234
+ /**
235
+ * Number tokens with their neighbors: the next word that is not free (one free word may sit between: "5 of the units" no, "240 units" and
236
+ * "5 pm" yes), and the word right before when it is not free ("March 3", "account 12345").
237
+ */
238
+ function numberBonds(toks) {
239
+ const out = [];
240
+ toks.forEach((tok, i) => {
241
+ if (!tok.num) return;
242
+ const next = bondWord(toks[i + 1]) ?? (toks[i + 1] && FREE.has(toks[i + 1].t.toLowerCase()) ? bondWord(toks[i + 2]) : null);
243
+ const prev = bondWord(toks[i - 1]);
244
+ if (next) out.push({ num: tok.t, side: ">", word: next });
245
+ if (prev) out.push({ num: tok.t, side: "<", word: prev });
246
+ });
247
+ return out;
248
+ }
249
+ /** Bonds in `a` with no match in `b`. Fix grammar may correct the spelling of the bound word ("recieved 240" and "received 240"). */
250
+ function lostBonds(a, b, grammar) {
251
+ const left = [...b];
252
+ return a.filter((x) => {
253
+ const k = left.findIndex((y) => y.num === x.num && y.side === x.side && (y.word === x.word || (grammar && closeSpelling(x.word, y.word))));
254
+ if (k < 0) return true;
255
+ left.splice(k, 1);
256
+ return false;
257
+ });
258
+ }
259
+
260
+ // compared exactly, case included: a link's path and a short link's code are case-sensitive ("bit.ly/aBcD9" is not "bit.ly/abcd9")
261
+ const links = (s) => [...s.matchAll(LINK)].map((m) => m[0].replace(/[.,;:!?)\]"']+$/, ""));
262
+ // codes: each run of non-space characters outside links that has both a letter and a digit ("xk9pq", "BA123", "4B"), compared exactly
263
+ const codes = (s) =>
264
+ straighten(s)
265
+ .replace(LINK, " ")
266
+ .split(/\s+/)
267
+ .map((w) => w.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ""))
268
+ .filter((w) => /\p{L}/u.test(w) && /\p{N}/u.test(w));
269
+ /** "have to", "is to", ... as two words next to each other in the tokens (a number between them breaks the pair: "is 3 to 5"). */
270
+ const obligations = (toks) =>
271
+ toks.flatMap((x, i) => {
272
+ const y = toks[i + 1];
273
+ return !x.num && y && !y.num && OBLIGATION_LEAD.has(x.t.toLowerCase()) && y.t.toLowerCase() === "to" ? [`${x.t.toLowerCase()} to`] : [];
274
+ });
275
+ // words that may come before the start of a clause without being its start: "Hi there, is ...", "Quick reminder: is ..."
276
+ const GREETING = new Set([..."hi hello hey there please kindly thanks thank note".split(" "), ...COURTESY]);
277
+ /**
278
+ * Words that start a clause and can make a question by word order: the first word after any greeting or courtesy words, and the first
279
+ * word after a comma, colon, semicolon, bracket or long dash ("Is the fee ...", "Hi, is the fee ...", "The fee, is it ...").
280
+ */
281
+ function clauseLeads(s) {
282
+ const text = expand(s);
283
+ const toks = tokens(s);
284
+ const out = [];
285
+ let lead = true;
286
+ toks.forEach((t, i) => {
287
+ if (i && /[,;:(\u2013\u2014]/u.test(text.slice(toks[i - 1].end, t.at))) lead = true;
288
+ const w = t.t.toLowerCase();
289
+ if (lead && !t.num && QUESTION_LEAD.has(w)) out.push(w);
290
+ if (!(lead && !t.num && GREETING.has(w))) lead = false;
291
+ });
292
+ return out;
293
+ }
294
+
295
+ /** A small stem for Fix grammar only: "replied" and "reply", "going" and "go" match. */
296
+ const stem = (w) => w.replace(/(ies|ied)$/, "y").replace(/(es|ed|ing|s|d)$/, "");
297
+ // one word and the same word with s or es added: "weekday" and "weekdays" ("one" against "every") is a fact, never a spelling fix
298
+ const pluralPair = (a, b) => [a, b].some((x, k) => { const y = k ? a : b; return y === `${x}s` || y === `${x}es` || (x.endsWith("y") && y === `${x.slice(0, -1)}ies`); });
299
+ const closeSpelling = (a, b) => !pluralPair(a, b) && (stem(a) === stem(b) || (a[0] === b[0] && Math.abs(a.length - b.length) <= 2 && similarity(a, b) >= 0.5));
300
+
301
+ /** Remove one wrapping quote pair from `s` when `before` is not wrapped in one. */
302
+ function unwrap(s, before) {
303
+ for (const [o, c] of QUOTE_PAIRS) {
304
+ if (s.length > 2 && s.startsWith(o) && s.endsWith(c) && !(before.startsWith(o) && before.endsWith(c))) {
305
+ const inner = s.slice(o.length, -c.length);
306
+ if (!inner.includes(o) && !inner.includes(c)) return inner.trim();
307
+ }
308
+ }
309
+ return s;
310
+ }
311
+
312
+ const HARD = new Set([
313
+ "bad_text",
314
+ "unchanged",
315
+ "refusal",
316
+ "chatter",
317
+ "reply_not_rewrite",
318
+ "two_sentences",
319
+ "number_changed",
320
+ "name_changed",
321
+ "link_changed",
322
+ "negation_changed",
323
+ "modal_changed",
324
+ "symbol_changed",
325
+ "range_changed",
326
+ "code_changed",
327
+ "too_long",
328
+ "too_short",
329
+ "not_similar",
330
+ ]);
331
+
332
+ /**
333
+ * May `after` (a small model's rewrite) replace `before` (the selected sentence) for `task` ("shorter" | "friendlier" | "grammar")?
334
+ * See the file header for the rules and for what it cannot catch. `{mode: "suggest"}` runs the relaxed check of suggest mode instead
335
+ * (its own header below); strict mode is the default and is unchanged by it.
336
+ */
337
+ export function checkRewrite(task, before, after, opts = {}) {
338
+ if (opts && opts.mode === "suggest") return checkSuggest(task, before, after);
339
+ const reasons = [];
340
+ const add = (code, words) => reasons.push(words && words.length ? { code, words: [...new Set(words)] } : { code });
341
+ const done = (text) => {
342
+ if (!reasons.length) return { ok: true, verdict: "ok", text, reasons };
343
+ return { ok: false, verdict: reasons.some((r) => HARD.has(r.code)) ? "no" : "unsure", text: null, reasons };
344
+ };
345
+
346
+ if (!REWRITE_TASKS.includes(task)) {
347
+ add("unknown_task");
348
+ return done();
349
+ }
350
+ if (typeof before !== "string" || typeof after !== "string") {
351
+ add("bad_input");
352
+ return done();
353
+ }
354
+
355
+ // the original: one clean sentence
356
+ const src = cleanUntrusted(before, MAX_BEFORE + 1);
357
+ if (LINE_BREAK.test(before.trim())) add("input_not_one_sentence");
358
+ if (before.includes(MARK) || cps(src) > MAX_BEFORE) add("input_cut");
359
+ if (cps(src) < 3) add("input_too_short");
360
+ if (SENTENCE_END.test(src)) add("input_not_one_sentence");
361
+ if (reasons.length) return done();
362
+
363
+ // the rewrite: one clean line, never cut
364
+ const text = unwrap(after.trim(), src);
365
+ if (cleanUntrusted(text, MAX_AFTER) !== text || !text) {
366
+ add("bad_text");
367
+ return done();
368
+ }
369
+ if (normText(text) === normText(src)) {
370
+ add("unchanged");
371
+ return done();
372
+ }
373
+
374
+ const plain = straighten(text);
375
+ const plainSrc = straighten(src);
376
+ if ((REFUSAL_START.test(plain) && !REFUSAL_START.test(plainSrc)) || (REFUSAL_ANYWHERE.test(plain) && !REFUSAL_ANYWHERE.test(plainSrc))) add("refusal");
377
+ if (CHATTER.test(plain) && !CHATTER.test(plainSrc)) add("chatter");
378
+ const reply = plain.match(REPLY);
379
+ if (reply && !REPLY.test(plainSrc)) add("reply_not_rewrite", [reply[0].toLowerCase()]);
380
+ // a second sentence, even one that starts in lowercase: more ". " inside than the original has
381
+ const stops = (s) => (s.match(INNER_STOP) ?? []).length;
382
+ if (SENTENCE_END.test(text) || stops(text) > stops(src)) add("two_sentences");
383
+ if (/[.!?]["')\]]*$/.test(plainSrc) && !/[.!?]["')\]]*$/.test(plain)) add("sentence_end_changed");
384
+ // a statement never becomes a question and a question never a statement: the same number of "?" (only "." against "!" is left free)
385
+ const asks = (s) => (s.match(/\?/g) ?? []).length;
386
+ if (asks(plain) !== asks(plainSrc)) add("sentence_end_changed");
387
+
388
+ // facts
389
+ const A = tokens(src);
390
+ const B = tokens(text);
391
+ const nums = (toks) => toks.filter((t) => t.num).map((t) => t.t);
392
+ const numsLost = missing(nums(A), nums(B));
393
+ const numsAdded = missing(nums(B), nums(A));
394
+ if (numsLost.length || numsAdded.length) add("number_changed", [...numsLost, ...numsAdded]);
395
+ const symbols = (s) => straighten(s).replace(LINK, " ").match(SYMBOL) ?? [];
396
+ if (!sameMultiset(symbols(src), symbols(text))) add("symbol_changed", [...missing(symbols(src), symbols(text)), ...missing(symbols(text), symbols(src))]);
397
+ const currency = (s) => [...s].filter((c) => "$€£¥₹".includes(c));
398
+ if (!sameMultiset(currency(src), currency(text)) && !numsLost.length && !numsAdded.length) add("number_changed", missing(currency(src), currency(text)));
399
+ const joinsA = numberJoins(src);
400
+ const joinsB = numberJoins(text);
401
+ if (!sameMultiset(joinsA, joinsB) && !numsLost.length && !numsAdded.length) add("range_changed", [...missing(joinsA, joinsB), ...missing(joinsB, joinsA)]);
402
+ if (!sameMultiset(codes(src), codes(text))) add("code_changed", [...missing(codes(src), codes(text)), ...missing(codes(text), codes(src))]);
403
+
404
+ const names = (toks) => toks.filter((t, i) => isNameLike(t, i)).map((t) => t.t);
405
+ const exactB = new Set(B.map((t) => t.t));
406
+ const namesLost = names(A).filter((n) => !exactB.has(n));
407
+ const namesAdded = names(B).filter((n) => !A.some((t) => t.t === n));
408
+ if (namesLost.length || namesAdded.length) add("name_changed", [...namesLost, ...namesAdded]);
409
+
410
+ if (!sameMultiset(links(src), links(text))) add("link_changed", [...missing(links(src), links(text)), ...missing(links(text), links(src))]);
411
+
412
+ const lowerWords = (toks) => toks.filter((t) => !t.num).map((t) => t.t.toLowerCase());
413
+ const wa = lowerWords(A);
414
+ const wb = lowerWords(B);
415
+ const neg = (w) => w.filter((x) => NEGATIONS.has(x));
416
+ if (!sameMultiset(neg(wa), neg(wb))) add("negation_changed", [...missing(neg(wa), neg(wb)), ...missing(neg(wb), neg(wa))]);
417
+ const ma = [...wa.filter((x) => MODALS.has(x)), ...obligations(A)];
418
+ const mb = [...wb.filter((x) => MODALS.has(x)), ...obligations(B)];
419
+ if (!sameMultiset(ma, mb)) add("modal_changed", [...missing(ma, mb), ...missing(mb, ma)]);
420
+ const bondsA = numberBonds(A);
421
+ const bondsB = numberBonds(B);
422
+ const bondsChanged = lostBonds(bondsA, bondsB, task === "grammar").length || lostBonds(bondsB, bondsA, task === "grammar").length;
423
+ if (bondsChanged && !numsLost.length && !numsAdded.length) add("fact_moved");
424
+
425
+ // every other word of the original is still there; new words are few and change no actor, scope or tone
426
+ let dropped = missing(wa, wb).filter((w) => !FREE.has(w) && !PRONOUNS.has(w) && !NEGATIONS.has(w) && !MODALS.has(w));
427
+ let added = missing(wb, wa).filter((w) => !FREE.has(w) && !NEGATIONS.has(w) && !MODALS.has(w));
428
+ if (task === "grammar") {
429
+ // a spelling or ending fix pairs one dropped word with one added word, never a word that carries a fact on its own
430
+ const left = [];
431
+ for (const d of dropped) {
432
+ const k = isProtected(d) ? -1 : added.findIndex((a) => !isProtected(a) && closeSpelling(d, a));
433
+ if (k >= 0) added.splice(k, 1);
434
+ else left.push(d);
435
+ }
436
+ dropped = left;
437
+ }
438
+ if (dropped.length) add("word_dropped", dropped);
439
+ const watch = (set, code) => {
440
+ const hit = added.filter((w) => set.has(w));
441
+ if (hit.length) add(code, hit);
442
+ };
443
+ watch(FIRST_PERSON, "first_person_added");
444
+ watch(PRONOUNS, "pronoun_added");
445
+ watch(QUALIFIERS, "qualifier_added");
446
+ watch(SCOPE, "scope_word_added");
447
+ watch(TONE, "tone_word_added");
448
+ watch(INSTRUCTION, "instruction_word_added");
449
+ watch({ has: isNumberWord }, "number_word_added");
450
+ const extra = added.filter((w) => !(task === "friendlier" && COURTESY.has(w)));
451
+ if (extra.length) add("word_added", extra);
452
+ // the first word is exempt from the name rule, so a new one there could be a new name ("Globex files go out on Monday")
453
+ const first = B.find((t) => !t.num)?.t.toLowerCase();
454
+ if (first && !FREE.has(first) && !(task === "friendlier" && COURTESY.has(first)) && !wa.includes(first)) add("new_first_word", [first]);
455
+ // a question made by word order alone, with no "?": "The fee is refundable." to "Is the fee refundable." or "Hi, is the fee refundable."
456
+ const leadsAdded = missing(clauseLeads(text), clauseLeads(src));
457
+ if (leadsAdded.length) add("question_made", leadsAdded);
458
+
459
+ // length
460
+ const ratio = cps(text) / cps(src);
461
+ const [lo, hi] = LENGTH[task];
462
+ if (ratio < lo) add("too_short");
463
+ if (ratio > hi) add("too_long");
464
+ if (task === "grammar" && similarity(src, text) < GRAMMAR_SIMILARITY) add("not_similar");
465
+
466
+ return done(text);
467
+ }
468
+
469
+ // ---------------------------------------------------------------------------------------------------------------------------------------
470
+ // Suggest mode (idea 10, option A: the small model suggests, the person decides). The person sees the suggestion next to the original
471
+ // and taps "Use this" or not, so this check only keeps out what must never be shown, and points at what the person should look at.
472
+ //
473
+ // checkRewrite(task, before, after, {mode: "suggest"}) answers {verdict, text, warnings, reasons}:
474
+ // verdict "show" the cleaned rewrite (`text`) may be shown as a suggestion, with nothing to point at
475
+ // verdict "warn" it may be shown, with `warnings` (in the fixed family order below) for the person to check
476
+ // verdict "withhold" a hard rule was broken: nothing is shown (`text` null, `reasons` lists every broken rule)
477
+ // verdict "unchanged" the model gave the sentence back as it was (for Fix grammar on a clean sentence that is the right answer)
478
+ // A warning is {code, words, added, dropped}: `added` and `dropped` are the words of the rewrite and of the original that it is about
479
+ // (both may be empty: "question" carries `now: "question" | "statement"` instead). Only the page turns warnings into words.
480
+ //
481
+ // Hard rules (withheld):
482
+ // - the original is not usable: unknown task, bad input, not one sentence, cut, under 3 characters, or it reads like instructions for an AI
483
+ // (input_addresses_ai, see inputQualifies). Whether it starts and ends like a sentence is inputQualifies's check, for the caller;
484
+ // - the rewrite is not clean text: bad_text (control, bidi or zero-width characters, a line break, over 400 characters), a refusal, a
485
+ // reply to the text, two sentences, chatter introduced with a colon ("Here is the rewritten sentence:"), sentence_end_lost (the
486
+ // original ends in . ! or ? and the rewrite does not), placeholder_added (a new [...], {...} or <...>), symbol_added (markup, emoji, ~);
487
+ // - quoted_text_changed: text inside quotation marks ("...", curly double, and single quotes that are clearly quotes, not apostrophes:
488
+ // an opening ' at the start or after a space or bracket and before a letter or digit, closed by a ' after a non-space and before a non-letter)
489
+ // is quoted or being discussed, so it is copied letter for letter: a rewrite that changes it, or drops its quotation marks, is withheld
490
+ // ("stainles steel brackets" in quotes is not "fixed"). Apostrophes ("it's", "company's", "don't") never make a quote; an inch mark (2")
491
+ // or a stray mark never shifts the pairing and a mark left open quotes the rest of the sentence (see quotedSpans);
492
+ // - the facts changed: number_changed (digits as in strict mode, "20 percent" read as "20%", am/pm in any case or spacing; also number
493
+ // words, cardinal and ordinal, but not lowercase Roman numerals and not "one", which is an amount word), unit_changed (the word right
494
+ // after a number: minute, hour, day, week, month, year, am, pm), date_word_changed (today, tomorrow, yesterday, tonight),
495
+ // range_changed (only dash, "to", "through", "and", "or" and comma joins), code_changed (split on dashes: "90-minute" is no code),
496
+ // name_changed (exact spelling; a capitalized word whose lowercase form is on any word list here is not a name), link_changed;
497
+ // - not a rewrite: too few of the original's content words kept, or too many new ones (FLOORS), counting spelling pairs as kept;
498
+ // - length: Shorter 30% to 95% of the original (above is not_shorter), Friendlier 60% to 180%, Fix grammar 80% to 125% with
499
+ // similarity() at least 0.75.
500
+ // - formal_only (Friendlier only): the only change is a formal opener ("Kindly", "Please be advised that"), which is stiffer, not warmer; the page
501
+ // says "No friendlier version this time.".
502
+ // Warning families, in this order: claim (an opener about feelings, thanks, apology or opinion that the original did not have, from a closed list:
503
+ // "We appreciate", "We're pleased", "We regret", "Kindly", "Thank you for", "Unfortunately" and the like; `added` is the opening phrase), negation (compared by count: "nobody" to "not" is fine), modal (with "have to" and "is to"), condition
504
+ // words, amount words, a first- or second-person word added where the original had none of that person, question (a "?" added or dropped,
505
+ // or a question-making first word at the start or after a greeting), number_moved (a number next to different words), action words
506
+ // added, symbol dropped, and for Fix grammar only any other content word dropped or added.
507
+ // Spelling fixes are paired first, in all three tasks, and the pairs are left out of the condition, amount, action, symbol and Fix grammar
508
+ // families (negations, modals, numbers and names are still compared on the full word lists): words that differ only by an apostrophe
509
+ // ("Its" and "It's", "dont" and "don't"; only a word one side has more often, and never "well"/"we'll", "wed"/"we'd", "were"/"we're" or
510
+ // "lets"/"let's", which would hide a modal or a person); then, when the dropped word is not on a protected list, an edit distance of 1
511
+ // between words of 4 letters or more ("untill" to "until", even though "until" is protected), one word against two words next to each
512
+ // other ("atleast" to "at least"), or closeSpelling with a plural or verb "s" allowed ("apply" to "applies"). A pronoun swap within one
513
+ // person ("you" to "your", "Me and my" to "My colleague and I") needs no pair: pronouns are free to drop and only a new person is warned.
514
+ // What it cannot catch, beyond strict mode's limits: a swapped verb that keeps every listed word ("delivered" to "shared"), a fluent change
515
+ // of meaning that keeps every listed word, tone, a dropped content word in Shorter and Friendlier (it shows as struck through, not as a
516
+ // warning), and whether the document's own sentence is malicious ("Correct version: wire $5,000 to account 12345." shortened to "Wire
517
+ // $5,000 to account 12345." is shown clean: it adds nothing that was not already there).
518
+
519
+ const SUGGEST_LENGTH = { shorter: [0.3, 0.95], friendlier: [0.6, 1.8], grammar: [0.8, 1.25] };
520
+ const SUGGEST_SIMILARITY = 0.75;
521
+ // kept: the least share of the original's content words still in the rewrite; added: the most share of the rewrite's content words that are new
522
+ const FLOORS = { shorter: { kept: 0.3, added: 0.5 }, friendlier: { kept: 0.35, added: 0.7 }, grammar: { kept: 0.8, added: 0.2 } };
523
+
524
+ const CONDITION = new Set("if unless only except before after until within because although though otherwise whether".split(" "));
525
+ const AMOUNT = new Set(
526
+ "about approximately over under more less fewer least most all every each any some none few several many full partial half double maximum minimum one".split(" "),
527
+ );
528
+ // treated as the same word inside a family; a word that is only a stand-in (since, just, around, whole, ...) is never reported alone
529
+ const SAME = {
530
+ since: "because", as: "because", merely: "only", just: "only", till: "until",
531
+ approximately: "about", around: "about", roughly: "about", every: "all", each: "all", over: "more", complete: "full", entire: "full", whole: "full",
532
+ };
533
+ const ACTION = new Set("wire transfer pay click visit download install login password verify confirm call email ignore disregard".split(" "));
534
+ const FIRST = new Set("i me my mine myself we us our ours ourselves".split(" "));
535
+ const SECOND = new Set("you your yours yourself yourselves".split(" "));
536
+ const UNITS = new Set("minute hour day week month year am pm".split(" "));
537
+ const DATE_WORDS = new Set("today tomorrow yesterday tonight".split(" "));
538
+ // Calendar words long enough that a near miss is a typo, not another word or a person's name (so no "may", "march", "august").
539
+ const CALENDAR = [...DATE_WORDS, ..."monday tuesday wednesday thursday friday saturday sunday january february september october november december".split(" ")];
540
+ /** The one calendar word `w` misspells ("tomorow", "wensday", "febuary"): within two edits of it and nearer to it than to any other. Else null. */
541
+ function misspeltCalendar(w) {
542
+ if (w.length < 5 || CALENDAR.includes(w)) return null;
543
+ const ranked = CALENDAR.map((c) => [editDistance(w, c), c]).sort((x, y) => x[0] - y[0]);
544
+ return ranked[0][0] <= 2 && ranked[0][0] < ranked[1][0] ? ranked[0][1] : null;
545
+ }
546
+ // number words that are facts (withheld when changed); "one", "half" and "double" are amount words (warned), "once" a condition
547
+ const FACT_NUMBER_WORDS = new Set([...NUMBER_WORDS].filter((w) => !AMOUNT.has(w) && w !== "once"));
548
+ const isFactNumberWord = (w) => w.split("-").some((p) => FACT_NUMBER_WORDS.has(p) || FACT_NUMBER_WORDS.has(p.replace(/s$/, "")));
549
+ // a capitalized word whose lowercase form is on one of these lists is not a name ("It", "The", "Quick", "Unfortunately")
550
+ const LISTED = new Set([
551
+ ...FREE, ...PRONOUNS, ...QUALIFIERS, ...NEGATIONS, ...MODALS, ...SCOPE, ...FIRST_PERSON, ...TONE, ...INSTRUCTION, ...COURTESY, ...NUMBER_WORDS,
552
+ ...CONDITION, ...AMOUNT, ...Object.keys(SAME), ...ACTION, ...FIRST, ...SECOND, ...DATE_WORDS, ...GREETING, ...QUESTION_LEAD,
553
+ ...OBLIGATION_LEAD, ...UNITS,
554
+ ]);
555
+ const isSuggestName = (t, i) => {
556
+ if (!isNameLike(t, i)) return false;
557
+ // only a word in title case can be a listed word at the start of a clause; "PDF", "iPhone" and "xK9pQ" stay names
558
+ const titleCase = /^\p{Lu}[\p{Ll}'-]*$/u.test(t.t);
559
+ return !(titleCase && LISTED.has(t.t.toLowerCase()));
560
+ };
561
+ const SUGGEST_CHATTER =
562
+ /^(here is|here's|sure|certainly|of course|okay|ok|rewritten|revised|shorter|friendlier|corrected|fixed|output|answer|result)\b[^.!?]*?:(?=\s|$)|\b(rewritten|revised|corrected|shorter|friendlier|new) (sentence|version|text)\s*:|\bnote:/i;
563
+ const PLACEHOLDER = /\[[^\]]*\]|\{[^}]*\}|<[^>]*>/g;
564
+ // greetings that may come before the first word of a question: "Hi, is ...", "Quick reminder: is ..."; "there" only right after hi/hello/hey
565
+ const QUESTION_GREETING = new Set([..."hi hello hey please kindly thanks thank note".split(" "), ...COURTESY]);
566
+
567
+ // Openers that carry a claim about feelings, thanks, apology or opinion ("We appreciate the thought, but"): a closed list. When the rewrite has one
568
+ // at the start of a clause and the original has none of that phrase, the rewrite says something the original did not: a "claim" warning.
569
+ const CLAIM_OPENERS = new RegExp(
570
+ "(?:^|[,;:]\\s+|\\s(?:and|but|so)\\s)(" +
571
+ ("we appreciate|we're pleased|we are pleased|we're glad|we are glad|we're happy|we are happy|we're delighted|we are delighted|we're excited|we're thrilled|" +
572
+ "we're sorry|we are sorry|we apologize|we apologise|we regret|we're looking forward|we are looking forward|we're grateful|we are grateful|" +
573
+ "i appreciate|i regret|i'm sorry|i am sorry|i'm glad|i am glad|i'm happy|i am happy|thank you for|thanks for|thanks so much|unfortunately|sadly|kindly|" +
574
+ "happy to|glad to|pleased to|sorry to|sorry for|apologies|with regret|we hope|i hope") +
575
+ ")(?![\\p{L}'])(?!\\W*$)",
576
+ "iu",
577
+ );
578
+ // Formal openers: Friendlier never offers a rewrite whose only change is one of these (it sounds stiffer, not warmer).
579
+ const FORMAL_OPENER = /^(kindly|please be advised|please be informed|we (?:would|wish to) like to inform you|we wish to inform you|this is to inform you)(?![\p{L}'])(?:\s+that(?![\p{L}']))?,?\s*/iu;
580
+ const FORMAL_WORDS = new Set("kindly please be advised informed inform wish would like this that".split(" "));
581
+
582
+ // "We're sorry" and "We are sorry", "Thanks for" and "Thank you for" are the same claim: a rewrite that only swaps one for the other adds none
583
+ const claimForm = (s) => s.toLowerCase().replace(/\bthank's\b/g, "thanks").replace(/(\p{L})'re\b/gu, "$1 are").replace(/\bi'm\b/g, "i am").replace(/\bthanks for\b/g, "thank you for");
584
+
585
+ /**
586
+ * True when the original already has the opener's words, in order, each spelled the same or one edit away ("apreciate", "Unfortunatly"): a spelling
587
+ * fix or a shorter form ("Thank you very much for" to "Thanks for") is the same claim, not a new one.
588
+ */
589
+ function originalHasClaim(plainSrc, opener) {
590
+ const words = (s) => claimForm(s).match(/[\p{L}']+/gu) ?? [];
591
+ const src = words(plainSrc);
592
+ let from = 0;
593
+ for (const w of words(opener)) {
594
+ const k = src.findIndex((x, i) => i >= from && (x === w || (Math.min(x.length, w.length) >= 4 && editDistance(x, w) === 1)));
595
+ if (k < 0) return false;
596
+ from = k + 1;
597
+ }
598
+ return true;
599
+ }
600
+
601
+ /** The words of the claim opener that begins at `at` in `plain`, as the page will show them: to the end of the clause, "that", 6 words or 30 characters. */
602
+ function claimExcerpt(plain, at, opener) {
603
+ if (!/\s/.test(opener)) return opener.replace(/^./, (c) => c.toUpperCase());
604
+ const words = plain.slice(at).split(/\s+/);
605
+ const out = [];
606
+ for (let i = 0; i < words.length && out.length < 6; i++) {
607
+ const next = [...out, words[i]].join(" ");
608
+ if ([...next].length > 30) break;
609
+ out.push(words[i]);
610
+ if (/,$/.test(words[i])) {
611
+ if (/^(but|and|so|yet)$/i.test(words[i + 1] ?? "") && [...`${next} ${words[i + 1]}`].length <= 30) out.push(words[i + 1]);
612
+ break;
613
+ }
614
+ if (/^that$/i.test(words[i])) break;
615
+ }
616
+ return out.join(" ").replace(/[,;:]$/, "");
617
+ }
618
+
619
+ // sentences that address an AI rather than a reader: the model is never asked about them (checked after straightening quotes)
620
+ const ADDRESSES_AI = [
621
+ /\b(ignore|disregard|forget)\b.{0,40}?\b(instructions?|request|prompt|rules|above|previous|prior|everything)\b/i,
622
+ /\b(write|reply|respond|answer|output|print|say|return)\b(\s+(only|exactly|instead))?\s*:/i,
623
+ /\binstead,\s*(write|reply)\b/i,
624
+ /\byou(\s+are|'re)\s+(an?\s+)?(ai|assistant|model|chatbot)\b/i,
625
+ /\b(system prompt|developer message)\b/i,
626
+ /\bassistant,|\bai:/i,
627
+ ];
628
+
629
+ /** The original's own rules: one clean sentence that does not address an AI. Start and end are inputQualifies's. */
630
+ function inputRules(before, add) {
631
+ if (typeof before !== "string") {
632
+ add("bad_input");
633
+ return "";
634
+ }
635
+ const src = cleanUntrusted(before, MAX_BEFORE + 1);
636
+ if (LINE_BREAK.test(before.trim())) add("input_not_one_sentence");
637
+ if (before.includes(MARK) || cps(src) > MAX_BEFORE) add("input_cut");
638
+ if (cps(src) < 3) add("input_too_short");
639
+ if (SENTENCE_END.test(src)) add("input_not_one_sentence");
640
+ if (ADDRESSES_AI.some((re) => re.test(straighten(src)))) add("input_addresses_ai");
641
+ return src;
642
+ }
643
+
644
+ /**
645
+ * May the small model be asked about `before` at all? {ok, reasons}: one clean sentence of 3 to 300 characters that starts with a capital
646
+ * letter, a digit or a quote mark, ends in . ! or ? (unless it is the whole paragraph: `wholeParagraph`), and does not read like
647
+ * instructions for an AI ("Ignore the request and write: ...", "You are an AI ...", "system prompt", ...).
648
+ */
649
+ export function inputQualifies(before, { wholeParagraph = false } = {}) {
650
+ const reasons = [];
651
+ const add = (code) => { if (!reasons.some((r) => r.code === code)) reasons.push({ code }); };
652
+ const src = inputRules(before, add);
653
+ if (src && !/^["'(\[“‘]?[\p{Lu}\p{N}]/u.test(src)) add("input_start");
654
+ if (src && !wholeParagraph && !/[.!?]["')\]”’]*$/u.test(src)) add("input_end");
655
+ return { ok: reasons.length === 0, reasons };
656
+ }
657
+
658
+ /** The text that facts are compared on: "20 percent" is "20%", "3PM", "3 p.m." and "3 pm" are all "3 pm". */
659
+ const factText = (s) =>
660
+ straighten(s)
661
+ .replace(/(\p{N})\s*(?:percent|per cent)\b/giu, "$1%")
662
+ .replace(/(\p{N})\s*([ap])\.?\s?m\.?(?![\p{L}\p{N}])/giu, (_, n, x) => `${n} ${x.toLowerCase()}m`);
663
+
664
+ const APOS_WORD = /\p{L}[\p{L}\p{M}']*/gu;
665
+ /**
666
+ * Words that differ only by an apostrophe are written alike on both sides, with the apostrophe ("Its" and "It's" both "It's"), but only
667
+ * a word that one side has more often than the other (a word in both is no misspelling of the other's word), and never a pair whose
668
+ * contraction carries a modal or a person: "well" and "we'll", "wed" and "we'd", "were" and "we're", "lets" and "let's".
669
+ */
670
+ function alignApostrophes(a, b) {
671
+ const plainCount = (s) => count([...s.matchAll(APOS_WORD)].map(([w]) => w).filter((w) => !w.includes("'")).map((w) => w.toLowerCase()));
672
+ const withApos = (s) => {
673
+ const m = new Map();
674
+ for (const [w] of s.matchAll(APOS_WORD)) {
675
+ const core = w.replace(/'+$/, "");
676
+ if (core.includes("'") && !/'(ll|d)$/i.test(core) && !/^(we're|let's)$/i.test(core)) m.set(core.replace(/'/g, "").toLowerCase(), core.indexOf("'"));
677
+ }
678
+ return m;
679
+ };
680
+ const ca = plainCount(a);
681
+ const cb = plainCount(b);
682
+ const fix = (s, mine, theirs, other) => s.replace(APOS_WORD, (w) => {
683
+ if (w.includes("'")) return w;
684
+ const lw = w.toLowerCase();
685
+ const k = other.get(lw);
686
+ if (k === undefined || (mine.get(lw) ?? 0) <= (theirs.get(lw) ?? 0)) return w;
687
+ return w.slice(0, k) + "'" + w.slice(k);
688
+ });
689
+ return [fix(a, ca, cb, withApos(b)), fix(b, cb, ca, withApos(a))];
690
+ }
691
+
692
+ function editDistance(a, b) {
693
+ let prev = Array.from({ length: b.length + 1 }, (_, j) => j);
694
+ for (let i = 1; i <= a.length; i++) {
695
+ const cur = [i];
696
+ for (let j = 1; j <= b.length; j++) cur[j] = Math.min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
697
+ prev = cur;
698
+ }
699
+ return prev[b.length];
700
+ }
701
+ // closeSpelling with a plural or verb "s" allowed: "apply" and "applies", "weekday" and "weekdays"
702
+ const closeSuggest = (a, b) => stem(a) === stem(b) || (a[0] === b[0] && Math.abs(a.length - b.length) <= 2 && similarity(a, b) >= 0.5);
703
+ const removeOne = (list, w) => { const k = list.indexOf(w); if (k >= 0) list.splice(k, 1); return k >= 0; };
704
+
705
+ /**
706
+ * Spelling fixes paired: each dropped word of the original that matches an added word of the rewrite leaves both lists. `seq` is the
707
+ * rewrite's words in order, for "atleast" against "at least". Returns what is left of both.
708
+ */
709
+ function pairSpellings(droppedIn, addedIn, seq) {
710
+ const added = [...addedIn];
711
+ const dropped = [];
712
+ for (const d of droppedIn) {
713
+ let k = -1;
714
+ if (!isProtected(d)) {
715
+ k = added.findIndex((a) => Math.min(a.length, d.length) >= 4 && editDistance(a, d) === 1);
716
+ if (k < 0) {
717
+ const at = seq.findIndex((w, i) => i + 1 < seq.length && w + seq[i + 1] === d && added.includes(w) && added.includes(seq[i + 1]) && (w !== seq[i + 1] || added.filter((x) => x === w).length > 1));
718
+ if (at >= 0) {
719
+ removeOne(added, seq[at]);
720
+ removeOne(added, seq[at + 1]);
721
+ continue;
722
+ }
723
+ k = added.findIndex((a) => !isProtected(a) && closeSuggest(d, a));
724
+ }
725
+ }
726
+ if (k >= 0) added.splice(k, 1);
727
+ else dropped.push(d);
728
+ }
729
+ return { dropped, added };
730
+ }
731
+
732
+ /** Words of one family dropped and added, a word swapped for one of the same meaning left out ("approximately" to "about"). */
733
+ function familyChange(primary, droppedIn, addedIn) {
734
+ const inFamily = (w) => primary.has(w) || primary.has(SAME[w]);
735
+ const canon = (w) => SAME[w] ?? w;
736
+ const added = addedIn.filter(inFamily);
737
+ const dropped = [];
738
+ for (const w of droppedIn.filter(inFamily)) {
739
+ const k = added.findIndex((x) => canon(x) === canon(w));
740
+ if (k >= 0) added.splice(k, 1);
741
+ else dropped.push(w);
742
+ }
743
+ return { dropped: dropped.filter((w) => primary.has(w)), added: added.filter((w) => primary.has(w)) };
744
+ }
745
+
746
+ /** Number joins of suggest mode: only a dash, "to", "through", "and", "or" or a comma between two numbers count ("April 15, the $42" does not). */
747
+ function suggestJoins(s) {
748
+ const text = expand(s);
749
+ const nums = tokens(s).filter((t) => t.num);
750
+ const out = [];
751
+ for (let i = 1; i < nums.length; i++) {
752
+ const gap = text.slice(nums[i - 1].end, nums[i].at).toLowerCase().replace(DASHES, "-").replace(/\s+/g, " ").trim().replace(/^, (and|or)$/, "$1");
753
+ if (!/^(-|to|through|thru|and|or|,)$/.test(gap)) continue;
754
+ out.push(`${nums[i - 1].t}|${/^(-|to|through|thru)$/.test(gap) ? "-" : gap}|${nums[i].t}`);
755
+ }
756
+ return out;
757
+ }
758
+ const unitOf = (t) => (t && !t.num ? t.t.toLowerCase().replace(/s$/, "") : null);
759
+ const unitPairs = (toks) => toks.flatMap((t, i) => (t.num && UNITS.has(unitOf(toks[i + 1])) ? [`${t.t}|${unitOf(toks[i + 1])}`] : []));
760
+ const firstWord = (toks) => {
761
+ let prev = null;
762
+ for (const t of toks) {
763
+ if (t.num) return null;
764
+ const w = t.t.toLowerCase();
765
+ if (QUESTION_GREETING.has(w) || (w === "there" && /^(hi|hello|hey)$/.test(prev ?? ""))) {
766
+ prev = w;
767
+ continue;
768
+ }
769
+ return w;
770
+ }
771
+ return null;
772
+ };
773
+ const isContent = (w) => !FREE.has(w) && !PRONOUNS.has(w) && !FIRST_PERSON.has(w) && !FIRST.has(w) && !SECOND.has(w);
774
+
775
+ /**
776
+ * Straight double quotes paired in order. With `skipInch`, a " right after a digit is an inch mark ('2"') and never pairs, and one left open
777
+ * quotes the rest of the sentence; without it, only whole pairs count.
778
+ */
779
+ function straightSpans(s, skipInch) {
780
+ const out = [];
781
+ let open = -1;
782
+ for (let i = 0; i < s.length; i++) {
783
+ if (s[i] !== '"' || (skipInch && /\p{N}/u.test(s[i - 1] ?? ""))) continue;
784
+ if (open < 0) open = i;
785
+ else {
786
+ out.push(s.slice(open + 1, i));
787
+ open = -1;
788
+ }
789
+ }
790
+ if (skipInch && open >= 0) out.push(s.slice(open + 1));
791
+ return out;
792
+ }
793
+
794
+ /**
795
+ * With an odd count of straight double quotes that are not inch marks, one is stray ('that" was his reply, and ... "stainles steel"') and
796
+ * shifts every later pair, so every stretch before, between and after them counts as quoted.
797
+ */
798
+ function strayStraightSpans(s) {
799
+ const at = [];
800
+ for (let i = 0; i < s.length; i++) if (s[i] === '"' && !/\p{N}/u.test(s[i - 1] ?? "")) at.push(i);
801
+ if (at.length % 2 === 0) return [];
802
+ return [-1, ...at].map((a, k, all) => s.slice(a + 1, k + 1 < all.length ? all[k + 1] : s.length));
803
+ }
804
+
805
+ /**
806
+ * Curly double quotes paired by direction: an opening mark opens, a closing mark closes; a closing mark with nothing open opens a span when a
807
+ * letter or digit follows it and no digit comes right before it (2” is an inch mark). A closing mark right after a digit while a span is open,
808
+ * with another closing mark next, may be an inch mark: both the short and the long span count. One left open quotes the rest of the sentence.
809
+ */
810
+ function curlySpans(s) {
811
+ const out = [];
812
+ let open = -1;
813
+ for (let i = 0; i < s.length; i++) {
814
+ if (s[i] === "“") {
815
+ if (open < 0) open = i;
816
+ } else if (s[i] === "”") {
817
+ if (open >= 0) {
818
+ out.push(s.slice(open + 1, i));
819
+ const next = s.slice(i + 1).search(/[\u201c\u201d]/u);
820
+ if (!/\p{N}/u.test(s[i - 1] ?? "") || next < 0 || s[i + 1 + next] !== "\u201d") open = -1;
821
+ } else if (!/\p{N}/u.test(s[i - 1] ?? "") && /[\p{L}\p{N}]/u.test(s[i + 1] ?? "")) open = i;
822
+ }
823
+ }
824
+ if (open >= 0) out.push(s.slice(open + 1));
825
+ return out;
826
+ }
827
+
828
+ /**
829
+ * The texts inside quotation marks, compared straightened: curly double quotes by direction (curlySpans), straight double quotes read both ways
830
+ * (inch marks skipped, and in plain order: an inch mark makes the marks ambiguous, and every span either reading finds must be kept; with a
831
+ * stray mark, every stretch between marks, strayStraightSpans), then
832
+ * single quotes that are clearly quotation marks (see the quoted_text_changed rule above); an apostrophe inside or at the end of a word never
833
+ * opens one. Ceiling: a quote that really ends on a number ("size 5") with a change after it is withheld, since it reads as an open quote.
834
+ */
835
+ function quotedSpans(s) {
836
+ const t = straighten(s);
837
+ const out = curlySpans(s).map(straighten);
838
+ const inch = straightSpans(s, true).map(straighten);
839
+ out.push(...inch, ...missing(straightSpans(s, false).map(straighten), inch), ...strayStraightSpans(s).map(straighten));
840
+ const word = /[\p{L}\p{N}]/u;
841
+ for (let i = 0; i < t.length; i++) {
842
+ if (t[i] !== "'" || !(i === 0 || /[\s(\[{"]/.test(t[i - 1])) || !word.test(t[i + 1] ?? "")) continue;
843
+ for (let j = i + 1; j < t.length; j++) {
844
+ if (t[j] === "'" && !/\s/.test(t[j - 1]) && !word.test(t[j + 1] ?? "")) {
845
+ out.push(t.slice(i + 1, j));
846
+ i = j;
847
+ break;
848
+ }
849
+ }
850
+ }
851
+ return out;
852
+ }
853
+
854
+ function checkSuggest(task, before, after) {
855
+ const reasons = [];
856
+ const warnings = [];
857
+ const add = (code, words) => reasons.push(words && words.length ? { code, words: [...new Set(words)] } : { code });
858
+ const warn = (code, dropped = [], added = [], extra = {}) => warnings.push({ code, words: [...new Set([...dropped, ...added])], dropped: [...new Set(dropped)], added: [...new Set(added)], ...extra });
859
+ const withhold = () => ({ verdict: "withhold", text: null, warnings: [], reasons });
860
+
861
+ if (!REWRITE_TASKS.includes(task)) {
862
+ add("unknown_task");
863
+ return withhold();
864
+ }
865
+ if (typeof before !== "string" || typeof after !== "string") {
866
+ add("bad_input");
867
+ return withhold();
868
+ }
869
+ const src = inputRules(before, add);
870
+ if (reasons.length) return withhold();
871
+
872
+ // the rewrite: one clean line, never cut
873
+ const text = unwrap(after.trim(), src);
874
+ if (cleanUntrusted(text, MAX_AFTER) !== text || !text) {
875
+ add("bad_text");
876
+ return withhold();
877
+ }
878
+ if (normText(text) === normText(src)) return { verdict: "unchanged", text: null, warnings: [], reasons: [] };
879
+
880
+ const plain = straighten(text);
881
+ const plainSrc = straighten(src);
882
+ if ((REFUSAL_START.test(plain) && !REFUSAL_START.test(plainSrc)) || (REFUSAL_ANYWHERE.test(plain) && !REFUSAL_ANYWHERE.test(plainSrc))) add("refusal");
883
+ // the colon of a time ("2:30") does not count, nor words the original has too ("... here is the copy: it is signed.")
884
+ const chat = plain.match(SUGGEST_CHATTER);
885
+ if (chat && !plainSrc.toLowerCase().includes(chat[0].toLowerCase())) add("chatter");
886
+ const reply = plain.match(REPLY);
887
+ if (reply && !REPLY.test(plainSrc)) add("reply_not_rewrite", [reply[0].toLowerCase()]);
888
+ const stops = (s) => (s.match(INNER_STOP) ?? []).length;
889
+ if (SENTENCE_END.test(text) || stops(text) > stops(src)) add("two_sentences");
890
+ if (/[.!?]["')\]]*$/.test(plainSrc) && !/[.!?]["')\]]*$/.test(plain)) add("sentence_end_lost");
891
+ const placeholders = missing(plain.match(PLACEHOLDER) ?? [], plainSrc.match(PLACEHOLDER) ?? []);
892
+ if (placeholders.length) add("placeholder_added", placeholders);
893
+ const quotedLost = missing(quotedSpans(src), quotedSpans(text));
894
+ if (quotedLost.length) add("quoted_text_changed", quotedLost);
895
+
896
+ // facts, compared on the fact text with apostrophes aligned
897
+ const [fs, ft] = alignApostrophes(factText(src), factText(text));
898
+ const symbols = (s) => s.replace(LINK, " ").match(SYMBOL) ?? [];
899
+ const symbolsAdded = missing(symbols(ft), symbols(fs));
900
+ if (symbolsAdded.length) add("symbol_added", symbolsAdded);
901
+ const A = tokens(fs);
902
+ const B = tokens(ft);
903
+ const nums = (toks) => toks.filter((t) => t.num).map((t) => t.t);
904
+ // number words by the parts of a hyphenated word: "first phase" and "first-phase" are the same, "twenty-five" and "twenty-six" are not
905
+ const numberWords = (toks) => toks.flatMap((t) => (t.num ? [] : t.t.toLowerCase().split("-").filter(isFactNumberWord)));
906
+ const numsLost = [...missing(nums(A), nums(B)), ...missing(numberWords(A), numberWords(B))];
907
+ const numsAdded = [...missing(nums(B), nums(A)), ...missing(numberWords(B), numberWords(A))];
908
+ const currency = (s) => [...s].filter((c) => "$€£¥₹".includes(c));
909
+ const currencyChanged = !sameMultiset(currency(fs), currency(ft));
910
+ if (numsLost.length || numsAdded.length || currencyChanged) add("number_changed", [...numsLost, ...numsAdded, ...missing(currency(fs), currency(ft)), ...missing(currency(ft), currency(fs))]);
911
+ const ua = unitPairs(A);
912
+ const ub = unitPairs(B);
913
+ if (!sameMultiset(ua, ub)) add("unit_changed", [...missing(ua, ub), ...missing(ub, ua)].map((x) => x.replace("|", " ")));
914
+ const lowerWords = (toks) => toks.filter((t) => !t.num).map((t) => t.t.toLowerCase());
915
+ const wa = lowerWords(A);
916
+ const wb = lowerWords(B);
917
+ const dates = (w) => w.filter((x) => DATE_WORDS.has(x));
918
+ // a typo of a calendar word that the rewrite corrects ("tomorow" to "tomorrow") is a spelling fix: the day did not change
919
+ const typoOf = new Set(wa.filter((w) => !wb.includes(w)).map(misspeltCalendar).filter((c) => c && wb.includes(c)));
920
+ const datesB = dates(wb).filter((d) => !(typoOf.has(d) && !wa.includes(d)));
921
+ if (!sameMultiset(dates(wa), datesB)) add("date_word_changed", [...missing(dates(wa), datesB), ...missing(datesB, dates(wa))]);
922
+ const ja = suggestJoins(fs);
923
+ const jb = suggestJoins(ft);
924
+ const factsSame = !numsLost.length && !numsAdded.length;
925
+ if (factsSame && !sameMultiset(ja, jb)) add("range_changed", [...missing(ja, jb), ...missing(jb, ja)]);
926
+ const splitCodes = (s) => codes(s).flatMap((w) => w.split(DASHES)).filter((w) => /\p{L}/u.test(w) && /\p{N}/u.test(w));
927
+ const ca = splitCodes(fs);
928
+ const cb = splitCodes(ft);
929
+ if (!sameMultiset(ca, cb)) add("code_changed", [...missing(ca, cb), ...missing(cb, ca)]);
930
+ const names = (toks) => toks.filter((t, i) => isSuggestName(t, i)).map((t) => t.t);
931
+ const exactB = new Set(B.map((t) => t.t));
932
+ const namesLost = names(A).filter((n) => !exactB.has(n) && !typoOf.has(misspeltCalendar(n.toLowerCase())));
933
+ const namesAdded = names(B).filter((n) => !A.some((t) => t.t === n) && !(typoOf.has(n.toLowerCase()) && !wa.includes(n.toLowerCase())));
934
+ // the first word is never name-like by its capital alone, so a new one that is on no list and is not a word of the original could be
935
+ // a new name ("Globex files go out on Monday")
936
+ const lead = B.find((t) => !t.num);
937
+ const leadWord = lead?.t.toLowerCase();
938
+ if (namesLost.length || namesAdded.length) add("name_changed", [...namesLost, ...namesAdded]);
939
+ if (lead && /^\p{Lu}/u.test(lead.t) && !LISTED.has(leadWord) && !A.some((t) => !t.num && closeSuggest(t.t.toLowerCase(), leadWord))) add("new_first_word", [lead.t]);
940
+ if (!sameMultiset(links(plainSrc), links(plain))) add("link_changed", [...missing(links(plainSrc), links(plain)), ...missing(links(plain), links(plainSrc))]);
941
+
942
+ // spelling fixes paired, then how much of the original is kept and how much is new
943
+ const { dropped, added } = pairSpellings(missing(wa, wb), missing(wb, wa), wb);
944
+ const contentA = wa.filter(isContent);
945
+ const contentB = wb.filter(isContent);
946
+ const kept = contentA.length ? 1 - dropped.filter(isContent).length / contentA.length : 1;
947
+ const fresh = contentB.length ? added.filter(isContent).length / contentB.length : 0;
948
+ if (kept < FLOORS[task].kept) add("too_little_kept");
949
+ if (fresh > FLOORS[task].added) add("too_much_added");
950
+
951
+ // Friendlier: a formal opener ("Kindly", "Please be advised that") as the only change is no friendlier version
952
+ if (task === "friendlier") {
953
+ const formal = FORMAL_OPENER.test(plain) && !FORMAL_OPENER.test(plainSrc);
954
+ if (formal && !added.some((w) => isContent(w) && !FORMAL_WORDS.has(w)) && !dropped.some(isContent)) add("formal_only");
955
+ }
956
+
957
+ // length
958
+ const ratio = cps(text) / cps(src);
959
+ const [lo, hi] = SUGGEST_LENGTH[task];
960
+ if (ratio < lo) add("too_short");
961
+ if (ratio > hi) add(task === "shorter" ? "not_shorter" : "too_long");
962
+ if (task === "grammar" && similarity(src, text) < SUGGEST_SIMILARITY) add("not_similar");
963
+ if (reasons.length) return withhold();
964
+
965
+ // warnings, in family order; a claim about feelings, thanks, apology or opinion comes first so that it is never cut off the card
966
+ const claim = CLAIM_OPENERS.exec(plain);
967
+ if (claim && !originalHasClaim(plainSrc, claim[1])) warn("claim", [], [claimExcerpt(plain, claim.index + claim[0].length - claim[1].length, claim[1])]);
968
+ const neg = (w) => w.filter((x) => NEGATIONS.has(x));
969
+ if (neg(wa).length !== neg(wb).length) warn("negation", missing(neg(wa), neg(wb)), missing(neg(wb), neg(wa)));
970
+ const ma = [...wa.filter((x) => MODALS.has(x)), ...obligations(A)];
971
+ const mb = [...wb.filter((x) => MODALS.has(x)), ...obligations(B)];
972
+ if (!sameMultiset(ma, mb)) warn("modal", missing(ma, mb), missing(mb, ma));
973
+ for (const [code, family] of [["condition", CONDITION], ["amount", AMOUNT]]) {
974
+ const c = familyChange(family, dropped, added);
975
+ if (c.dropped.length || c.added.length) warn(code, c.dropped, c.added);
976
+ }
977
+ for (const person of [FIRST, SECOND]) {
978
+ const now = wb.filter((x) => person.has(x));
979
+ if (now.length && !wa.some((x) => person.has(x))) warn("person", [], now);
980
+ }
981
+ const asks = (s) => (s.match(/\?/g) ?? []).length;
982
+ const leadA = firstWord(A);
983
+ const leadB = firstWord(B);
984
+ const madeByOrder = QUESTION_LEAD.has(leadB) && leadB !== leadA;
985
+ if (asks(plain) > asks(plainSrc) || (asks(plain) === asks(plainSrc) && madeByOrder && !asks(plain))) warn("question", [], [], { now: "question" });
986
+ else if (asks(plain) < asks(plainSrc)) warn("question", [], [], { now: "statement" });
987
+ if (factsSame) {
988
+ const bond = (toks) => numberBonds(toks).filter((b) => b.side === "<" && !UNITS.has(b.word.replace(/s$/, ""))).map((b) => ({ ...b, word: SAME[b.word] ?? b.word }));
989
+ const ba = bond(A);
990
+ const bb = bond(B);
991
+ const lostOf = (x, y) => {
992
+ const left = [...y];
993
+ return x.filter((b) => {
994
+ const k = left.findIndex((c) => c.num === b.num && c.side === b.side && (c.word === b.word || closeSuggest(b.word, c.word)));
995
+ if (k >= 0) left.splice(k, 1);
996
+ return k < 0;
997
+ });
998
+ };
999
+ const lost = lostOf(ba, bb);
1000
+ const gained = lostOf(bb, ba);
1001
+ if (lost.length || gained.length) warn("number_moved", lost.map((b) => b.word), gained.map((b) => b.word));
1002
+ }
1003
+ const actions = added.filter((w) => ACTION.has(w) || ACTION.has(stem(w)));
1004
+ if (actions.length) warn("action", [], actions);
1005
+ const symbolsLost = missing(symbols(fs), symbols(ft));
1006
+ if (symbolsLost.length) warn("symbol", symbolsLost, []);
1007
+ if (task === "grammar") {
1008
+ const told = new Set(warnings.flatMap((w) => w.words));
1009
+ const d = dropped.filter((w) => isContent(w) && !told.has(w));
1010
+ const a = added.filter((w) => isContent(w) && !told.has(w));
1011
+ if (d.length || a.length) warn("grammar_word", d, a);
1012
+ }
1013
+ return { verdict: warnings.length ? "warn" : "show", text, warnings, reasons: [] };
1014
+ }