@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.20.0

package/src/personal-evidence.js

@@ -0,0 +1,233 @@
+import { collectMatches } from './detection.js';
+import { credentialPlaceholder } from './credentials.js';
+
+// Phrase-level evidence collected from competitor test suites (#128):
+// self-introductions, letter salutations and sign-offs, mail display names,
+// spelled-out e-mail addresses, post-office boxes, labelled messenger
+// handles, username/password pairs and labelled card numbers.
+const personal = (type, rule, confidence = 0.85) => ({
+  type,
+  rule,
+  category: 'personal',
+  confidence,
+});
+const word = "\\p{Lu}[\\p{Ll}\\p{M}'’]+(?:\\p{Lu}[\\p{Ll}\\p{M}]+)?";
+const nameWords = `${word}(?:-${word})?(?:[ \\t]+(?:${word}(?:-${word})?|\\p{Lu}\\.)){0,2}`;
+// Addressees and senders that are roles, not people.
+const roleWord =
+  /^(?:Sir|Madam|Madame|Sirs|All|Everyone|Everybody|Team|Folks|Friends?|Colleagues?|Customer|Client|User|Users|Members?|Support|Admin|Team[ \t].*|Hiring[ \t]+Manager|Recruiter|There|Guys|Again|You|Me|Us|Them|God|Mom|Dad|Sorry|Thanks|Yes|No|Not|Here|Also|Just|Back|Done|OK|Okay|The|Our|Your|My|This|World)$/iu;
+const person = (value) =>
+  !roleWord.test(value) && !roleWord.test(value.split(/[ \t]+/)[0]);
+
+const dash = '(?:-{1,2}|—|–|\\.{2,3}|~)';
+function detectNames(text, emit) {
+  // "My name is Bond", "I'm called Jane Doe"
+  collectMatches(
+    text,
+    new RegExp(
+      `(?<![\\p{L}\\p{N}_])(?:[Mm]y[ \\t]+name[ \\t]+is|[Nn]ame's|I[ \\t]+am[ \\t]+called|[Cc]all[ \\t]+me)[ \\t]+(${nameWords})(?![\\p{L}\\p{M}])`,
+      'gu'
+    ),
+    emit,
+    personal('PERSON', 'self-introduction', 0.9),
+    1,
+    person
+  );
+  // "Dear Clifford," and "Hi Jane Doe," at the start of a line.
+  collectMatches(
+    text,
+    new RegExp(
+      `(?:^|(?<=[\\r\\n]))[ \\t]*(?:Dear|Hi|Hello|Hey|Greetings)[ \\t]+(${nameWords})[ \\t]*[,!:]`,
+      'gmu'
+    ),
+    emit,
+    personal('PERSON', 'salutation', 0.85),
+    1,
+    person
+  );
+  // "Thanks -Jon", "All the best,\n\n--Meg C.", "Thank you ..Risemamy McCrubben"
+  collectMatches(
+    text,
+    new RegExp(
+      `(?<![\\p{L}\\p{N}_])(?:Thanks|Thank[ \\t]+you|Many[ \\t]+thanks|Regards|Best[ \\t]+regards|Kind[ \\t]+regards|Warm[ \\t]+regards|Best[ \\t]+wishes|All[ \\t]+the[ \\t]+best|Cheers|Sincerely|Yours[ \\t]+(?:truly|sincerely)|Best)(?:[ \\t]*[,!.]?[ \\t]*(?:\\r?\\n[ \\t]*){1,3}${dash}?|[ \\t]*[,!.][ \\t]*${dash}?|[ \\t]*${dash})[ \\t]*(${nameWords})[ \\t]*\\.?[ \\t]*(?=\\r?\\n|$)`,
+      'gu'
+    ),
+    emit,
+    personal('PERSON', 'sign-off', 0.85),
+    1,
+    person
+  );
+  // Mail display names: `Bob Stone <bob.stone@example.com>`.
+  collectMatches(
+    text,
+    new RegExp(
+      `(?<![\\p{L}\\p{N}_])"?(${word}(?:[ \\t]+(?:${word}|\\p{Lu}\\.)){1,3})"?[ \\t]*<[^\\s<>@]+@[^\\s<>@]+>`,
+      'gu'
+    ),
+    emit,
+    personal('PERSON', 'mail-display-name', 0.9),
+    1,
+    person
+  );
+}
+
+const freemail =
+  'gmail|googlemail|yahoo|ymail|hotmail|outlook|live|msn|icloud|me|mac|aol|proton|protonmail|pm|gmx|mail|yandex|zoho|fastmail|hey|tutanota|qq|163|126|naver|web';
+const at =
+  '(?:[ \\t]*[[({<][ \\t]*at[ \\t]*[\\])}>][ \\t]*|[ \\t]+at[ \\t]+|@)';
+const dot =
+  '(?:[ \\t]*[[({<][ \\t]*dot[ \\t]*[\\])}>][ \\t]*|[ \\t]+dot[ \\t]+|\\.)';
+const obfuscatedEmail = new RegExp(
+  `(?<![\\p{L}\\p{N}_.+-])[\\p{L}\\p{N}_+-]+(?:${dot}[\\p{L}\\p{N}_+-]+)*${at}[\\p{L}\\p{N}-]+(?:${dot}[\\p{L}\\p{N}-]+)*${dot}\\p{L}{2,24}(?![\\p{L}\\p{N}_-])`,
+  'giu'
+);
+// Spelled-out separators are evidence; a plain " at " needs a well-known
+// mailbox provider so that "meet at example.com" stays prose.
+function spelledEmail(value) {
+  const bracketed = /[[({<][ \t]*(?:at|dot)[ \t]*[\])}>]/i.test(value);
+  const spelled = /[ \t]dot[ \t]/i.test(value);
+  const plainAt = new RegExp(
+    `[ \\t]at[ \\t]+(?:${freemail})\\.[a-z]{2,6}(?:\\.[a-z]{2})?$`,
+    'i'
+  ).test(value);
+  return (
+    !/^[^@]+@[^@\s[({<]+$/.test(value) && (bracketed || spelled || plainAt)
+  );
+}
+
+function detectContactForms(text, emit) {
+  collectMatches(
+    text,
+    obfuscatedEmail,
+    emit,
+    personal('EMAIL', 'obfuscated-email', 0.85),
+    0,
+    spelledEmail
+  );
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])(?:P\.?[ \t]?O\.?[ \t]?Box|Post[ \t]+Office[ \t]+Box|Postfach|Apartado(?:[ \t]+de[ \t]+correos)?|Caixa[ \t]+Postal|Boîte[ \t]+postale|а\/я|абонентский[ \t]+ящик)[ \t]*(?:#[ \t]*)?\d{1,10}(?!\p{N})/giu,
+    emit,
+    personal('ADDRESS', 'po-box')
+  );
+  // `skype: dean.malmgren`, `contact me on skype (dean.malmgren)`.
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])(?:skype|discord|telegram|signal|wechat|whatsapp|keybase|matrix|mastodon|twitter|github|gitlab|instagram|tiktok)(?:[ \t]+(?:id|name|handle|username|user))?[ \t]*(?:[:=][ \t]*|\([ \t]*)(@?[\p{L}\p{N}_][\p{L}\p{N}_.:-]{1,63}[\p{L}\p{N}_])(?=[ \t]*\)|[\s,;!?]|\.(?:\s|$)|$)/giu,
+    emit,
+    personal('USERNAME', 'messenger-label', 0.85),
+    1
+  );
+}
+
+// "login snoop pw biggreenhat", "u: snoop\np: biggreenhat",
+// "user: thislibrary\npass: …": a username label followed by a password
+// label marks both values; a lone `user:` is a chat role, not an account.
+// Labels with a colon may be short (`u:`/`p:`); bare labels must be the
+// unambiguous `login`/`username` and `pw`/`pwd`/`password` words.
+const pairSeparator = '[ \\t]*[,;]?[ \\t]*(?:\\r?\\n[ \\t]*)?';
+const credentialPair = new RegExp(
+  [
+    `(?:login|user(?:name)?|u)[ \\t]*[:=][ \\t]*([^\\s:=,;]{2,64})${pairSeparator}(?:pass(?:word)?|passwd|pwd|pw|p)[ \\t]*[:=][ \\t]*(\\S{4,128})`,
+    `(?:login|username)[ \\t]+([^\\s:=,;]{2,64})${pairSeparator}(?:password|passwd|pwd|pw)[ \\t]+(\\S{4,128})`,
+  ]
+    .map((form) => `(?<![\\p{L}\\p{N}_])${form}`)
+    .join('|'),
+  'giu'
+);
+const placeholder = /^\[[\p{L}\p{N}_ -]+\]$/u;
+function detectCredentialPairs(text, emit) {
+  for (const match of text.matchAll(credentialPair)) {
+    const user = match[1] ?? match[3];
+    const secret = match[2] ?? match[4];
+    if (
+      placeholder.test(user) ||
+      placeholder.test(secret) ||
+      credentialPlaceholder(secret)
+    ) {
+      continue;
+    }
+    const userStart = match.index + match[0].indexOf(user);
+    const secretStart = match.index + match[0].lastIndexOf(secret);
+    emit({
+      start: userStart,
+      end: userStart + user.length,
+      ...personal('USERNAME', 'credential-pair', 0.9),
+    });
+    emit({
+      start: secretStart,
+      end: secretStart + secret.length,
+      type: 'SECRET',
+      category: 'credential',
+      rule: 'credential-pair',
+      confidence: 0.9,
+    });
+  }
+}
+
+// "my AMEX card: 1234 567890 12345": a labelled card keeps its label even when
+// the number is a documentation placeholder that fails Luhn.
+function detectLabelledCards(text, emit) {
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])(?:card|visa|mastercard|amex|discover|diners|jcb|unionpay|maestro|карта)(?:[ \t]+(?:no\.?|number|num|#))?[ \t]*(?:[:#][ \t]*)?((?:\d[ -]?){12,18}\d)(?![\p{N}]|[ -]\d)/giu,
+    emit,
+    personal('CREDIT_CARD', 'card-context', 0.9),
+    1
+  );
+}
+
+// "SSN 030 72 7381", "social security number: 003 06-8815": the label
+// carries spaced and mixed separators that the bare dashed form omits.
+function detectLabelledSsn(text, emit) {
+  collectMatches(
+    text,
+    /(?<![\p{L}\p{N}_])(?:SSN|SS#|social[ \t]+security(?:[ \t]+(?:number|no\.?|#))?)[ \t]*(?:(?:[:#]|is)[ \t]*)?(\d{3}[ .-]?\d{2}[ .-]?\d{4})(?![\p{N}]|[ .-]\d)/giu,
+    emit,
+    personal('SSN', 'ssn-label', 0.9),
+    1,
+    (value) => !/^(?:000|666|9\d\d)|^\d{3}\D?00|0000$/.test(value)
+  );
+}
+
+const states =
+  'AL|AK|AZ|AR|CA|CO|CT|DE|DC|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|PR';
+function detectPostalCodes(text, emit) {
+  // "ZIP 94105", "ZIPs 10001 and 30301", "zip code: 94105-1234".
+  for (const label of text.matchAll(
+    /(?<![\p{L}\p{N}_])(?:ZIP(?:[ \t]?codes?)?s?|postal[ \t]+codes?|PLZ|CEP)[ \t]*(?:[:#][ \t]*)?/giu
+  )) {
+    const start = label.index + label[0].length;
+    const list =
+      /^\d{5}(?:-\d{4})?(?:(?:[ \t]*,[ \t]*|[ \t]+(?:and|or|&)[ \t]+)\d{5}(?:-\d{4})?)*(?![\p{L}\p{N}_-])/u.exec(
+        text.slice(start, start + 200)
+      );
+    for (const code of list?.[0].matchAll(/\d{5}(?:-\d{4})?/g) ?? []) {
+      emit({
+        start: start + code.index,
+        end: start + code.index + code[0].length,
+        ...personal('POSTAL_CODE', 'postal-label', 0.9),
+      });
+    }
+  }
+  // "New York City, NY 10002": a state abbreviation anchors the ZIP code.
+  collectMatches(
+    text,
+    new RegExp(
+      `(?<=[\\p{L}.],)[ \\t]*(?:${states})[ \\t]+(\\d{5}(?:-\\d{4})?)(?![\\p{L}\\p{N}_-])`,
+      'gu'
+    ),
+    emit,
+    personal('POSTAL_CODE', 'state-zip', 0.85),
+    1
+  );
+}
+
+export function detectPersonalEvidence(text, emit) {
+  detectNames(text, emit);
+  detectContactForms(text, emit);
+  detectCredentialPairs(text, emit);
+  detectLabelledCards(text, emit);
+  detectLabelledSsn(text, emit);
+  detectPostalCodes(text, emit);
+}

package/src/personal.js

@@ -11,6 +11,12 @@ const details = (type, rule, confidence = 0.85) => ({
   category: 'personal',
   confidence,
 });
+// "John C Smith", "John A. Smith": a Latin-script middle initial. A bare "I"
+// is the pronoun ("Will I Go"), so it needs its period.
+const initial = (language) =>
+  ['ar', 'he', 'hi', 'th', 'ja', 'zh', 'ko'].includes(language)
+    ? ''
+    : '(?:[ \\t]+(?:\\p{Lu}\\.|(?!I[ \\t])\\p{Lu}(?=[ \\t])))?';
 const names = Object.entries(firstNames).map(([language, list]) => {
   const words = list.split(' ').map(escapePattern).join('|');
   const surname = ['ar', 'he', 'hi', 'th'].includes(language)
@@ -18,7 +24,7 @@ const names = Object.entries(firstNames).map(([language, list]) => {
     : "[\\p{Lu}][\\p{L}\\p{M}’'-]{1,32}";
   const pattern = ['ja', 'zh', 'ko'].includes(language)
     ? `(?:${words})[\\p{Script=Han}\\p{Script=Hiragana}\\p{Script=Katakana}\\p{Script=Hangul}]{1,4}`
-    : `(?:${words})[ \\t]+${surname}${language === 'vi' ? `(?:[ \\t]+${surname})?` : ''}`;
+    : `(?:${words})${initial(language)}[ \\t]+${surname}${language === 'vi' ? `(?:[ \\t]+${surname})?` : ''}`;
   return new RegExp(`(?<![\\p{L}\\p{M}])${pattern}(?![\\p{L}\\p{M}])`, 'gu');
 });
 
@@ -33,7 +39,7 @@ const ibanLengths = Object.fromEntries(
 const mentionedEmail =
   /^\/[\p{L}\p{N}_.!#$%&'*+?^`{|}~-]+@[\p{L}\p{N}][\p{L}\p{N}.-]{0,252}\.\p{L}{2,63}(?![\p{L}\p{N}_-])/u;
 export function iban(value) {
-  const compact = value.replace(/\s/g, '').toUpperCase();
+  const compact = value.replace(/[\s-]/g, '').toUpperCase();
   if (
     !/^[A-Z]{2}\d{2}[A-Z0-9]+$/.test(compact) ||
     compact.length !== ibanLengths[compact.slice(0, 2)]
@@ -53,7 +59,7 @@ export function iban(value) {
 
 function detectIban(text, emit) {
   for (const match of text.matchAll(
-    /(?<![A-Z0-9])[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){8,40}/g
+    /(?<![A-Z0-9])[A-Z]{2} ?\d{2}(?:[ -]?[A-Z0-9]){8,40}/g
   )) {
     const needed = ibanLengths[match[0].slice(0, 2)];
     if (!needed) {
@@ -62,7 +68,7 @@ function detectIban(text, emit) {
     let end = 0,
       count = 0;
     while (end < match[0].length && count < needed) {
-      if (match[0][end] !== ' ') {
+      if (!/[ -]/.test(match[0][end])) {
         count++;
       }
       end++;
@@ -180,6 +186,16 @@ function detectDocumentNames(text, emit) {
 }
 const russianFull =
   /(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}[А-ЯЁ][а-яё]{2,30}(?:вич|вна|ична)|(?<![\p{L}])(?:[А-ЯЁ][а-яё]{2,30}[ \t]+){2}(?:оглы|кызы)(?![\p{L}])/gu;
+// "My ITIN", "Our BIN": a possessive word or a short all-capital acronym
+// matches the Slavic surname shape but carries no name evidence.
+const possessiveWord =
+  /^(?:my|our|your|his|her|their|its|the|this|that|мой|моя|наш|наша|ваш|ваша)$/iu;
+function slavicPairEvidence(value) {
+  const words = value.split(/[ \t]+/);
+  return !words.some(
+    (word) => possessiveWord.test(word) || /^\p{Lu}{2,4}$/u.test(word)
+  );
+}
 export function detectPersonal(text, emit) {
   detectDocumentNames(text, emit);
   for (const pattern of [
@@ -194,7 +210,9 @@ export function detectPersonal(text, emit) {
       emit,
       details('PERSON', 'name-identity', 0.9),
       0,
-      (value) => !placeWord.test(value.split(/[ \t]+/).at(-1))
+      (value) =>
+        !placeWord.test(value.split(/[ \t]+/).at(-1)) &&
+        (pattern !== slavicPairs || slavicPairEvidence(value))
     );
   }
   for (const pattern of names) {
@@ -218,21 +236,35 @@ export function detectPersonal(text, emit) {
   );
   collectMatches(
     text,
-    /(?<![\d(])(?:[78][ -]?)?\(\d{3}\)[ -]?\d{3}[ -]\d{2}[ -]?\d{2}(?!\d)|(?<!\d)(?:\d{3}[.-]){2}\d{4}(?!\d)/g,
+    /(?<![\d(])(?:[78][ -]?)?\(\d{3}\)[ -]?\d{3}[ -]\d{2}[ -]?\d{2}(?!\d)|(?<![\d-])(?:(?<![\p{L}_.])1[.-])?(?:\d{3}[.-]){2}\d{4}(?!\d|-[\p{L}\p{N}])/gu,
     emit,
     details('PHONE', 'national-phone')
   );
   collectMatches(
     text,
-    /(?<![\p{L}\p{N}])\d{1,6}[A-Za-z]?[ \t]+(?:[\p{Lu}][\p{L}.'-]*[ \t]+){1,5}(?:Street|St|Road|Rd|Avenue|Ave|Lane|Ln|Drive|Dr|Boulevard|Blvd|Way|Straße|Strasse|Rue|Calle|Rua)\b(?:,[ \t]+(?:(?:[^ \t\r\n<>][^\r\n<>]{0,79}?[ \t]+)?(?:[A-Z]{1,2}\d[A-Z\d]?[ \t]+\d[A-Z]{2}|\d{5}(?:-\d{4})?)\b|(?:[\p{Lu}][\p{Ll}.'-]+[ \t]*){1,4}))?/gu,
+    /(?<![\p{L}\p{N}])\d{1,6}[A-Za-z]?[ \t]+(?:[\p{Lu}][\p{L}.'-]*[ \t]+){1,5}(?:Street|St|Road|Rd|Avenue|Ave|Lane|Ln|Drive|Dr|Boulevard|Blvd|Way|Terrace|Court|Ct|Place|Pl|Parkway|Pkwy|Circle|Highway|Hwy|Square|Crescent|Close|Straße|Strasse|Rue|Calle|Rua)\b(?:,?[ \t]+(?:Apt|Apartment|Suite|Ste|Unit|Floor|Fl|Room|Rm)\.?[ \t]*#?[A-Za-z0-9-]{1,6}\b|[ \t]+#[A-Za-z0-9-]{1,6}\b)?(?:,[ \t]+(?:(?:[^ \t\r\n<>][^\r\n<>]{0,79}?[ \t]+)?(?:[A-Z]{1,2}\d[A-Z\d]?[ \t]+\d[A-Z]{2}|\d{5}(?:-\d{4})?)\b|(?:[\p{Lu}][\p{Ll}.'-]+[ \t]*){1,4}))?/gu,
     emit,
     details('ADDRESS', 'postal-address')
   );
+  // "Mr. O'Brien", "Dr Jane Smith": a courtesy title is name evidence;
+  // after a number, "30 Ms Retry" is a millisecond unit.
   collectMatches(
     text,
-    /(?<![\da-fA-F:.-])(?:[\da-fA-F]{2}[:-]){5}[\da-fA-F]{2}(?![\da-fA-F:.-])/g,
+    /(?<![\p{L}\p{N}_])(?:Mr|Mrs|Ms|Mx|Miss|Dr|Prof)(?<!\p{N}[ \t]*\p{L}+)\.?[ \t]+(\p{Lu}(?:['’]\p{Lu})?[\p{Ll}\p{M}]+(?:-\p{Lu}[\p{Ll}\p{M}]+)?(?:[ \t]+\p{Lu}(?:['’]\p{Lu})?[\p{Ll}\p{M}]+(?:-\p{Lu}[\p{Ll}\p{M}]+)?)?)(?![\p{L}\p{M}])/gu,
     emit,
-    details('MAC_ADDRESS', 'mac')
+    details('PERSON', 'honorific-name'),
+    1,
+    (value) => !placeWord.test(value.split(/[ \t]+/).at(-1))
+  );
+  collectMatches(
+    text,
+    // Colon, dash or Cisco dotted notation with one separator; the broadcast
+    // and all-zero addresses identify no device.
+    /(?<![\da-fA-F:.-])(?:[\da-fA-F]{2}([:-])(?:[\da-fA-F]{2}\1){4}[\da-fA-F]{2}|[\da-fA-F]{4}\.[\da-fA-F]{4}\.[\da-fA-F]{4})(?![\da-fA-F:-]|\.[\da-fA-F])/g,
+    emit,
+    details('MAC_ADDRESS', 'mac'),
+    0,
+    (value) => !/^(?:[fF]{2}[:.-]?|0{2}[:.-]?)+$/.test(value.replace(/\./g, ''))
   );
   collectMatches(
     text,
@@ -249,7 +281,7 @@ export function detectPersonal(text, emit) {
   );
   collectMatches(
     text,
-    /(?:\/Users\/|\/home\/|[A-Za-z]:\\Users\\)([^/\\\s"'<>[\]]+)/g,
+    /(?:\/Users\/|\/home\/|[A-Za-z]:\\(?:Users|Documents and Settings|(?:Windows|WINNT)\\Profiles)\\)([^/\\\s"'<>[\]]+)/g,
     emit,
     details('USERNAME', 'home-path', 0.95),
     1,

package/src/projection.js

@@ -21,12 +21,91 @@ function asciiDigits(value) {
   });
 }
 
+// Invisible format characters and NUL (UTF-16 text read as bytes) never
+// carry meaning inside a token, so they are removed from the detection view.
+const invisible =
+  /^[\0\u00ad\u061c\u180e\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]$/u;
+// HTML formatting tags split tokens (`SECRET_</span><span>REST`). Inline tags
+// are dropped and block tags become line breaks. Only presentation attributes
+// are allowed, so other attribute values (URLs, data-*) stay in the view.
+const inlineTags =
+  'abbr|b|cite|code|del|em|font|i|ins|kbd|mark|q|s|samp|small|span|strong|sub|sup|tt|u|var';
+const blockTags =
+  'blockquote|br|dd|div|dt|h[1-6]|hr|li|ol|p|pre|table|tbody|td|th|thead|tr|ul';
+const htmlTag = String.raw`<\/?(?:${inlineTags}|${blockTags})(?:\s+(?:class|dir|lang|style)="[^"<>=]*")*\s*\/?>`;
+const inlineTag = new RegExp(String.raw`^<\/?(?:${inlineTags})[\s/>]`);
+// TruffleHog-style escape runs: `U+0074 U+006F ...` and CSS `\74 \6f ...`.
+// Runs need four or more units so prose such as "U+200B" stays untouched.
+// The space after the last unit stays in the view as a word boundary.
+const codeRun = String.raw`(?:U\+[\da-fA-F]{4,6} ?){3,}U\+[\da-fA-F]{4,6}`;
+const cssRun = String.raw`(?:\\[\da-fA-F]{1,6} ){3,}\\[\da-fA-F]{1,6}`;
+const projectionTrigger = new RegExp(
+  String.raw`\x1b\[|\0|\\u[\da-fA-F{]|\\x\{[\da-fA-F]|\\U[\da-fA-F]{8}|%u[\da-fA-F]{4}|U\+[\da-fA-F]{4}|\\[\da-fA-F]{1,6} \\|[\u00ad\u061c\u180e\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]|${htmlTag}`,
+  'u'
+);
+const projectionUnits = new RegExp(
+  String.raw`\x1b\[[0-?]*[ -/]*[@-~]|\\u[\da-fA-F]{4}|\\[ux]\{[\da-fA-F]{1,6}\}|\\U[\da-fA-F]{8}|%u[\da-fA-F]{4}|${codeRun}|${cssRun}|${htmlTag}|[\s\S]\p{M}*`,
+  'gu'
+);
+
+function decodeEscape(raw) {
+  const escape = /^(?:\\[ux]\{?|\\U|%u)([\da-fA-F]+)\}?$/.exec(raw);
+  if (!escape) {
+    return raw;
+  }
+  const code = parseInt(escape[1], 16);
+  // `\uXXXX` keeps UTF-16 surrogate halves; wider forms are code points.
+  return raw.length === 6 && raw[0] === '\\'
+    ? String.fromCharCode(code)
+    : code <= 0x10ffff
+      ? String.fromCodePoint(code)
+      : raw;
+}
+
+// Every character decoded from an escape run maps to the whole run. Redacting
+// only some units would leave a short run, such as a lone closing quote, that
+// the next view no longer decodes, so a finding inside a run covers all of it.
+function appendRun(writer, raw, offset) {
+  const unit = raw.startsWith('U+')
+    ? /U\+([\da-fA-F]{4,6}) ?/g
+    : /\\([\da-fA-F]{1,6}) ?/g;
+  const end = offset + raw.length;
+  for (const match of raw.matchAll(unit)) {
+    const code = parseInt(match[1], 16);
+    const value = code <= 0x10ffff ? String.fromCodePoint(code) : match[0];
+    if (!invisible.test(value)) {
+      writer.append(value, offset, end);
+    }
+  }
+}
+
+function projectUnit(writer, raw, offset) {
+  const end = offset + raw.length;
+  if (raw.startsWith('\x1b[') || invisible.test(raw)) {
+    return true;
+  }
+  if (raw[0] === '<' && raw.length > 2 && raw.endsWith('>')) {
+    if (!inlineTag.test(raw)) {
+      writer.append('\n', offset, end);
+    }
+    return true;
+  }
+  if (raw.length > 8 && /^(?:U\+|\\[\da-fA-F])/.test(raw)) {
+    appendRun(writer, raw, offset);
+    return true;
+  }
+  const decoded = decodeEscape(raw);
+  if (invisible.test(decoded)) {
+    return true;
+  }
+  const value = asciiDigits(decoded.normalize('NFKC'));
+  writer.append(value, offset, end);
+  return value !== raw;
+}
+
 export function projectText(input) {
   if (
-    // eslint-disable-next-line no-control-regex -- ANSI escape sequences are removed from the detection view.
-    !/\x1b\[|\\u[\da-fA-F]{4}|[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]/u.test(
-      input
-    ) &&
+    !projectionTrigger.test(input) &&
     // ASCII already has normalized letters and digits. Avoid constructing a
     // second multi-megabyte string just to check its decimal characters.
     // eslint-disable-next-line no-control-regex -- Check the ASCII range.
@@ -37,29 +116,9 @@ export function projectText(input) {
     return { text: input, changed: false };
   }
   const writer = projectionWriter(input.length);
-  // eslint-disable-next-line no-control-regex -- Tokenize ANSI sequences with their original offsets.
-  const units = /\x1b\[[0-?]*[ -/]*[@-~]|\\u[\da-fA-F]{4}|[\s\S]\p{M}*/gu;
   let changed = false;
-  for (const match of input.matchAll(units)) {
-    const raw = match[0];
-    let value = raw;
-    if (
-      raw.startsWith('\x1b[') ||
-      /^[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]$/u.test(raw)
-    ) {
-      changed = true;
-      continue;
-    }
-    if (/^\\u[\da-fA-F]{4}$/.test(raw)) {
-      value = String.fromCharCode(parseInt(raw.slice(2), 16));
-    }
-    if (/^[\u200b-\u200f\u202a-\u202e\u2060-\u206f\ufeff]$/u.test(value)) {
-      changed = true;
-      continue;
-    }
-    value = asciiDigits(value.normalize('NFKC'));
-    changed ||= value !== raw;
-    writer.append(value, match.index, match.index + raw.length);
+  for (const match of input.matchAll(projectionUnits)) {
+    changed = projectUnit(writer, match[0], match.index) || changed;
   }
   return { ...writer.finish(), changed };
 }