@link-foundation/sensitive-data-sanitizer 0.21.0 → 0.21.1

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.21.1
+
+### Patch Changes
+
+- e0e05a4: Cover 34 missing strict personal-attribute terms, preserve their typed placeholders over generic prose names, cache caller vocabulary and cue sets, avoid unnecessary placeholder scans and JSON parsing, and preserve exact public technology names in Cyrillic prose.
+
 ## 0.21.0
 
 ### Minor Changes

package/docs/API.md

@@ -750,7 +750,10 @@ Cues ignore case and support bounded Russian noun/adjective forms; personal fiel
 use normalized field names. Capitalized proper values are required after prose
 cues. Inputs allow at most 256 nonempty strings of 128 code units per list, with
 no line breaks/NUL. Unknown lists and malformed replacement records fail with
-`ERR_CONFIG`. Compilation uses a bounded 64-configuration cache.
+`ERR_CONFIG`. Compilation uses a bounded 64-configuration cache and a weak
+object-identity cache. Treat a vocabulary object as immutable after first use;
+pass a new object to change its lists. Repeated lookups reuse compiled patterns
+and a cue-word Set without normalizing or serializing the lists again.
 
 `personalAttributes: ['вепс', 'старовер']` adds special-category stems to the
 `strict` rule (it has no effect without `strict: true`). Each word matches as a
@@ -759,6 +762,18 @@ words of a multi-word entry may be separated by spaces or tabs. Matches are
 reported as `PERSONAL_ATTRIBUTE` with the `strict-attribute` rule. The default
 list is empty.
 
+Typed special-category attributes take precedence over generic `ProperName`
+guesses, including capitalized caller stems in Cyrillic prose. Strict coverage
+includes the 34 additional nationality, religion, health, criminal-record,
+political and union terms reported in issue 144, even as standalone words.
+
+With default `publicKnowledge`, exact public technology spellings such as Python,
+SQL, PostgreSQL, Docker, Kubernetes, React, AWS and Azure remain readable when
+the capitalized prose detector flags them. The offline catalog includes source
+URLs in `src/public-technologies.js`. `publicKnowledge: false` disables this
+exception; known private values and credentials still redact, and an adjacent
+personal name never inherits a technology's exemption.
+
 ## Callback transformations and stages
 
 ```js

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-foundation/sensitive-data-sanitizer",
-  "version": "0.21.0",
+  "version": "0.21.1",
   "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
   "type": "module",
   "main": "src/index.js",

package/src/placeholder-types.js

@@ -129,6 +129,9 @@ export function namespacePattern(format = defaultFormat, extraTypes = []) {
 // Tokens of the namespace with the types each spelling can stand for.
 export function placeholderTokens(text, pattern) {
   const tokens = [];
+  if (!/[1-9]/u.test(text)) {
+    return tokens;
+  }
   for (const match of text.matchAll(new RegExp(pattern.source, 'gu'))) {
     const [part, number] = pattern.typedFirst
       ? [match[1], match[2]]
@@ -143,13 +146,20 @@ export function placeholderTokens(text, pattern) {
   return tokens;
 }
 export function containsPlaceholder(text, pattern) {
-  return new RegExp(pattern.source, 'u').test(text);
+  return /[1-9]/u.test(text) && new RegExp(pattern.source, 'u').test(text);
 }
 // Tokens spelled with JSON escapes inside string literals, such as
 // `Person-1`, decoded so that they cannot hide from the namespace check.
 export function escapedPlaceholderTokens(text, pattern) {
   const tokens = [];
+  // Without an escape every literal has already been checked in the raw text.
+  if (!text.includes('\\')) {
+    return tokens;
+  }
   for (const literal of jsonStringLiterals(text)) {
+    if (!literal.includes('\\')) {
+      continue;
+    }
     try {
       tokens.push(...placeholderTokens(JSON.parse(literal), pattern));
     } catch {

package/src/placeholders.js

@@ -349,10 +349,25 @@ function callPattern(findings, options) {
   );
 }
 const keepsExisting = (options) => transforms(options)[0]?.existing === 'keep';
+// Default tokens always contain a hyphen followed by a positive number.
+// Escapes may conceal either character, and custom formats need the full scan.
+function mayContainPlaceholder(text, options, escaped = false) {
+  const transform = transforms(options)[0];
+  const format =
+    options[placeholderState]?.map.format ??
+    transform?.format ??
+    transform?.map?.format ??
+    '{Type}-{n}';
+  return (
+    format !== '{Type}-{n}' ||
+    /-[1-9]/u.test(text) ||
+    (escaped && text.includes('\\'))
+  );
+}
 // With `existing: 'keep'` the tokens of an earlier pass stay as they are, so
 // findings made only of those tokens are not replaced again.
 export function keptPlaceholderSpans(text, findings, options) {
-  if (!keepsExisting(options)) {
+  if (!keepsExisting(options) || !mayContainPlaceholder(text, options)) {
     return [];
   }
   return placeholderTokens(text, callPattern(findings, options));
@@ -399,8 +414,10 @@ export function preparePlaceholderText(text, findings, options) {
   if (!state || !hasPlaceholders(options)) {
     return;
   }
-  const pattern = callPattern(findings, options);
-  if (state.existing === 'keep') {
+  const pattern = mayContainPlaceholder(text, options, true)
+    ? callPattern(findings, options)
+    : undefined;
+  if (pattern && state.existing === 'keep') {
     // New values are numbered after the highest kept number of their type.
     for (const token of [
       ...placeholderTokens(text, pattern),
@@ -413,7 +430,7 @@ export function preparePlaceholderText(text, findings, options) {
         );
       }
     }
-  } else if (namespaceCollision(text, pattern)) {
+  } else if (pattern && namespaceCollision(text, pattern)) {
     throw failure('ERR_PLACEHOLDER_COLLISION');
   }
   state.people = [

package/src/prose-words.js

@@ -74,10 +74,6 @@ function sentenceStart(text, start) {
 function nextWordLowercase(text, end) {
   return /^[ \t]+\p{Ll}/u.test(text.slice(end, end + 4));
 }
-function vocabularyCues(options) {
-  const lists = compiledVocabulary(options?.vocabulary)?.lists;
-  return lists ? [...lists.roles, ...lists.personalLabels] : [];
-}
 const defaultCues = new Set(
   [...defaultVocabulary.roles, ...defaultVocabulary.personalLabels].map(fold)
 );
@@ -89,7 +85,7 @@ export function isProseCue(word, options) {
   return (
     (cues.has(lower) ||
       defaultCues.has(lower) ||
-      vocabularyCues(options).includes(lower)) &&
+      compiledVocabulary(options?.vocabulary)?.cues.has(lower)) &&
     !isNameForm(word)
   );
 }

package/src/public-technologies.js

@@ -0,0 +1,32 @@
+// Exact spellings only: a technology token never exempts a surrounding name,
+// caller-specified private value, credential, URL or identifier.
+export const publicTechnologies = [
+  ['Python', 'https://www.python.org/'],
+  ['SQL', 'https://www.iso.org/standard/76583.html'],
+  ['JavaScript', 'https://developer.mozilla.org/en-US/docs/Web/JavaScript'],
+  ['TypeScript', 'https://www.typescriptlang.org/'],
+  ['Java', 'https://dev.java/'],
+  ['C++', 'https://isocpp.org/'],
+  ['C#', 'https://learn.microsoft.com/en-us/dotnet/csharp/'],
+  ['PHP', 'https://www.php.net/'],
+  ['Kotlin', 'https://kotlinlang.org/'],
+  ['PostgreSQL', 'https://www.postgresql.org/'],
+  ['MySQL', 'https://www.mysql.com/'],
+  ['SQLite', 'https://sqlite.org/'],
+  ['MongoDB', 'https://www.mongodb.com/'],
+  ['Redis', 'https://redis.io/'],
+  ['Elasticsearch', 'https://www.elastic.co/elasticsearch'],
+  ['Docker', 'https://www.docker.com/'],
+  ['Kubernetes', 'https://kubernetes.io/'],
+  ['React', 'https://react.dev/'],
+  ['Vue', 'https://vuejs.org/'],
+  ['Angular', 'https://angular.dev/'],
+  ['Django', 'https://www.djangoproject.com/'],
+  ['Flask', 'https://flask.palletsprojects.com/'],
+  ['Node.js', 'https://nodejs.org/'],
+  ['TensorFlow', 'https://www.tensorflow.org/'],
+  ['PyTorch', 'https://pytorch.org/'],
+  ['AWS', 'https://aws.amazon.com/'],
+  ['Azure', 'https://azure.microsoft.com/'],
+  ['GCP', 'https://cloud.google.com/'],
+].map(([value, source]) => ({ value, source }));

package/src/public.js

@@ -1,9 +1,12 @@
 import { isIP } from 'node:net';
+import { publicTechnologies } from './public-technologies.js';
 
 // Exact public knowledge and narrowly scoped role-mail heuristics. A role name
 // alone is never an exemption on an unknown/personal domain. Credentials and
 // caller-specified private values always outrank these automatic exceptions.
 export const publicKnowledge = {
+  PROPER_NAME: publicTechnologies,
+  URL: publicTechnologies.filter((entry) => entry.value === 'Node.js'),
   PERSON: [
     ['Albert Einstein', 'Q937'],
     ['Marie Curie', 'Q7186'],

package/src/sanitizer.js

@@ -870,11 +870,17 @@ function preferredType(current, candidate, options) {
   if (options.transformations?.[candidate] || candidate === 'PASSPORT_MRZ') {
     return true;
   }
+  return preferredSpecificType(current, candidate);
+}
+function preferredSpecificType(current, candidate) {
   const broad = (type) =>
     ['ID', 'PASSPORT'].includes(type) ||
     (/PASSPORT/.test(type) &&
       !['PASSPORT_MRZ', 'PASSPORT_NUMBER'].includes(type));
-  return broad(current) && isIdentityType(candidate) && !broad(candidate);
+  return (
+    (current === 'PROPER_NAME' && candidate === 'PERSONAL_ATTRIBUTE') ||
+    (broad(current) && isIdentityType(candidate) && !broad(candidate))
+  );
 }
 export function redactResult(
   text,

package/src/strict-attributes.js

@@ -18,6 +18,11 @@ const softNoun = '(?:а|у|ом|е|и|ей|ам|ами|ах)?';
 const ecNoun = '(?:ец|ц(?:а|у|ем|е|ы|ев|ам|ами|ах))';
 const femaleNoun = '(?:а|и|е|у|ой)';
 const nationalities = [
+  `кабардин${ecNoun}|кабардинк${femaleNoun}`,
+  `казашк${femaleNoun}`,
+  `мордвин${hardNoun}|мордовк${femaleNoun}`,
+  `удмурт${hardNoun}|удмуртк${femaleNoun}`,
+  `молдаванин(?:а|у|ом|е)?|молдаван${hardNoun}|молдаванк${femaleNoun}`,
   `узбек${softNoun}|узбечк${femaleNoun}`,
   `таджик${softNoun}|таджичк${femaleNoun}`,
   `(?:киргиз|кыргыз)${hardNoun}|(?:киргиз|кыргыз)к${femaleNoun}`,
@@ -41,6 +46,12 @@ const nationalities = [
   `уйгур${hardNoun}|уйгурк${femaleNoun}`,
 ].join('|');
 const believerNoun = `(?:а|у|ом|е|ы|ов|ам|ами|ах|к${femaleNoun}|кой)?`;
+// These terms are sensitive even without an adjacent personal cue. Keep
+// bounded endings and whole-word boundaries, as for the existing categories.
+const followUpVocabulary = new RegExp(
+  `(?<![\\p{L}\\p{N}_])(?:иудаизм${hardNoun}|старовер${believerNoun}|эпилептик${softNoun}|паралич${softNoun}|наркомани[\\p{L}]{1,4}|профсоюз${hardNoun}|коммунист${believerNoun}|uzbek|tajik|chechen|romani|gypsy|latino|latina|baptist|aids|bipolar|adhd|wheelchair|lgbt|convicted|felony|trade[ \\t]+union)(?![\\p{L}\\p{N}_])`,
+  'giu'
+);
 const specialVocabulary = new RegExp(
   `(?<![\\p{L}\\p{N}_])(?:${nationalities}|(?:протестант|баптист|адвентист|индуист|кришнаит|иеговист|пятидесятник)${believerNoun}|лютеран(?:ин(?:а|у|ом|е)?|к${femaleNoun}|кой|е|ам|ами|ах)|старообряд${ecNoun}|старообрядк${femaleNoun}|свидетел[\\p{L}]{1,3}[ \\t]+иеговы|инсульт[\\p{L}]{0,3}|инфаркт[\\p{L}]{0,3}|шизофрени[\\p{L}]{1,4}|аутизм[\\p{L}]{0,3}|аутист(?:ка|ки|ке|ку|кой|а|у|ом|ы|ов)?|слабослышащ[\\p{L}]{1,4}|слабовидящ[\\p{L}]{1,4}|незряч[\\p{L}]{1,4}|дцп|колясочни(?:к${hardNoun}|ц${femaleNoun})|(?:передвига[\\p{L}]{1,5}|инвалидн[\\p{L}]{2,4})[ \\t]+(?:на[ \\t]+)?коляск[\\p{L}]{1,3}|болезн[\\p{L}]{1,3}[ \\t]+(?:паркинсона|альцгеймера)|деменци[\\p{L}]{1,3}|алкоголизм[\\p{L}]{0,2}|наркозависим[\\p{L}]{0,4}|биполярн[\\p{L}]{2,4}[ \\t]+расстройств[\\p{L}]{0,3}|(?:би|транс|пан|а)сексуал[\\p{L}]{0,6}|трансгендер[\\p{L}]{0,6}|(?:сидел[аи]?|отсидел[аи]?|осужд[её]н[аы]?|привлекал(?:ся|ась|ись)|отбывал[аи]?[ \\t]+срок)[ \\t]+(?:за|по)[ \\t]+(?:краж|грабе|разбо|убийств|мошенничеств|наркотик|хулиганств|изнасиловани|взятк|стать)[\\p{L}]{0,4}(?:[ \\t]+\\d{1,3}(?:\\.\\d)?)?|(?:сидел[аи]?|отсидел[аи]?)[ \\t]+(?:в[ \\t]+)?(?:тюрьм[\\p{L}]{1,2}|колони[\\p{L}]{1,2}|срок)|бывш[\\p{L}]{2,3}[ \\t]+заключ[её]нн[\\p{L}]{2,3}|член[\\p{L}]{0,3}[ \\t]+профсоюз[\\p{L}]{0,3}|состо(?:ит|ял[аи]?|ю|ят)[ \\t]+в[ \\t]+профсоюзе|профсоюзн(?:ый|ая|ого|ой)[ \\t]+(?:активист|деятел|лидер)[\\p{L}]{0,3}|autism|autistic|schizophreni[ac]|transgender|transsexual|pansexual|asexual|wheelchair[ \\t]+users?|uses[ \\t]+a[ \\t]+wheelchair|in[ \\t]+a[ \\t]+wheelchair|(?:hearing|visually)[ \\t-]+impaired|had[ \\t]+a[ \\t]+(?:stroke|heart[ \\t]+attack)|dementia|alzheimer'?s|parkinson'?s|bipolar[ \\t]+disorder|alcoholism|trade[ \\t]+union[ \\t]+member(?:ship)?|member[ \\t]+of[ \\t]+(?:a|the)[ \\t]+(?:trade|labou?r)[ \\t]+union|convicted[ \\t]+(?:of|for)[ \\t]+[\\p{L}]{3,32}|served[ \\t]+(?:time[ \\t]+(?:in|for)|a[ \\t]+(?:prison[ \\t]+)?sentence)|protestant|lutheran|mormon|hindu|sikh|jehovah'?s[ \\t]+witness)(?![\\p{L}\\p{N}_])`,
   'giu'
@@ -49,7 +60,7 @@ const categoryCue =
   /(?<![\p{L}\p{N}_])(?:по[ \t]+(?:национальности|происхождению)|национальность|этничность|вероисповедание|религия|диагноз|исповеду(?:ет|ют|ю|ешь)|боле(?:ет|ют|ю|ешь)|страда(?:ет|ют|ю|ешь)[ \t]+от|на[ \t]+уч[её]те[ \t]+у|diagnosed[ \t]+with|suffers[ \t]+from|nationality|ethnicity|religion|diagnosis)(?:["']?[ \t]*[:=][ \t]*|[ \t]+)["']?([\p{L}][\p{L}\p{M}-]{0,63})(?![\p{L}\p{M}\p{N}_])/giu;
 // Words that are also languages, scripts or styles need a personal context.
 const ambiguous =
-  /(?<![\p{L}\p{N}_])(?:disabled|single|man|men|russian|asian|uzbek|tajik|chechen|azerbaijani|korean|chinese|georgian|kazakh|kyrgyz|turkmen|uyghur|roma|baptist|union[ \t]+member)(?![\p{L}\p{N}_])/giu;
+  /(?<![\p{L}\p{N}_])(?:disabled|single|man|men|russian|asian|azerbaijani|korean|chinese|georgian|kazakh|kyrgyz|turkmen|uyghur|roma|union[ \t]+member)(?![\p{L}\p{N}_])/giu;
 
 function personalContext(text, match) {
   const prefix = text.slice(Math.max(0, match.index - 128), match.index);
@@ -97,6 +108,7 @@ export function detectStrictAttributes(text, emit, options) {
     additionalAttributes,
     categoryForms,
     specialVocabulary,
+    followUpVocabulary,
     compiledVocabulary(options.vocabulary)?.attributes,
   ]) {
     if (pattern) {

package/src/vocabulary.js

@@ -55,6 +55,7 @@ export const defaultVocabulary = Object.freeze(
   )
 );
 const cache = new Map();
+const identities = new WeakMap();
 const boundary = '[\\p{L}\\p{N}_]';
 const name =
   '[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}(?:[ \\t]+[\\p{Lu}][\\p{L}\\p{M}.-]{1,47}){0,3}';
@@ -140,10 +141,16 @@ export function compiledVocabulary(vocabulary) {
   if (vocabulary === undefined) {
     return undefined;
   }
+  const existing = identities.get(vocabulary);
+  if (existing) {
+    return existing;
+  }
   const lists = normalizedLists(vocabulary);
   const key = JSON.stringify(lists);
   if (cache.has(key)) {
-    return cache.get(key);
+    const result = cache.get(key);
+    identities.set(vocabulary, result);
+    return result;
   }
   const alternatives = (values) =>
     values.length
@@ -184,11 +191,15 @@ export function compiledVocabulary(vocabulary) {
     'giu'
   );
   result.fields = new Set(lists.personalFields.map(normalizeFieldName));
+  result.cues = new Set(
+    [...lists.roles, ...lists.personalLabels].map((s) => s.replaceAll('ё', 'е'))
+  );
   result.attributes = attributePattern(lists.personalAttributes);
   if (cache.size >= 64) {
     cache.clear();
   }
   cache.set(key, result);
+  identities.set(vocabulary, result);
   return result;
 }
 export function vocabularyField(field, options) {