@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0

package/src/sanitizer.js

@@ -1,14 +1,12 @@
-import { generatedValues } from './fake.js';
+import { generatedValues } from './fake-memory.js';
+import { feature, withFeature } from './features.js';
 import { isCompiledKnown, projectedKnown } from './compiled-known.js';
 import { detectSync, validateDetectors } from './sync-detectors.js';
 import { validatePolicyControls, builtInEnabled } from './policy.js';
 import { compiledVocabulary } from './vocabulary.js';
-import { validateProperNames } from './proper-names.js';
-import { prepareFakeText } from './fake-entities.js';
 import { alignEscapes } from './escape-spans.js';
 import { actionable } from './confidence.js';
 import { propagate, confirmationOptions } from './propagation.js';
-import { fakeDocumentSpans } from './specimens.js';
 import {
   failure,
   internalFailure,
@@ -26,8 +24,12 @@ import {
   hasCustomTransforms,
   renderCallbackAsync,
 } from './transforms.js';
-import { fakeIdentityValue, isIdentityType } from './identity.js';
+import { isIdentityType } from './identity-types.js';
 import { rejectRestoreMap } from './restore-map.js';
+import { allowListed, validateAllowList } from './allow-list.js';
+import { templatedCredential } from './credentials.js';
+import { exemptFindings, validateExempt } from './exempt.js';
+import { builtInRules } from './rule-catalog.js';
 import {
   initializePlaceholders,
   placeholderKnown,
@@ -36,6 +38,8 @@ import {
   hasPlaceholders,
   placeholderValues,
   placeholderNameFindings,
+  keptPlaceholderSpans,
+  keptPlaceholderFinding,
   coverAdjacentNames,
   nestedPlaceholderOptions,
 } from './placeholders.js';
@@ -99,6 +103,10 @@ const optionNames = new Set([
   'strict',
   'exactFields',
   'allowedFields',
+  'minConfidenceByType',
+  'allowList',
+  'inlineAllow',
+  'exempt',
 ]);
 
 export function applyProfile(options = {}) {
@@ -216,7 +224,9 @@ export function validateOptions(options = {}) {
     }
   }
   validateConfidence(options);
-  validateProperNames(options);
+  if (options.properNames !== undefined || options.scripts !== undefined) {
+    feature('validateProperNames')(options);
+  }
   validatePolicy(options);
   if (
     options.verifyPublic !== undefined &&
@@ -224,7 +234,9 @@ export function validateOptions(options = {}) {
   ) {
     throw failure('ERR_CONFIG');
   }
+  validateExempt(options.exempt);
   validateCollections(options);
+  validateAllowList(options);
   validateDetectors(options.detectors);
   validatePolicyControls(options);
   compiledVocabulary(options.vocabulary);
@@ -255,6 +267,22 @@ function validateConfidence(options) {
   ) {
     throw failure('ERR_CONFIG');
   }
+  const byType = options.minConfidenceByType;
+  if (
+    byType !== undefined &&
+    (!byType ||
+      typeof byType !== 'object' ||
+      Array.isArray(byType) ||
+      !Object.entries(byType).every(
+        ([type, value]) =>
+          validType(type) &&
+          typeof value === 'number' &&
+          value >= 0 &&
+          value <= 1
+      ))
+  ) {
+    throw failure('ERR_CONFIG');
+  }
 }
 
 function validatePolicy(options) {
@@ -306,6 +334,9 @@ export function sanitizeFinding(finding, length) {
     rule: finding.rule,
     confidence: validConfidence(finding),
     likelihood: finding.likelihood ?? likelihood(validConfidence(finding)),
+    ...(finding.type === 'JSON_VALUE' && validType(finding.valueType)
+      ? { valueType: finding.valueType }
+      : {}),
     ...(finding.kept === 'fake' ? { kept: 'fake' } : {}),
   };
 }
@@ -426,7 +457,14 @@ function locations(text, findings) {
 
 export function inspect(text, options = {}) {
   try {
-    return inspectText(text, options);
+    const exempt = options?.exempt;
+    if (!exempt) {
+      return inspectText(text, options);
+    }
+    // Nested views (structured values, decoded runs) are screened once, at
+    // the offsets of the text the caller sees.
+    const findings = inspectText(text, { ...options, exempt: undefined });
+    return exemptFindings(text, findings, exempt);
   } catch (error) {
     throw internalFailure(error);
   }
@@ -518,12 +556,34 @@ function inspectText(text, options = {}) {
   return reviewFindings(text, findings, options);
 }
 
+// Allow-listed values and built-in credential matches whose value is only a
+// template or mask (`${PASSWORD}`, `<token>`, `****`) are not findings.
+function policyKept(text, finding, options) {
+  if (allowListed(text, finding, options)) {
+    return false;
+  }
+  if (builtInRule(finding) && templatedCredential(text, finding)) {
+    options.debug?.({ event: 'template-value', rule: finding.rule });
+    return false;
+  }
+  return true;
+}
+
+// Custom detectors keep their findings: they may flag placeholder shapes on
+// purpose, and the residual check must still see them.
+const builtInRuleIds = new Set(builtInRules);
+const builtInRule = (finding) =>
+  builtInRuleIds.has(finding.rule) ||
+  finding.rule === 'secretlint' ||
+  finding.rule.startsWith('gitleaks/');
+
 // Reapply policy to the native/required-engine union without rescanning text.
 export function reviewFindings(text, findings, options) {
   findings = placeholderNameFindings(text, findings, options);
   if (findings.length > (options.maxFindings ?? 100000)) {
     throw failure('ERR_LIMIT');
   }
+  findings = findings.filter((f) => policyKept(text, f, options));
   const unique = new Map();
   const exempt = findings.filter((f) =>
     allowed(text, f, options.publicEntities ?? [])
@@ -565,9 +625,9 @@ function coveredByPublic(finding, exempt) {
 }
 
 export function inspectFindingsOnly(text, findings, options) {
-  const all = [...findings, ...(options.findings ?? [])].map((f) =>
-    sanitizeFinding(f, text.length)
-  );
+  const all = [...findings, ...(options.findings ?? [])]
+    .map((f) => sanitizeFinding(f, text.length))
+    .filter((f) => policyKept(text, f, options));
   if (all.length > (options.maxFindings ?? 100000)) {
     throw failure('ERR_LIMIT');
   }
@@ -590,14 +650,41 @@ export function inspectFindingsOnly(text, findings, options) {
   );
 }
 
+// A whole JSON value that is already a token of this run, `"Sensitive-1"`.
+function generatedJsonValue(text, finding, options) {
+  const raw = text.slice(finding.start, finding.end);
+  return (
+    raw.startsWith('"') &&
+    placeholderValues(options).some(
+      (record) => JSON.stringify(record.value) === raw
+    )
+  );
+}
 function selectedFindings(findings, credentialsOnly) {
   return credentialsOnly
     ? findings.filter((finding) => finding.category === 'credential')
     : findings;
 }
 
+// Findings over the tokens of an earlier pass, recorded in `tokenKept`.
+function withKeptPlaceholders(text, findings, options, tokenKept) {
+  const tokens = keptPlaceholderSpans(text, findings, options);
+  if (!tokens.length) {
+    return findings;
+  }
+  return findings.map((f) => {
+    const next = keptPlaceholderFinding(text, f, tokens);
+    if (next.kept === 'fake' && next !== f) {
+      tokenKept.add(next);
+    }
+    return next;
+  });
+}
+
 export function markFake(text, findings, options) {
   findings = selectedFindings(findings, options.credentialsOnly);
+  const tokenKept = new Set();
+  findings = withKeptPlaceholders(text, findings, options, tokenKept);
   const generated = [
     ...generatedValues(options),
     ...placeholderValues(options),
@@ -607,9 +694,12 @@ export function markFake(text, findings, options) {
     !options.fakeValues?.length &&
     !generated.length
   ) {
-    return findings.map((f) => ({ ...f, kept: undefined }));
+    return findings.map((f) => ({
+      ...f,
+      kept: tokenKept.has(f) ? 'fake' : undefined,
+    }));
   }
-  const documents = fakeDocumentSpans(text, options);
+  const documents = withFeature('fakeDocumentSpans', [], text, options);
   const trusted = [];
   for (const record of generated) {
     for (const value of new Set([
@@ -644,7 +734,7 @@ export function markFake(text, findings, options) {
   const candidates = findings.filter(
     (f) =>
       f.category === 'personal' &&
-      fakeIdentityValue(text.slice(f.start, f.end), f.type, options)
+      feature('fakeIdentityValue')(text.slice(f.start, f.end), f.type, options)
   );
   // Tolerant MRZ matches can cross a line boundary. An automatic specimen
   // exemption must never cover an overlapping real identity value. Exact
@@ -668,8 +758,13 @@ export function markFake(text, findings, options) {
     ),
   ];
   return findings.map((f) => {
+    if (tokenKept.has(f)) {
+      return f;
+    }
     if (f.type === 'JSON_VALUE') {
-      return { ...f, kept: undefined };
+      return generatedJsonValue(text, f, options)
+        ? { ...f, kept: 'fake' }
+        : { ...f, kept: undefined };
     }
     if (
       trusted.some(
@@ -730,6 +825,7 @@ function validatedSpans(text, findings, options) {
         last.category = 'credential';
       }
       if (last.type === 'JSON_VALUE' || f.type === 'JSON_VALUE') {
+        last.valueType = jsonValueType(last, f);
         last.type = 'JSON_VALUE';
       } else if (credentialRun) {
         last.type = 'SECRET';
@@ -743,6 +839,15 @@ function validatedSpans(text, findings, options) {
   return spans;
 }
 
+// Two whole values merged into one keep a value type only when they agree.
+function jsonValueType(last, f) {
+  if (last.type !== 'JSON_VALUE') {
+    return f.valueType;
+  }
+  return f.type !== 'JSON_VALUE' || f.valueType === last.valueType
+    ? last.valueType
+    : undefined;
+}
 function preferredType(current, candidate, options) {
   if (current === 'JSON_VALUE') {
     return false;
@@ -779,7 +884,8 @@ export function redactResult(
 ) {
   options = applyProfile(options);
   const spans = validatedSpans(text, findings, options);
-  options = prepareFakeText(text, findings, initializePlaceholders(options));
+  options = initializePlaceholders(options);
+  options = withFeature('prepareFakeText', options, text, findings, options);
   preparePlaceholderText(text, findings, options);
   const parts = [];
   const faked = [];
@@ -905,22 +1011,26 @@ export async function redactResultAsync(
         }
         continue;
       }
-      if (!run) {
-        if (options.preserveEncoding) {
-          throw failure('ERR_ENCODING');
-        }
-        continue;
-      }
-      if (!options.preserveEncoding && run.encoding !== 'json-content') {
+      if (
+        !run ||
+        (!options.preserveEncoding && run.encoding !== 'json-content')
+      ) {
         continue;
       }
       const result = await sanitizeDecoded(
         run.text,
         nestedPlaceholderOptions(confirmationOptions(text, findings, options))
       );
+      // Without a recognizable re-encoding, the encoded text itself is masked.
       replacements.set(
         `${span.start}:${span.end}`,
-        encodeRun(run, result.text === run.text ? REDACTED : result.text)
+        encodeRun(run, result.text === run.text ? REDACTED : result.text) ??
+          renderReplacement(
+            text,
+            span,
+            { ...options, preserveEncoding: false },
+            () => ({ text: REDACTED })
+          )
       );
     }
   }

package/src/sarif.js

@@ -0,0 +1,127 @@
+// SARIF 2.1.0 export for code-scanning dashboards (GitHub, GitLab, Azure
+// DevOps), matching the reporting formats of Gitleaks/Betterleaks, TruffleHog,
+// detect-secrets and Kingfisher. Only metadata is emitted: rule, type,
+// category, confidence and location. Matched values are never included.
+const schema = 'https://json.schemastore.org/sarif-2.1.0.json';
+const levels = { credential: 'error', personal: 'warning' };
+
+function artifactUri(path) {
+  if (path === '-' || path === undefined) {
+    return 'stdin';
+  }
+  return encodeURI(String(path).replaceAll('\\', '/'));
+}
+
+function region(finding) {
+  const result = {
+    charOffset: finding.start,
+    charLength: finding.end - finding.start,
+  };
+  if (Number.isSafeInteger(finding.line)) {
+    result.startLine = finding.line;
+  }
+  if (Number.isSafeInteger(finding.column)) {
+    result.startColumn = finding.column;
+  }
+  return result;
+}
+
+function sarifResult(finding, uri, ruleIndex) {
+  return {
+    ruleId: finding.rule,
+    ruleIndex,
+    level: levels[finding.category] ?? 'note',
+    message: {
+      text: `${finding.type} ${finding.category === 'credential' ? 'credential' : 'personal data'} detected`,
+    },
+    locations: [
+      {
+        physicalLocation: {
+          artifactLocation: { uri },
+          region: region(finding),
+        },
+      },
+    ],
+    properties: {
+      type: finding.type,
+      category: finding.category,
+      confidence: finding.confidence,
+    },
+  };
+}
+
+/**
+ * Converts scan results to a SARIF 2.1.0 log. Accepts the CLI scan report
+ * (`{complete, results: [{path, findings}], skipped}`), an array of
+ * `{path, findings}` entries, or a findings array for one artifact. Values
+ * are never copied into the log; paths should already be sanitized.
+ */
+function entriesOf(report, path) {
+  if (!Array.isArray(report)) {
+    return report?.results ?? [];
+  }
+  return report.every((item) => Array.isArray(item?.findings))
+    ? report
+    : [{ path, findings: report }];
+}
+
+export function toSarif(report, { version, path = '-' } = {}) {
+  const entries = entriesOf(report, path);
+  const rules = [],
+    indexes = new Map(),
+    results = [];
+  for (const entry of entries) {
+    const uri = artifactUri(entry.path);
+    for (const finding of entry.findings) {
+      if (!indexes.has(finding.rule)) {
+        indexes.set(finding.rule, rules.length);
+        rules.push({
+          id: finding.rule,
+          name: finding.rule,
+          shortDescription: { text: `${finding.type} (${finding.category})` },
+          defaultConfiguration: {
+            level: levels[finding.category] ?? 'note',
+          },
+          properties: { tags: [finding.category, finding.type] },
+        });
+      }
+      results.push(sarifResult(finding, uri, indexes.get(finding.rule)));
+    }
+  }
+  const skipped = report?.skipped ?? [];
+  return {
+    $schema: schema,
+    version: '2.1.0',
+    runs: [
+      {
+        tool: {
+          driver: {
+            name: 'sensitive-data-sanitizer',
+            informationUri:
+              'https://github.com/link-foundation/sensitive-data-sanitizer',
+            ...(version ? { version } : {}),
+            rules,
+          },
+        },
+        invocations: [
+          {
+            executionSuccessful: report?.complete ?? true,
+            toolExecutionNotifications: skipped.map((item) => ({
+              level: 'warning',
+              message: { text: `Skipped: ${item.reason}` },
+              locations: [
+                {
+                  physicalLocation: {
+                    artifactLocation: { uri: artifactUri(item.path) },
+                  },
+                },
+              ],
+            })),
+          },
+        ],
+        columnKind: 'utf16CodeUnits',
+        results,
+      },
+    ],
+  };
+}

package/src/stream-worker.js

@@ -1,4 +1,5 @@
 import { parentPort, workerData } from 'node:worker_threads';
+import './personal-features.js';
 import { sanitizeFileToFile } from './stream.js';
 import { compileKnown } from './compiled-known.js';
 try {

package/src/strict-attributes.js

@@ -1,4 +1,5 @@
 import { collectMatches } from './detection.js';
+import { compiledVocabulary } from './vocabulary.js';
 
 const attributes =
   /(?<![\p{L}\p{N}_])(?:женщин[аыуеой]{1,2}|мужчин[аыуеой]{1,2}|замужем|женат(?:а|ы)?|(?:один|одна|двое|трое|четверо|пятеро|\d{1,2})[ \t]+(?:детей|реб[её]нок|реб[её]нка)|русск(?:ая|ий|ого|ой|ие)|татар(?:ин|ка|ы)|православн(?:ый|ая|ого|ой)|мусульман(?:ин|ка)|инвалидност[ьи](?:[ \t]+\d[ \t]+группы)?|беременн(?:а|ая|ость)|судимость(?:[ \t]+погашена)?|член[ \t]+партии|woman|women|male|female|married|divorced|pregnant|pregnancy|disability|christian|muslim|jewish|orthodox|hispanic|criminal[ \t]+record|party[ \t]+member|(?:one|two|three|\d{1,2})[ \t]+children)(?![\p{L}\p{N}_])/giu;
@@ -10,10 +11,45 @@ const additionalAttributes =
 // unfamiliar diagnoses, nationalities and religions outside this lexicon.
 const categoryForms =
   /(?<![\p{L}\p{N}_])(?:(?:диабет|католик|католич|буддист|атеист|белорус|православн|беременн|судим)[\p{L}]{0,8}|татар(?:ин(?:а|у|ом|е)?|к[аиуе]|кой|ы|ам|ами|ах)?|мусульман(?:ин(?:а|у|ом|е)?|к[аиуе]|кой|е|ам|ами|ах)?|украин(?:ец|ца|цу|цем|це|цы|цев|цам|цами|цах|к[аиуе]|кой)|инвалид(?:а|у|ом|е|ы|ов|ам|ами|ах|к[аиу]|кой)?|армян(?:ин|ина|ину|ином|ине|е|ам|ами|ах|ка|ки|ке|ку|кой)|евре(?:й|я|ю|ем|и|ев|ям|ями|ях|йк[аиуе]|йкой)|казах(?:а|у|ом|е|и|ов|ам|ами|ах|к[аиуе]|кой)?|христиан(?:ин|ина|ину|ином|ине|е|ам|ами|ах|к[аиуе]|кой)|иуде(?:й|я|ю|ем|и|ев|ям|ями|ях|йк[аиуе]|йкой)|ислам[\p{L}]{0,5}|эпилепси[\p{L}]{1,5}|астм[\p{L}]{1,5}|онкологи[\p{L}]{1,5}|туберкул[её]з[\p{L}]{0,5}|психиатр[\p{L}]{0,6}|депресси[\p{L}]{1,5}|гепатит[\p{L}]{0,5}|гомосексуал[\p{L}]{0,7}|лесбиян[\p{L}]{1,5}|ге(?:й|я|ю|ем|и|ев|ям|ями|ях)|нетрадиционн[\p{L}]{2,3}[ \t]+ориентаци[\p{L}]{1,3}|отбыва(?:л[аи]?|ет|ют|вш[\p{L}]{2,3})[ \t]+наказани[\p{L}]{1,3}|ВИЧ-инфицирован[\p{L}]{0,4}|epilepsy|asthma|tuberculosis|hepatitis|depression|gay|lesbian|homosexual|bisexual|armenian)(?![\p{L}\p{N}_])/giu;
+// Nationality nouns with their case endings and feminine forms. Exact endings
+// keep country and region names (`Узбекистан`, `Чувашия`) out of the match.
+const hardNoun = '(?:а|у|ом|е|ы|ов|ам|ами|ах)?';
+const softNoun = '(?:а|у|ом|е|и|ей|ам|ами|ах)?';
+const ecNoun = '(?:ец|ц(?:а|у|ем|е|ы|ев|ам|ами|ах))';
+const femaleNoun = '(?:а|и|е|у|ой)';
+const nationalities = [
+  `узбек${softNoun}|узбечк${femaleNoun}`,
+  `таджик${softNoun}|таджичк${femaleNoun}`,
+  `(?:киргиз|кыргыз)${hardNoun}|(?:киргиз|кыргыз)к${femaleNoun}`,
+  `туркмен${hardNoun}|туркменк${femaleNoun}`,
+  `чечен${ecNoun}|чеченк${femaleNoun}`,
+  `ингуш${softNoun}|ингушк${femaleNoun}`,
+  `осетин${hardNoun}|осетинк${femaleNoun}`,
+  `грузин${hardNoun}|грузинк${femaleNoun}`,
+  `азербайджан${ecNoun}|азербайджанк${femaleNoun}`,
+  `цыган${hardNoun}|цыганк${femaleNoun}`,
+  `коре(?:ец|йц(?:а|у|ем|е|ы|ев|ам|ами|ах))|кореянк${femaleNoun}`,
+  `кита(?:ец|йц(?:а|у|ем|е|ы|ев|ам|ами|ах))|китаянк${femaleNoun}`,
+  `башкир${hardNoun}|башкирк${femaleNoun}`,
+  `чуваш${softNoun}|чувашк${femaleNoun}`,
+  `якут${hardNoun}|якутк${femaleNoun}`,
+  `бурят${hardNoun}|бурятк${femaleNoun}`,
+  `калмык${softNoun}|калмычк${femaleNoun}`,
+  `лезгин${hardNoun}|лезгинк${femaleNoun}`,
+  `авар${ecNoun}|аварк${femaleNoun}`,
+  `дагестан${ecNoun}|дагестанк${femaleNoun}`,
+  `уйгур${hardNoun}|уйгурк${femaleNoun}`,
+].join('|');
+const believerNoun = `(?:а|у|ом|е|ы|ов|ам|ами|ах|к${femaleNoun}|кой)?`;
+const specialVocabulary = new RegExp(
+  `(?<![\\p{L}\\p{N}_])(?:${nationalities}|(?:протестант|баптист|адвентист|индуист|кришнаит|иеговист|пятидесятник)${believerNoun}|лютеран(?:ин(?:а|у|ом|е)?|к${femaleNoun}|кой|е|ам|ами|ах)|старообряд${ecNoun}|старообрядк${femaleNoun}|свидетел[\\p{L}]{1,3}[ \\t]+иеговы|инсульт[\\p{L}]{0,3}|инфаркт[\\p{L}]{0,3}|шизофрени[\\p{L}]{1,4}|аутизм[\\p{L}]{0,3}|аутист(?:ка|ки|ке|ку|кой|а|у|ом|ы|ов)?|слабослышащ[\\p{L}]{1,4}|слабовидящ[\\p{L}]{1,4}|незряч[\\p{L}]{1,4}|дцп|колясочни(?:к${hardNoun}|ц${femaleNoun})|(?:передвига[\\p{L}]{1,5}|инвалидн[\\p{L}]{2,4})[ \\t]+(?:на[ \\t]+)?коляск[\\p{L}]{1,3}|болезн[\\p{L}]{1,3}[ \\t]+(?:паркинсона|альцгеймера)|деменци[\\p{L}]{1,3}|алкоголизм[\\p{L}]{0,2}|наркозависим[\\p{L}]{0,4}|биполярн[\\p{L}]{2,4}[ \\t]+расстройств[\\p{L}]{0,3}|(?:би|транс|пан|а)сексуал[\\p{L}]{0,6}|трансгендер[\\p{L}]{0,6}|(?:сидел[аи]?|отсидел[аи]?|осужд[её]н[аы]?|привлекал(?:ся|ась|ись)|отбывал[аи]?[ \\t]+срок)[ \\t]+(?:за|по)[ \\t]+(?:краж|грабе|разбо|убийств|мошенничеств|наркотик|хулиганств|изнасиловани|взятк|стать)[\\p{L}]{0,4}(?:[ \\t]+\\d{1,3}(?:\\.\\d)?)?|(?:сидел[аи]?|отсидел[аи]?)[ \\t]+(?:в[ \\t]+)?(?:тюрьм[\\p{L}]{1,2}|колони[\\p{L}]{1,2}|срок)|бывш[\\p{L}]{2,3}[ \\t]+заключ[её]нн[\\p{L}]{2,3}|член[\\p{L}]{0,3}[ \\t]+профсоюз[\\p{L}]{0,3}|состо(?:ит|ял[аи]?|ю|ят)[ \\t]+в[ \\t]+профсоюзе|профсоюзн(?:ый|ая|ого|ой)[ \\t]+(?:активист|деятел|лидер)[\\p{L}]{0,3}|autism|autistic|schizophreni[ac]|transgender|transsexual|pansexual|asexual|wheelchair[ \\t]+users?|uses[ \\t]+a[ \\t]+wheelchair|in[ \\t]+a[ \\t]+wheelchair|(?:hearing|visually)[ \\t-]+impaired|had[ \\t]+a[ \\t]+(?:stroke|heart[ \\t]+attack)|dementia|alzheimer'?s|parkinson'?s|bipolar[ \\t]+disorder|alcoholism|trade[ \\t]+union[ \\t]+member(?:ship)?|member[ \\t]+of[ \\t]+(?:a|the)[ \\t]+(?:trade|labou?r)[ \\t]+union|convicted[ \\t]+(?:of|for)[ \\t]+[\\p{L}]{3,32}|served[ \\t]+(?:time[ \\t]+(?:in|for)|a[ \\t]+(?:prison[ \\t]+)?sentence)|protestant|lutheran|mormon|hindu|sikh|jehovah'?s[ \\t]+witness)(?![\\p{L}\\p{N}_])`,
+  'giu'
+);
 const categoryCue =
-  /(?<![\p{L}\p{N}_])(?:по[ \t]+национальности|национальность|этничность|вероисповедание|религия|диагноз|исповеду(?:ет|ют|ю|ешь)|боле(?:ет|ют|ю|ешь)|страда(?:ет|ют|ю|ешь)[ \t]+от|на[ \t]+уч[её]те[ \t]+у|diagnosed[ \t]+with|suffers[ \t]+from|nationality|ethnicity|religion|diagnosis)(?:["']?[ \t]*[:=][ \t]*|[ \t]+)["']?([\p{L}][\p{L}\p{M}-]{0,63})(?![\p{L}\p{M}\p{N}_])/giu;
+  /(?<![\p{L}\p{N}_])(?:по[ \t]+(?:национальности|происхождению)|национальность|этничность|вероисповедание|религия|диагноз|исповеду(?:ет|ют|ю|ешь)|боле(?:ет|ют|ю|ешь)|страда(?:ет|ют|ю|ешь)[ \t]+от|на[ \t]+уч[её]те[ \t]+у|diagnosed[ \t]+with|suffers[ \t]+from|nationality|ethnicity|religion|diagnosis)(?:["']?[ \t]*[:=][ \t]*|[ \t]+)["']?([\p{L}][\p{L}\p{M}-]{0,63})(?![\p{L}\p{M}\p{N}_])/giu;
+// Words that are also languages, scripts or styles need a personal context.
 const ambiguous =
-  /(?<![\p{L}\p{N}_])(?:disabled|single|man|men|russian|asian)(?![\p{L}\p{N}_])/giu;
+  /(?<![\p{L}\p{N}_])(?:disabled|single|man|men|russian|asian|uzbek|tajik|chechen|azerbaijani|korean|chinese|georgian|kazakh|kyrgyz|turkmen|uyghur|roma|baptist|union[ \t]+member)(?![\p{L}\p{N}_])/giu;
 
 function personalContext(text, match) {
   const prefix = text.slice(Math.max(0, match.index - 128), match.index);
@@ -56,8 +92,16 @@ export function detectStrictAttributes(text, emit, options) {
     detected++;
     emit(finding);
   };
-  for (const pattern of [attributes, additionalAttributes, categoryForms]) {
-    collectMatches(text, pattern, report, details);
+  for (const pattern of [
+    attributes,
+    additionalAttributes,
+    categoryForms,
+    specialVocabulary,
+    compiledVocabulary(options.vocabulary)?.attributes,
+  ]) {
+    if (pattern) {
+      collectMatches(text, pattern, report, details);
+    }
   }
   collectMatches(text, categoryCue, report, details, 1);
   collectMatches(text, ambiguous, report, details, 0, (_value, match) => {

package/src/strict.js

@@ -2,7 +2,7 @@ import { collectMatches } from './detection.js';
 import { months } from './personal-context.js';
 import { isDocumentTechnology } from './document-experience.js';
 import { placeholderValues } from './placeholders.js';
-import { generatedValues } from './fake.js';
+import { generatedValues } from './fake-memory.js';
 import { detectStrictAttributes } from './strict-attributes.js';
 
 const boundary = '[\\p{L}\\p{N}_]';

package/src/structured.js

@@ -171,6 +171,7 @@ export function structuredFieldFindings(batch, options) {
         start: entry.start + (denied ? 0 : 1),
         end: entry.start + entry.raw.length - (denied ? 0 : 1),
         type: denied ? 'JSON_VALUE' : type,
+        ...(denied && type ? { valueType: type } : {}),
         category: 'personal',
         rule: 'known-personal',
         confidence: 1,
@@ -516,6 +517,7 @@ function declaredValueFinding(text, token, options) {
     start,
     end,
     type: 'JSON_VALUE',
+    ...(type ? { valueType: type } : {}),
     category: 'personal',
     rule: 'known-personal',
     confidence: 1,

package/src/token-checksum.js

@@ -0,0 +1,49 @@
+// Offline structural check for prefixed tokens that end in a checksum.
+// GitHub (ghp_, gho_, ghu_, ghs_, ghr_) and npm (npm_) tokens append six
+// base62 characters: CRC32 (IEEE) of the characters after the first `_`.
+// A valid checksum raises confidence; an invalid one never removes a finding
+// because truncated or retyped tokens still need redaction.
+const BASE62 = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
+const CHECKSUM_TOKEN = /^(?:gh[opsur]|npm)_([A-Za-z0-9]{30})([A-Za-z0-9]{6})$/;
+
+let table;
+function crcTable() {
+  table ??= Array.from({ length: 256 }, (_, n) => {
+    let c = n;
+    for (let k = 0; k < 8; k++) {
+      c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+    }
+    return c >>> 0;
+  });
+  return table;
+}
+
+export function crc32(text) {
+  let crc = 0xffffffff;
+  for (let i = 0; i < text.length; i++) {
+    crc = crcTable()[(crc ^ text.charCodeAt(i)) & 0xff] ^ (crc >>> 8);
+  }
+  return (crc ^ 0xffffffff) >>> 0;
+}
+
+export function base62(value, width = 6) {
+  let out = '';
+  for (let i = 0; i < width; i++) {
+    out = BASE62[value % 62] + out;
+    value = Math.floor(value / 62);
+  }
+  return out;
+}
+
+// Builds a checksum-valid token from a prefix (including `_`) and a
+// 30-character body, so tests and examples need no stored token literals.
+export function tokenWithChecksum(prefix, body) {
+  return `${prefix}${body}${base62(crc32(body))}`;
+}
+
+// true: checksum matches; false: format matches but checksum does not;
+// undefined: the value is not a checksummed token format.
+export function tokenChecksum(value) {
+  const match = CHECKSUM_TOKEN.exec(value);
+  return match ? base62(crc32(match[1])) === match[2] : undefined;
+}