@link-foundation/sensitive-data-sanitizer 0.19.3 → 0.21.0

package/docs/RULES.md

@@ -12,16 +12,221 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `anthropic`
 - `authorization`
 - `aws`
+- `betterleaks/abuseipdb-api-key.1`
+- `betterleaks/aikido-ci-token`
+- `betterleaks/aikido-client-secret`
+- `betterleaks/airtable-oauth-token`
+- `betterleaks/aiven-auth-token`
+- `betterleaks/alibaba-sts-access-key-secret`
+- `betterleaks/amplitude-secret-key`
+- `betterleaks/apify-api-token`
+- `betterleaks/apollo-api-key.1`
+- `betterleaks/asaas-api-token`
+- `betterleaks/assemblyai-api-key`
+- `betterleaks/auth0-client-secret.1`
+- `betterleaks/azure-app-configuration-connection-string`
+- `betterleaks/azure-servicebus-connection-string`
+- `betterleaks/azure-storage-account-key`
+- `betterleaks/bitbucket-data-center-token`
+- `betterleaks/bitly-access-token`
+- `betterleaks/bitrise-access-token`
+- `betterleaks/box-api-access-token`
+- `betterleaks/brave-search-api-key`
+- `betterleaks/browserstack-access-key.1`
+- `betterleaks/buildkite-service-token`
+- `betterleaks/buildkite-user-access-token`
+- `betterleaks/canadian-digital-service-notify-api-key`
+- `betterleaks/canva-client-secret`
+- `betterleaks/cartesia-api-key.1`
+- `betterleaks/cerebras-api-key`
+- `betterleaks/checkout-secret-key`
+- `betterleaks/circleci-personal-token`
+- `betterleaks/circleci-project-token`
+- `betterleaks/civo-api-key`
+- `betterleaks/clerk-secret-key`
+- `betterleaks/clickup-personal-api-token`
+- `betterleaks/cloudflare-api-key.2`
+- `betterleaks/cloudinary-api-secret`
+- `betterleaks/cloudsmith-api-key`
+- `betterleaks/cockroachlabs-cloud-api-key`
+- `betterleaks/configcat-sdk-key`
+- `betterleaks/configcat-sdk-key-extended`
+- `betterleaks/couchbase-capella-api-key`
+- `betterleaks/coveralls-personal-api-token`
+- `betterleaks/crates-io-api-key`
+- `betterleaks/cursor-api-key`
+- `betterleaks/databento-api-key`
+- `betterleaks/datadog-api-key`
+- `betterleaks/datadog-application-key`
+- `betterleaks/datagov-api-key`
+- `betterleaks/datastax-astra-application-token`
+- `betterleaks/deepgram-api-key`
+- `betterleaks/deepseek-api-key`
+- `betterleaks/deno-account-token`
+- `betterleaks/devcycle-client-sdk-key`
+- `betterleaks/devcycle-mobile-sdk-key`
+- `betterleaks/devcycle-server-sdk-key`
+- `betterleaks/devin-personal-api-key`
+- `betterleaks/devin-service-api-key`
+- `betterleaks/devin-service-user-token`
+- `betterleaks/disqus-api-key`
+- `betterleaks/docker-swarm-join-token`
+- `betterleaks/docker-swarm-unlock-key`
+- `betterleaks/dockerhub-organization-access-token`
+- `betterleaks/dockerhub-personal-access-token`
+- `betterleaks/ebay-client-secret`
+- `betterleaks/elastic-cloud-api-key`
+- `betterleaks/elevenlabs-api-key`
+- `betterleaks/endorlabs-api-key`
+- `betterleaks/endorlabs-api-secret`
+- `betterleaks/etsy-open-api-key`
+- `betterleaks/exoscale-api-key`
+- `betterleaks/fal-api-key.1`
+- `betterleaks/figma-personal-access-header-token`
+- `betterleaks/figma-personal-access-token`
+- `betterleaks/fullstory-api-key`
+- `betterleaks/gcp-application-default-credentials`
+- `betterleaks/gcp-gemini-api`
+- `betterleaks/gcp-service-account`
+- `betterleaks/gitea-access-token`
+- `betterleaks/gitlab-incoming-mail-address-token`
+- `betterleaks/gitlab-pat-routable-versioned`
+- `betterleaks/greptile-api-key`
+- `betterleaks/groq-api-key`
+- `betterleaks/gumroad-access-token`
+- `betterleaks/highnote-secret-live-key`
+- `betterleaks/honeycomb-api-key`
+- `betterleaks/hunter-api-key.1`
+- `betterleaks/ibm-cloud-user-api-key`
+- `betterleaks/influxdb-api-token`
+- `betterleaks/infomaniak-api-token`
+- `betterleaks/instantly-api-key.1`
+- `betterleaks/ionic-personal-access-token`
+- `betterleaks/jumpcloud-api-key`
+- `betterleaks/kagi-api-key`
+- `betterleaks/kimi-api-key`
+- `betterleaks/klaviyo-api-key`
+- `betterleaks/langchain-langsmith-personal-access-token`
+- `betterleaks/langchain-langsmith-service-key`
+- `betterleaks/langfuse-secret-key.1`
+- `betterleaks/lark-app-secret`
+- `betterleaks/lichess-personal-access-token`
+- `betterleaks/lighton-paradigm-api-key`
+- `betterleaks/llama-cloud-api-key.1`
+- `betterleaks/mailersend-api-token`
+- `betterleaks/mem0-api-key.1`
+- `betterleaks/mercury-production-api-token`
+- `betterleaks/mergify-application-key`
+- `betterleaks/midtrans-production-server-client-key`
+- `betterleaks/minimax-api-key`
+- `betterleaks/miro-access-token`
+- `betterleaks/miro-client-secret`
+- `betterleaks/mistral-api-key`
+- `betterleaks/monday-api-token.1`
+- `betterleaks/mongodb-atlas-service-account-secret`
+- `betterleaks/mongodb-connection-string`
+- `betterleaks/mux-access-token-secret.1`
+- `betterleaks/neon-api-key`
+- `betterleaks/ngrok-api-key.1`
+- `betterleaks/nuget-api-key`
+- `betterleaks/nvidia-api-key`
+- `betterleaks/nylas-api-key.1`
+- `betterleaks/ollama-api-key`
+- `betterleaks/onesignal-rich-authentication-token`
+- `betterleaks/onfido-live-api-token-ca`
+- `betterleaks/onfido-live-api-token-eu`
+- `betterleaks/onfido-live-api-token-us`
+- `betterleaks/openrouter-api-key`
+- `betterleaks/openweather-api-key`
+- `betterleaks/opsgenie-api-key.1`
+- `betterleaks/ovh-application-secret`
+- `betterleaks/paddle-live-api-key`
+- `betterleaks/pagerduty-authorization-token.1`
+- `betterleaks/paypal-client-secret.1`
+- `betterleaks/persona-production-api-key`
+- `betterleaks/pinecone-api-key.1`
+- `betterleaks/pinecone-api-key.2`
+- `betterleaks/pinterest-access-token`
+- `betterleaks/plivo-auth-token`
+- `betterleaks/polar-oauth-access-token`
+- `betterleaks/polar-organization-access-token`
+- `betterleaks/polar-personal-access-token`
+- `betterleaks/polymarket-api-key`
+- `betterleaks/polymarket-private-key`
+- `betterleaks/posthog-personal-api-key`
+- `betterleaks/posthog-project-api-key`
+- `betterleaks/postmark-api-token.1`
+- `betterleaks/proof-full-access-api-key`
+- `betterleaks/rainforest-pay-production-api-key`
+- `betterleaks/ramp-client-secret`
+- `betterleaks/razorpay-key-secret.1`
+- `betterleaks/redirect-pizza-api-token.1`
+- `betterleaks/render-api-key`
+- `betterleaks/replicate-api-token`
+- `betterleaks/resend-api-key.1`
+- `betterleaks/retell-api-key.1`
+- `betterleaks/rootly-api-key.1`
+- `betterleaks/runpod-api-key.1`
+- `betterleaks/salesforce-access-token.1`
+- `betterleaks/samsara-api-token.1`
+- `betterleaks/scaleway-secret-key`
+- `betterleaks/scalr-api-access-token.1`
+- `betterleaks/segment-public-api-token.1`
+- `betterleaks/slack-session-cookie`
+- `betterleaks/slack-session-token`
+- `betterleaks/snowflake-programmatic-access-token.1`
+- `betterleaks/sslmate-api-key.1`
+- `betterleaks/stability-ai-api-key`
+- `betterleaks/supabase-management-token`
+- `betterleaks/supabase-project-api-key`
+- `betterleaks/tableau-personal-access-token.1`
+- `betterleaks/tailscale-api-key.1`
+- `betterleaks/telnyx-api-v2-key.1`
+- `betterleaks/temporal-cloud-api-key.1`
+- `betterleaks/thunderstore-api-token.1`
+- `betterleaks/togetherai-api-key`
+- `betterleaks/unkey-root-key.1`
+- `betterleaks/upcloud-api-token`
+- `betterleaks/upstage-api-key`
+- `betterleaks/upstash-redis-rest-token.1`
+- `betterleaks/val-town-api-token.1`
+- `betterleaks/vercel-ai-gateway-key`
+- `betterleaks/vercel-api-token`
+- `betterleaks/vercel-app-access-token`
+- `betterleaks/vercel-app-refresh-token`
+- `betterleaks/vercel-integration-token`
+- `betterleaks/vercel-personal-access-token`
+- `betterleaks/virustotal-api-key.1`
+- `betterleaks/voyageai-api-key`
+- `betterleaks/vultr-api-key.1`
+- `betterleaks/wakatime-api-key.1`
+- `betterleaks/wakatime-api-key.2`
+- `betterleaks/weatherstack-api-key.1`
+- `betterleaks/weights-and-biases-api-key`
+- `betterleaks/weights-and-biases-api-key-v1`
+- `betterleaks/wiz-client-secret.1`
+- `betterleaks/woocommerce-consumer-secret.1`
+- `betterleaks/workato-developer-api-token.1`
+- `betterleaks/workos-production-api-key.1`
+- `betterleaks/xai-api-key`
+- `betterleaks/xendit-production-api-key.1`
+- `betterleaks/zai-api-key`
+- `betterleaks/zoho-client-secret.1`
+- `betterleaks/zoho-oauth-token.1`
+- `betterleaks/zoho-zapi-key.1`
+- `betterleaks/zuplo-consumer-api-key.1`
 - `birth-context`
 - `birth-year`
 - `booking-context`
 - `booking-filename`
+- `card-context`
 - `catalog/aws`
 - `catalog/azure`
 - `catalog/google`
 - `cli-password`
 - `context`
 - `cookie`
+- `credential-pair`
 - `databricks`
 - `detect-secrets`
 - `digitalocean`
@@ -315,6 +520,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `google`
 - `grafana`
 - `home-path`
+- `honorific-name`
 - `huggingface`
 - `iban-checksum`
 - `identity-number`
@@ -329,9 +535,12 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `location-gazetteer`
 - `location-label`
 - `location-residence`
+- `login-call`
 - `luhn`
 - `mac`
+- `mail-display-name`
 - `mailchimp`
+- `marker-token`
 - `messenger-handle`
 - `messenger-label`
 - `mrz`
@@ -342,6 +551,7 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `name-initials`
 - `name-title`
 - `name-variant`
+- `named-value`
 - `national-format`
 - `national-phone`
 - `ner/offline`
@@ -365,10 +575,13 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `passport-format`
 - `passport-issuer`
 - `passport-series`
+- `password-hash`
 - `personal-profile-url`
 - `phone-context`
+- `po-box`
 - `postal-address`
 - `postal-code`
+- `postal-label`
 - `presidio`
 - `private-key`
 - `profile-attribute`
@@ -378,12 +591,17 @@ Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
 - `pypi`
 - `quoted-entropy`
 - `residence-prose`
+- `salutation`
 - `school-acronym-context`
 - `secretlint`
+- `self-introduction`
 - `sendgrid`
 - `shopify`
+- `sign-off`
 - `slack`
 - `square`
+- `ssn-label`
+- `state-zip`
 - `strict-age`
 - `strict-attribute`
 - `strict-date`

package/examples/second-pass-placeholders.mjs

@@ -0,0 +1,28 @@
+// Sanitizes a CV note into readable placeholders, then sanitizes the result
+// again after a model added text, keeping the tokens already in it.
+import {
+  createSyncSanitizer,
+  isPlaceholderToken,
+} from '@link-foundation/sensitive-data-sanitizer';
+
+const policy = (existing) => ({
+  profile: 'documents',
+  properNames: 'capitalized',
+  strict: true,
+  transformation: { mode: 'placeholder', map: false, existing },
+});
+
+const first = createSyncSanitizer(policy()).sanitize(
+  'Кандидат Петров Иван Сергеевич, тел. +7 916 123-45-67. Работал в ООО «Ромашка», живу в Казани. Перенёс инсульт.'
+).text;
+console.log(first);
+// Кандидат Person-1, тел. Phone-1. Работал в Organization-1, живу в Location-1. Перенёс PersonalAttribute-1.
+
+const answer = `${first} Рекомендует Анна Смирнова.`;
+const second = createSyncSanitizer(policy('keep')).sanitize(answer).text;
+console.log(second);
+// ... Перенёс PersonalAttribute-1. Рекомендует Person-2.
+
+console.log(
+  second.split(/[\s,.]+/u).filter((word) => isPlaceholderToken(word))
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-foundation/sensitive-data-sanitizer",
-  "version": "0.19.3",
+  "version": "0.21.0",
   "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
   "type": "module",
   "main": "src/index.js",
@@ -27,12 +27,21 @@
     "examples/reversible-placeholders.mjs",
     "examples/prose-logs.mjs",
     "docs/RULES.md",
-    "examples/custom-log-policy.mjs"
+    "examples/custom-log-policy.mjs",
+    "examples/second-pass-placeholders.mjs"
   ],
   "exports": {
     ".": {
       "types": "./src/index.d.ts",
-      "import": "./src/index.js"
+      "import": "./src/index.js",
+      "require": "./src/index.js",
+      "default": "./src/index.js"
+    },
+    "./credentials": {
+      "types": "./src/credentials-entry.d.ts",
+      "import": "./src/credentials-entry.js",
+      "require": "./src/credentials-entry.js",
+      "default": "./src/credentials-entry.js"
     },
     "./eslint-plugin": {
       "types": "./src/eslint-plugin.d.ts",

package/src/allow-list.js

@@ -0,0 +1,93 @@
+import { failure } from './detection.js';
+
+// Caller-declared values are explicit private policy; an allow list must not
+// silently cancel them. Schema denials (JSON_VALUE) always win as well.
+const declaredRules = new Set([
+  'known-personal',
+  'confirmed-personal',
+  'known-secret',
+]);
+const anchored = new WeakMap();
+
+function fullMatcher(pattern) {
+  let compiled = anchored.get(pattern);
+  if (!compiled) {
+    compiled = new RegExp(
+      `^(?:${pattern.source})$`,
+      pattern.flags.replace(/[gy]/g, '')
+    );
+    anchored.set(pattern, compiled);
+  }
+  return compiled;
+}
+
+// Inline suppression comments understood by Gitleaks/Betterleaks
+// (gitleaks:allow), detect-secrets and ripsecrets (pragma: allowlist secret),
+// Kingfisher, TruffleHog and SonarQube. Honoured only with inlineAllow: true,
+// because anyone who writes to a log could otherwise suppress redaction.
+const sameLine =
+  /(?:gitleaks|betterleaks|kingfisher|trufflehog|sanitizer):(?:allow|ignore)\b|pragma:\s*allowlist(?:\s+secret)?\b(?!\s+nextline)|\bNOSONAR\b/i;
+const nextLine = /pragma:\s*allowlist\s+nextline(?:\s+secret)?\b/i;
+
+function lineAt(text, index) {
+  const start = text.lastIndexOf('\n', index - 1) + 1;
+  const end = text.indexOf('\n', index);
+  return { start, end: end < 0 ? text.length : end };
+}
+
+function inlineSuppressed(text, finding) {
+  const line = lineAt(text, finding.start);
+  if (sameLine.test(text.slice(line.start, line.end))) {
+    return true;
+  }
+  if (line.start === 0) {
+    return false;
+  }
+  const previous = lineAt(text, line.start - 1);
+  return nextLine.test(text.slice(previous.start, previous.end));
+}
+
+export function validateAllowList(options) {
+  if (
+    options.inlineAllow !== undefined &&
+    typeof options.inlineAllow !== 'boolean'
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+  const list = options.allowList;
+  if (list === undefined) {
+    return;
+  }
+  if (
+    !Array.isArray(list) ||
+    list.length > 10000 ||
+    !list.every(
+      (entry) =>
+        (typeof entry === 'string' && entry.length > 0) ||
+        entry instanceof RegExp
+    )
+  ) {
+    throw failure('ERR_CONFIG');
+  }
+}
+
+// Presidio allow_list, Google exclusion rules (MATCHING_TYPE_FULL_MATCH) and
+// DataFog allowlist_patterns: a listed string or pattern must cover the whole
+// detected value. Partial matches never shorten a finding.
+export function allowListed(text, finding, options) {
+  if (
+    !(options.allowList?.length || options.inlineAllow) ||
+    declaredRules.has(finding.rule) ||
+    finding.type === 'JSON_VALUE' ||
+    (finding.category === 'credential' && !options.allowCredentialDisable)
+  ) {
+    return false;
+  }
+  if (options.inlineAllow && inlineSuppressed(text, finding)) {
+    return true;
+  }
+  const value = text.slice(finding.start, finding.end);
+  return (options.allowList ?? []).some((entry) =>
+    typeof entry === 'string' ? entry === value : fullMatcher(entry).test(value)
+  );
+}

package/src/confidence.js

@@ -1,13 +1,18 @@
 // A valid check digit is common in arbitrary log numbers. Context decides
 // whether that evidence merits a replacement; inspect retains low scores.
-export function threshold(options) {
-  return options.minConfidence ?? options.threshold ?? 0.5;
+export function threshold(options, type) {
+  return (
+    options.minConfidenceByType?.[type] ??
+    options.minConfidence ??
+    options.threshold ??
+    0.5
+  );
 }
 export function actionable(finding, options) {
   return (
     finding.kept !== 'fake' &&
     (finding.category === 'credential' ||
-      finding.confidence >= threshold(options))
+      finding.confidence >= threshold(options, finding.type))
   );
 }
 export function numericNoise(text, start) {
@@ -29,11 +34,10 @@ export function epochNumber(value) {
   );
 }
 export function cardEvidence(text, start, value) {
-  const identityContext =
-    /(?<![\p{L}\p{N}_])(?:card|visa|mastercard|amex|discover|jcb|unionpay|mir|diners|карта)(?![\p{L}\p{N}_])/iu.test(
-      text.slice(Math.max(0, start - 64), start + value.length + 32)
-    );
-  return identityContext || (!numericNoise(text, start) && !epochNumber(value));
+  return (
+    cardContext(text, start, value) ||
+    (!numericNoise(text, start) && !epochNumber(value))
+  );
 }
 export function cardNetwork(value) {
   const d = value.replace(/\D/g, ''),
@@ -60,6 +64,17 @@ function otherCard(d, n) {
       n <= 19) ||
     (/^62/.test(d) && n >= 16 && n <= 19) ||
     (/^220[0-4]/.test(d) && n >= 16 && n <= 19) ||
-    (/^3(?:0[0-5]|[68])/.test(d) && n === 14)
+    (/^3(?:0[0-5]|[68])/.test(d) && n === 14) ||
+    // Maestro and Dankort IINs listed by Presidio and the card networks.
+    (/^(?:50(?:1[89]|20|38)|5893|6304|6759|676[1-3])/.test(d) &&
+      n >= 12 &&
+      n <= 19)
   );
 }
+// UATP cards (prefix 1, 15 digits) collide with numeric IDs, so they need a
+// card word nearby.
+export const cardContext = (text, start, value) =>
+  /(?<![\p{L}\p{N}_])(?:card|visa|mastercard|amex|discover|jcb|unionpay|mir|diners|maestro|dankort|uatp|карта)(?![\p{L}\p{N}_])/iu.test(
+    text.slice(Math.max(0, start - 64), start + value.length + 32)
+  );
+export const uatpCard = (value) => /^1\d{14}$/.test(value.replace(/\D/g, ''));