@link-foundation/sensitive-data-sanitizer 0.18.0 → 0.19.0

package/docs/RULES.md

@@ -0,0 +1,403 @@
+# Stable finding rule identifiers
+
+Use keys from the exported frozen `builtInRules` list in `rules: {id: false}`.
+Unknown ids fail with ERR_CONFIG. Credential and mixed encoded/scanner rules
+require `allowCredentialDisable: true`. A disabled rule does not suppress
+another rule's overlapping finding. Rules gate automatic built-in findings;
+explicit findings and custom callbacks retain caller policy. See [API](API.md#rule-type-and-field-policy).
+
+Regenerate with `node experiments/issue-118/generate-rule-catalog.mjs`.
+
+- `age-context`
+- `anthropic`
+- `authorization`
+- `aws`
+- `birth-context`
+- `birth-year`
+- `booking-context`
+- `booking-filename`
+- `catalog/aws`
+- `catalog/azure`
+- `catalog/google`
+- `cli-password`
+- `context`
+- `cookie`
+- `databricks`
+- `detect-secrets`
+- `digitalocean`
+- `discord`
+- `document-age-unit`
+- `document-at-employer`
+- `document-bank-suffix`
+- `document-birthday`
+- `document-citizenship`
+- `document-email-obfuscation`
+- `document-email-spaces`
+- `document-employer-dash`
+- `document-employer-prefix`
+- `document-industry`
+- `document-introduction`
+- `document-legal-lowercase`
+- `document-local-phone`
+- `document-messenger`
+- `document-military-id`
+- `document-mixed-name`
+- `document-mobile-bare`
+- `document-mobile-comma`
+- `document-mobile-context`
+- `document-msu`
+- `document-patronymic`
+- `document-personal-path`
+- `document-phone`
+- `document-phone-label`
+- `document-propagation`
+- `document-region`
+- `document-resume-link`
+- `document-reversed-role`
+- `document-role`
+- `document-role-employer`
+- `document-school-context`
+- `document-school-year`
+- `document-share-link`
+- `document-share-path`
+- `document-social-link`
+- `document-spaced-mobile`
+- `document-street`
+- `document-street-complete`
+- `document-unfamiliar-name`
+- `document-uppercase-initials`
+- `document-vehicle-plate`
+- `document-vin`
+- `education-acronym`
+- `education-bracket`
+- `education-label`
+- `education-named-acronym`
+- `education-russian`
+- `education-training`
+- `education-type`
+- `email`
+- `employer-label`
+- `employer-prose`
+- `employer-years`
+- `encoded`
+- `encoded-engine`
+- `encoded-limit`
+- `entity-format`
+- `entropy`
+- `firebase`
+- `gender-context`
+- `github`
+- `gitlab`
+- `gitlab-runner`
+- `gitleaks/1password-secret-key`
+- `gitleaks/1password-service-account-token`
+- `gitleaks/adafruit-api-key`
+- `gitleaks/adobe-client-id`
+- `gitleaks/adobe-client-secret`
+- `gitleaks/age-secret-key`
+- `gitleaks/airtable-api-key`
+- `gitleaks/airtable-personnal-access-token`
+- `gitleaks/algolia-api-key`
+- `gitleaks/alibaba-access-key-id`
+- `gitleaks/alibaba-secret-key`
+- `gitleaks/anthropic-admin-api-key`
+- `gitleaks/anthropic-api-key`
+- `gitleaks/artifactory-api-key`
+- `gitleaks/artifactory-reference-token`
+- `gitleaks/asana-client-id`
+- `gitleaks/asana-client-secret`
+- `gitleaks/atlassian-api-token`
+- `gitleaks/authress-service-client-access-key`
+- `gitleaks/aws-access-token`
+- `gitleaks/aws-amazon-bedrock-api-key-long-lived`
+- `gitleaks/aws-amazon-bedrock-api-key-short-lived`
+- `gitleaks/azure-ad-client-secret`
+- `gitleaks/beamer-api-token`
+- `gitleaks/bitbucket-client-id`
+- `gitleaks/bitbucket-client-secret`
+- `gitleaks/bittrex-access-key`
+- `gitleaks/bittrex-secret-key`
+- `gitleaks/cisco-meraki-api-key`
+- `gitleaks/clickhouse-cloud-api-secret-key`
+- `gitleaks/clojars-api-token`
+- `gitleaks/cloudflare-api-key`
+- `gitleaks/cloudflare-global-api-key`
+- `gitleaks/cloudflare-origin-ca-key`
+- `gitleaks/codecov-access-token`
+- `gitleaks/cohere-api-token`
+- `gitleaks/coinbase-access-token`
+- `gitleaks/confluent-access-token`
+- `gitleaks/confluent-secret-key`
+- `gitleaks/contentful-delivery-api-token`
+- `gitleaks/curl-auth-header`
+- `gitleaks/curl-auth-user`
+- `gitleaks/databricks-api-token`
+- `gitleaks/datadog-access-token`
+- `gitleaks/defined-networking-api-token`
+- `gitleaks/digitalocean-access-token`
+- `gitleaks/digitalocean-pat`
+- `gitleaks/digitalocean-refresh-token`
+- `gitleaks/discord-api-token`
+- `gitleaks/discord-client-id`
+- `gitleaks/discord-client-secret`
+- `gitleaks/doppler-api-token`
+- `gitleaks/droneci-access-token`
+- `gitleaks/dropbox-api-token`
+- `gitleaks/dropbox-long-lived-api-token`
+- `gitleaks/dropbox-short-lived-api-token`
+- `gitleaks/duffel-api-token`
+- `gitleaks/dynatrace-api-token`
+- `gitleaks/easypost-api-token`
+- `gitleaks/easypost-test-api-token`
+- `gitleaks/etsy-access-token`
+- `gitleaks/facebook-access-token`
+- `gitleaks/facebook-page-access-token`
+- `gitleaks/facebook-secret`
+- `gitleaks/fastly-api-token`
+- `gitleaks/finicity-api-token`
+- `gitleaks/finicity-client-secret`
+- `gitleaks/finnhub-access-token`
+- `gitleaks/flickr-access-token`
+- `gitleaks/flutterwave-encryption-key`
+- `gitleaks/flutterwave-public-key`
+- `gitleaks/flutterwave-secret-key`
+- `gitleaks/flyio-access-token`
+- `gitleaks/frameio-api-token`
+- `gitleaks/freemius-secret-key`
+- `gitleaks/freshbooks-access-token`
+- `gitleaks/gcp-api-key`
+- `gitleaks/generic-api-key`
+- `gitleaks/github-app-token`
+- `gitleaks/github-fine-grained-pat`
+- `gitleaks/github-oauth`
+- `gitleaks/github-pat`
+- `gitleaks/github-refresh-token`
+- `gitleaks/gitlab-cicd-job-token`
+- `gitleaks/gitlab-deploy-token`
+- `gitleaks/gitlab-feature-flag-client-token`
+- `gitleaks/gitlab-feed-token`
+- `gitleaks/gitlab-incoming-mail-token`
+- `gitleaks/gitlab-kubernetes-agent-token`
+- `gitleaks/gitlab-oauth-app-secret`
+- `gitleaks/gitlab-pat`
+- `gitleaks/gitlab-pat-routable`
+- `gitleaks/gitlab-ptt`
+- `gitleaks/gitlab-rrt`
+- `gitleaks/gitlab-runner-authentication-token`
+- `gitleaks/gitlab-runner-authentication-token-routable`
+- `gitleaks/gitlab-scim-token`
+- `gitleaks/gitlab-session-cookie`
+- `gitleaks/gitter-access-token`
+- `gitleaks/gocardless-api-token`
+- `gitleaks/grafana-api-key`
+- `gitleaks/grafana-cloud-api-token`
+- `gitleaks/grafana-service-account-token`
+- `gitleaks/harness-api-key`
+- `gitleaks/hashicorp-tf-api-token`
+- `gitleaks/hashicorp-tf-password`
+- `gitleaks/heroku-api-key`
+- `gitleaks/heroku-api-key-v2`
+- `gitleaks/hubspot-api-key`
+- `gitleaks/huggingface-access-token`
+- `gitleaks/huggingface-organization-api-token`
+- `gitleaks/infracost-api-token`
+- `gitleaks/intercom-api-key`
+- `gitleaks/intra42-client-secret`
+- `gitleaks/jfrog-api-key`
+- `gitleaks/jfrog-identity-token`
+- `gitleaks/jwt`
+- `gitleaks/jwt-base64`
+- `gitleaks/kraken-access-token`
+- `gitleaks/kubernetes-secret-yaml`
+- `gitleaks/kucoin-access-token`
+- `gitleaks/kucoin-secret-key`
+- `gitleaks/launchdarkly-access-token`
+- `gitleaks/linear-api-key`
+- `gitleaks/linear-client-secret`
+- `gitleaks/linkedin-client-id`
+- `gitleaks/linkedin-client-secret`
+- `gitleaks/lob-api-key`
+- `gitleaks/lob-pub-api-key`
+- `gitleaks/looker-client-id`
+- `gitleaks/looker-client-secret`
+- `gitleaks/mailchimp-api-key`
+- `gitleaks/mailgun-private-api-token`
+- `gitleaks/mailgun-pub-key`
+- `gitleaks/mailgun-signing-key`
+- `gitleaks/mapbox-api-token`
+- `gitleaks/mattermost-access-token`
+- `gitleaks/maxmind-license-key`
+- `gitleaks/messagebird-api-token`
+- `gitleaks/messagebird-client-id`
+- `gitleaks/microsoft-teams-webhook`
+- `gitleaks/netlify-access-token`
+- `gitleaks/new-relic-browser-api-token`
+- `gitleaks/new-relic-insert-key`
+- `gitleaks/new-relic-user-api-id`
+- `gitleaks/new-relic-user-api-key`
+- `gitleaks/notion-api-token`
+- `gitleaks/npm-access-token`
+- `gitleaks/nuget-config-password`
+- `gitleaks/nytimes-access-token`
+- `gitleaks/octopus-deploy-api-key`
+- `gitleaks/okta-access-token`
+- `gitleaks/openai-api-key`
+- `gitleaks/openshift-user-token`
+- `gitleaks/perplexity-api-key`
+- `gitleaks/pkcs12-file`
+- `gitleaks/plaid-api-token`
+- `gitleaks/plaid-client-id`
+- `gitleaks/plaid-secret-key`
+- `gitleaks/planetscale-api-token`
+- `gitleaks/planetscale-oauth-token`
+- `gitleaks/planetscale-password`
+- `gitleaks/postman-api-token`
+- `gitleaks/prefect-api-token`
+- `gitleaks/private-key`
+- `gitleaks/privateai-api-token`
+- `gitleaks/pulumi-api-token`
+- `gitleaks/pypi-upload-token`
+- `gitleaks/rapidapi-access-token`
+- `gitleaks/readme-api-token`
+- `gitleaks/rubygems-api-token`
+- `gitleaks/scalingo-api-token`
+- `gitleaks/sendbird-access-id`
+- `gitleaks/sendbird-access-token`
+- `gitleaks/sendgrid-api-token`
+- `gitleaks/sendinblue-api-token`
+- `gitleaks/sentry-access-token`
+- `gitleaks/sentry-org-token`
+- `gitleaks/sentry-user-token`
+- `gitleaks/settlemint-application-access-token`
+- `gitleaks/settlemint-personal-access-token`
+- `gitleaks/settlemint-service-access-token`
+- `gitleaks/shippo-api-token`
+- `gitleaks/shopify-access-token`
+- `gitleaks/shopify-custom-access-token`
+- `gitleaks/shopify-private-app-access-token`
+- `gitleaks/shopify-shared-secret`
+- `gitleaks/sidekiq-secret`
+- `gitleaks/sidekiq-sensitive-url`
+- `gitleaks/slack-app-token`
+- `gitleaks/slack-bot-token`
+- `gitleaks/slack-config-access-token`
+- `gitleaks/slack-config-refresh-token`
+- `gitleaks/slack-legacy-bot-token`
+- `gitleaks/slack-legacy-token`
+- `gitleaks/slack-legacy-workspace-token`
+- `gitleaks/slack-user-token`
+- `gitleaks/slack-webhook-url`
+- `gitleaks/snyk-api-token`
+- `gitleaks/sonar-api-token`
+- `gitleaks/sourcegraph-access-token`
+- `gitleaks/square-access-token`
+- `gitleaks/squarespace-access-token`
+- `gitleaks/stripe-access-token`
+- `gitleaks/sumologic-access-id`
+- `gitleaks/sumologic-access-token`
+- `gitleaks/telegram-bot-api-token`
+- `gitleaks/travisci-access-token`
+- `gitleaks/twilio-api-key`
+- `gitleaks/twitch-api-token`
+- `gitleaks/twitter-access-secret`
+- `gitleaks/twitter-access-token`
+- `gitleaks/twitter-api-key`
+- `gitleaks/twitter-api-secret`
+- `gitleaks/twitter-bearer-token`
+- `gitleaks/typeform-api-token`
+- `gitleaks/vault-batch-token`
+- `gitleaks/vault-service-token`
+- `gitleaks/yandex-access-token`
+- `gitleaks/yandex-api-key`
+- `gitleaks/yandex-aws-access-token`
+- `gitleaks/zendesk-secret-key`
+- `google`
+- `grafana`
+- `home-path`
+- `huggingface`
+- `iban-checksum`
+- `identity-number`
+- `international-phone`
+- `ipv4`
+- `ipv6`
+- `jwt`
+- `known-personal`
+- `known-secret`
+- `label`
+- `linear`
+- `location-gazetteer`
+- `location-label`
+- `location-residence`
+- `luhn`
+- `mac`
+- `mailchimp`
+- `messenger-handle`
+- `messenger-label`
+- `mrz`
+- `name-case`
+- `name-document`
+- `name-gazetteer`
+- `name-identity`
+- `name-initials`
+- `name-title`
+- `name-variant`
+- `national-format`
+- `national-phone`
+- `ner/offline`
+- `netrc`
+- `notion`
+- `npm`
+- `nuget`
+- `numbered-school`
+- `obfuscated-email`
+- `openai`
+- `organization-context`
+- `organization-continuation`
+- `organization-core`
+- `organization-cue`
+- `organization-duration`
+- `organization-experience`
+- `organization-experience-prose`
+- `organization-group`
+- `organization-group-cue`
+- `organization-legal-form`
+- `passport-format`
+- `passport-issuer`
+- `passport-series`
+- `personal-profile-url`
+- `phone-context`
+- `postal-address`
+- `postal-code`
+- `presidio`
+- `private-key`
+- `profile-attribute`
+- `profile-prose`
+- `proper-name-prose`
+- `putty-private-key`
+- `pypi`
+- `quoted-entropy`
+- `residence-prose`
+- `school-acronym-context`
+- `secretlint`
+- `sendgrid`
+- `shopify`
+- `slack`
+- `square`
+- `strict-age`
+- `strict-attribute`
+- `strict-date`
+- `strict-number`
+- `stripe`
+- `tel-uri`
+- `telegram`
+- `ticket-context`
+- `twilio`
+- `url-userinfo`
+- `us-ssn`
+- `vault`
+- `vercel`
+- `visa-context`
+- `vocabulary-cue`
+- `vocabulary-role`
+- `webhook`
+- `wordpress`

package/examples/custom-log-policy.mjs

@@ -0,0 +1,23 @@
+import { compileKnown, createSyncSanitizer } from '../src/index.js';
+
+// Reuse caller-owned compilation in a synchronous logging hook.
+const known = compileKnown([{ type: 'ORGANIZATION', value: 'Ромашка' }]);
+const loggerPolicy = createSyncSanitizer({
+  profile: 'documents',
+  knownPersonal: known,
+  vocabulary: { roles: ['маркетолог'] },
+  structured: 'json',
+  allowedFields: ['level', 'msg'],
+  fields: { location: 'scan' },
+  transformations: { ORGANIZATION: () => 'PrivateCompany' },
+});
+console.log(
+  loggerPolicy.sanitize(
+    JSON.stringify({
+      level: 'info',
+      msg: 'работал в Ромашке',
+      privateNote: 'caller-private text',
+      count: 1,
+    })
+  ).text
+);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@link-foundation/sensitive-data-sanitizer",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "Offline sensitive-text sanitization with credential, multilingual personal-data and scanner union detection",
   "type": "module",
   "main": "src/index.js",
@@ -25,7 +25,9 @@
     "README.md",
     "examples/shared-tool-logs.mjs",
     "examples/reversible-placeholders.mjs",
-    "examples/prose-logs.mjs"
+    "examples/prose-logs.mjs",
+    "docs/RULES.md",
+    "examples/custom-log-policy.mjs"
   ],
   "exports": {
     ".": {
@@ -40,6 +42,10 @@
       "types": "./src/legacy.d.ts",
       "browser": "./src/arithmetic.js",
       "import": "./src/legacy.js"
+    },
+    "./ner": {
+      "types": "./src/ner.d.ts",
+      "import": "./src/ner.js"
     }
   },
   "scripts": {
@@ -96,6 +102,7 @@
   "devDependencies": {
     "@changesets/cli": "^3.0.3",
     "@eslint/js": "^10.0.1",
+    "@nlpjs/ner": "4.27.0",
     "browser-commander": "0.34.1",
     "cmd-shim": "^9.0.2",
     "command-stream": "2.6.0",
@@ -151,5 +158,13 @@
   },
   "overrides": {
     "minimatch@>=7 <11": "^11.0.5"
+  },
+  "peerDependencies": {
+    "@nlpjs/ner": "^4.27.0"
+  },
+  "peerDependenciesMeta": {
+    "@nlpjs/ner": {
+      "optional": true
+    }
   }
 }

package/src/compiled-known.js

@@ -0,0 +1,55 @@
+import { failure } from './detection.js';
+import { personalRegex } from './personal-variants.js';
+import { projectText } from './projection.js';
+
+// Caller-owned frozen arrays remain compatible with maps, streams and fakes.
+// Weak keys allow the caller to release all values and compiled expressions.
+const compiled = new WeakMap();
+export const isCompiledKnown = (entries) => compiled.has(entries);
+export const compiledPatterns = (entries) => compiled.get(entries)?.patterns;
+export const projectedKnown = (entries) => compiled.get(entries)?.projected;
+
+export function compileKnown(...lists) {
+  if (lists.some((list) => !Array.isArray(list))) {
+    throw failure('ERR_CONFIG');
+  }
+  const values = new Map();
+  for (const entry of lists.flat()) {
+    if (
+      !entry ||
+      typeof entry.value !== 'string' ||
+      !entry.value ||
+      !/^[A-Z][A-Z0-9_]{0,63}$/.test(entry.type) ||
+      (entry.properCase !== undefined && typeof entry.properCase !== 'boolean')
+    ) {
+      throw failure('ERR_CONFIG');
+    }
+    const copy = Object.freeze({
+      type: entry.type,
+      value: entry.value,
+      ...(entry.properCase === undefined
+        ? {}
+        : { properCase: entry.properCase }),
+    });
+    values.set(
+      `${entry.type}\0${entry.value}\0${entry.properCase ?? false}`,
+      copy
+    );
+  }
+  const entries = Object.freeze([...values.values()]);
+  const patterns = entries.map((entry) =>
+    personalRegex(entry.value, entry.type)
+  );
+  const projected = entries.map((entry) => ({
+    ...entry,
+    value: projectText(entry.value).text,
+  }));
+  const changed = projected.some(
+    (entry, i) => entry.value !== entries[i].value
+  );
+  compiled.set(entries, {
+    patterns,
+    projected: changed ? compileKnown(projected) : entries,
+  });
+  return entries;
+}

package/src/detection.js

@@ -16,6 +16,8 @@ import { localAddress } from './public.js';
 import { detectKnownPersonal } from './personal-variants.js';
 import { detectDocumentEntities } from './document-entities.js';
 import { detectProseAndDocuments } from './proper-names.js';
+import { builtInEnabled } from './policy.js';
+import { detectStrict } from './strict.js';
 
 export const REDACTED = '[REDACTED]';
 export const failure = (code = 'ERR_SANITIZATION', location = {}) =>
@@ -137,6 +139,12 @@ function detectCookies(text, emit) {
 }
 
 export function nativeDetect(text, options, emit) {
+  const output = emit;
+  emit = (finding) => {
+    if (builtInEnabled(finding, options)) {
+      output(finding);
+    }
+  };
   const credential = { type: 'SECRET', category: 'credential' };
   for (const value of options.knownSecrets ?? []) {
     collectMatches(text, new RegExp(escapePattern(value), 'g'), emit, {
@@ -179,6 +187,7 @@ export function nativeDetect(text, options, emit) {
     detectIdentity(text, emit);
     detectNational(text, emit);
     detectPersonalContext(text, emit);
+    detectStrict(text, emit, options);
   }
   detectCredentialFormats(text, emit);
   // detect-secrets-style quoted entropy detection; bare commit hashes and