@doccloak/core 0.8.0 → 0.10.0

package/dist/types.d.ts

@@ -1,4 +1,4 @@
-export type EntityType = 'PERSON' | 'EMAIL' | 'PHONE' | 'SSN' | 'CREDIT_CARD' | 'DATE' | 'CURRENCY' | 'IP_ADDRESS' | 'IBAN' | 'ADDRESS' | 'COMPANY' | 'OTHER';
+export type EntityType = 'PERSON' | 'EMAIL' | 'PHONE' | 'SSN' | 'CREDIT_CARD' | 'DATE' | 'CURRENCY' | 'IP_ADDRESS' | 'IBAN' | 'ADDRESS' | 'COMPANY' | 'SECRET' | 'API_KEY' | 'OTHER';
 export interface DetectedEntity {
     type: EntityType;
     value: string;

package/dist/types.js

@@ -10,6 +10,8 @@ export const ENTITY_COLORS = {
     IBAN: '#A16207',
     ADDRESS: '#DC2626',
     COMPANY: '#DB2777',
+    SECRET: '#E11D48',
+    API_KEY: '#059669',
     OTHER: '#6B7280',
 };
 export const ENTITY_LABELS = {
@@ -24,5 +26,7 @@ export const ENTITY_LABELS = {
     IBAN: 'IBAN',
     ADDRESS: 'Address',
     COMPANY: 'Company',
+    SECRET: 'Secret',
+    API_KEY: 'API Key',
     OTHER: 'Other',
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doccloak/core",
-  "version": "0.8.0",
+  "version": "0.10.0",
   "description": "Environment-agnostic document anonymization engine for the DocCloak ecosystem.",
   "license": "Apache-2.0",
   "author": "Witold Lojek",
@@ -12,6 +12,10 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./restore-tokens": {
+      "types": "./dist/restore-tokens.d.ts",
+      "default": "./dist/restore-tokens.js"
+    },
     "./dom": {
       "types": "./dist/dom/index.d.ts",
       "default": "./dist/dom/index.js"
@@ -57,6 +61,9 @@
     "postpack": "node scripts/package-exports.mjs src",
     "rules:gen": "node scripts/generate-rules-module.mjs",
     "rules:check": "node scripts/generate-rules-module.mjs --check",
+    "bench": "node --experimental-strip-types eval/run.mts",
+    "bench:multilang": "node --experimental-strip-types eval/multilang.mts",
+    "bench:data": "node eval/download.mjs",
     "test": "vitest run",
     "test:watch": "vitest",
     "typecheck": "tsc --noEmit"
@@ -69,7 +76,7 @@
   "dependencies": {
     "cfb": "1.2.2",
     "jszip": "3.10.1",
-    "onnxruntime-web": "1.19.2",
+    "onnxruntime-web": "1.29.0",
     "tesseract.js": "7.0.0"
   }
 }

package/rules/gb.json

@@ -189,6 +189,23 @@
         "Tesco Ltd",
         "Deloitte LLP"
       ]
+    },
+    {
+      "id": "regex:gb:driving_licence",
+      "entityType": "SSN",
+      "pattern": "\\b[A-Z9]{5}\\s?\\d{6}\\s?[A-Z9]{2}\\d[A-Z]{2}\\b",
+      "flags": "g",
+      "confidence": 0.8,
+      "domains": [
+        "identity"
+      ],
+      "description": "UK driving licence number (16 chars: surname block, encoded date of birth, initials, check chars)",
+      "examples": [
+        "MORGA657054SM9IJ",
+        "SMITH 751125 JM9AB"
+      ],
+      "falsePositiveNotes": "Best-effort: other 16-char uppercase codes can match the shape; the validator checks the embedded date-of-birth digit block",
+      "validate": "ukDrivingLicence"
     }
   ]
 }

package/rules/schema.json

@@ -41,6 +41,8 @@
             "IBAN",
             "ADDRESS",
             "COMPANY",
+            "SECRET",
+            "API_KEY",
             "OTHER"
           ]
         },

package/rules/universal.json

@@ -274,6 +274,175 @@
         "75008 Paris"
       ],
       "falsePositiveNotes": "May match non-address number+word sequences"
+    },
+    {
+      "id": "regex:universal:aws_access_key",
+      "entityType": "API_KEY",
+      "pattern": "\\b(?:AKIA|ASIA|ABIA|ACCA|AGPA|AIDA|AIPA|ANPA|ANVA|AROA)[0-9A-Z]{16}\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "AWS access key ID (AKIA/ASIA + 16 uppercase alphanumeric)",
+      "examples": [
+        "AKIAIOSFODNN7EXAMPLE",
+        "ASIAJEXAMPLEKEY12345"
+      ]
+    },
+    {
+      "id": "regex:universal:github_token",
+      "entityType": "API_KEY",
+      "pattern": "\\b(?:gh[pousr]_[A-Za-z0-9]{36,255}|github_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59})\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "GitHub token (classic ghp_/gho_/ghu_/ghs_/ghr_ or fine-grained github_pat_)",
+      "examples": [
+        "ghp_16C7e42F292c6912E7710c838347Ae178B4a",
+        "github_pat_11ABCDEFG0123456789abc_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456"
+      ]
+    },
+    {
+      "id": "regex:universal:slack_token",
+      "entityType": "API_KEY",
+      "pattern": "\\bxox[abprs]-[A-Za-z0-9-]{10,64}\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "Slack token (xoxb- bot, xoxp- user, xoxa-/xoxr-/xoxs- variants)",
+      "examples": [
+        "xoxb-NotARealBotToken-DocCloakTestFixture",
+        "xoxp-FakeUserToken-DocCloakTestFixture00"
+      ]
+    },
+    {
+      "id": "regex:universal:openai_api_key",
+      "entityType": "API_KEY",
+      "pattern": "\\bsk-(?:(?:proj|svcacct|admin)-[A-Za-z0-9_-]{40,}|[A-Za-z0-9]{40,})\\b",
+      "flags": "g",
+      "confidence": 0.92,
+      "domains": [
+        "technical"
+      ],
+      "description": "OpenAI API key (legacy sk- + 40+ alphanumeric, or sk-proj-/sk-svcacct-/sk-admin- project keys)",
+      "examples": [
+        "sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGh",
+        "sk-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKl"
+      ],
+      "falsePositiveNotes": "Anthropic sk-ant- keys never reach the alphanumeric-only branch (hyphen after ant stops it); the dedicated anthropic rule wins overlap resolution via its longer prefix"
+    },
+    {
+      "id": "regex:universal:anthropic_api_key",
+      "entityType": "API_KEY",
+      "pattern": "\\bsk-ant-[A-Za-z0-9_-]{24,}\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "Anthropic API key (sk-ant- prefix)",
+      "examples": [
+        "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789-AbCdEfGh"
+      ]
+    },
+    {
+      "id": "regex:universal:google_api_key",
+      "entityType": "API_KEY",
+      "pattern": "\\bAIza[0-9A-Za-z_-]{35}\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "Google API key (AIza + 35 characters)",
+      "examples": [
+        "AIzaSyD-9tSrke72PouQMnMX-a7eZSW0jkFMBWY"
+      ]
+    },
+    {
+      "id": "regex:universal:private_key_block",
+      "entityType": "SECRET",
+      "pattern": "-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----[\\s\\S]*?-----END (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----",
+      "flags": "g",
+      "confidence": 0.99,
+      "domains": [
+        "technical"
+      ],
+      "description": "PEM private key block (RSA/EC/DSA/OPENSSH/ENCRYPTED/PGP), including SSH private keys",
+      "examples": [
+        "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA0Z3VS5JJcds3xfn\n-----END RSA PRIVATE KEY-----",
+        "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmU\n-----END OPENSSH PRIVATE KEY-----"
+      ]
+    },
+    {
+      "id": "regex:universal:jwt",
+      "entityType": "SECRET",
+      "pattern": "\\beyJ[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\.[A-Za-z0-9_-]{8,}\\b",
+      "flags": "g",
+      "confidence": 0.95,
+      "domains": [
+        "technical"
+      ],
+      "description": "JSON Web Token (three base64url segments, header starts with eyJ = base64 of '{\")",
+      "examples": [
+        "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
+      ]
+    },
+    {
+      "id": "regex:universal:connection_string_creds",
+      "entityType": "SECRET",
+      "pattern": "\\b[a-zA-Z][a-zA-Z0-9+.-]{1,30}:\\/\\/[^\\s:@\\/]{1,64}:[^\\s@]{1,128}@[^\\s\"']{1,256}",
+      "flags": "g",
+      "confidence": 0.9,
+      "domains": [
+        "technical"
+      ],
+      "description": "Connection string with inline credentials (scheme://user:password@host)",
+      "examples": [
+        "postgres://admin:S3cr3tPass@db.example.com:5432/mydb",
+        "mongodb+srv://appuser:hunter2pass@cluster0.abc.mongodb.net/prod"
+      ],
+      "falsePositiveNotes": "Plain URLs without a user:password pair never match (the colon between user and password is required before the @)"
+    },
+    {
+      "id": "regex:universal:secret_assignment",
+      "entityType": "SECRET",
+      "pattern": "\\b[a-zA-Z0-9_-]{0,32}?(?:api[_-]?key|apikey|secret|token|passwd|password|credential|auth|access[_-]?key|client[_-]?secret|private[_-]?key|signing[_-]?key)[a-zA-Z0-9_-]{0,32}[\"']?\\s*[:=]>?\\s*[\"']?[A-Za-z0-9+\\/_=-]{16,}",
+      "flags": "gi",
+      "confidence": 0.75,
+      "domains": [
+        "technical"
+      ],
+      "description": "Secret-ish keyword assignment (api_key = ..., password: ..., aws_secret_access_key: ...) with a 16+ char value; the value is Shannon-entropy gated by the secretAssignment validator",
+      "examples": [
+        "api_key = \"9fQz3kX7Lm2Pw8Rt5Vy1Jh6Nd4Bg0Cs\"",
+        "aws_secret_access_key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+        "session_token=a3f8c2d914b7e605d2c8a1f4b9e7d305"
+      ],
+      "falsePositiveNotes": "The keyword and separator are part of the match (lookbehind is banned for new rules), so the config key name is redacted along with the value. Low-entropy placeholder values (your_api_key_here, changeme) are rejected by the validator.",
+      "validate": "secretAssignment"
+    },
+    {
+      "id": "regex:universal:high_entropy_token",
+      "entityType": "SECRET",
+      "pattern": "\\b[A-Za-z0-9][A-Za-z0-9+\\/]{39,}={0,2}",
+      "flags": "g",
+      "confidence": 0.6,
+      "domains": [
+        "technical"
+      ],
+      "description": "Bare high-entropy base64/alphanumeric token (40+ chars); the highEntropyToken validator requires mixed character classes and Shannon entropy >= 4.5 bits/char",
+      "examples": [
+        "Zq7PmX2vR9kLtY4wNb8QcJ3fHd6GsA1uEi5oT0xVrKp",
+        "kM3nP8qR2sT7uV1wX5yZ9aB4cD6eF0gH1jK2lM3nP4qR5sT6"
+      ],
+      "falsePositiveNotes": "Low confidence by design. Git SHAs and UUID fragments are single-case hex (rejected by the class-mix check); base64 data blobs exceed the 256 char cap; identifiers and prose fall below the entropy threshold.",
+      "validate": "highEntropyToken"
     }
   ]
 }