@privacyscrubber/mcp-server 1.7.2 → 1.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config-schema.json +10 -0
- package/index.js +222 -41
- package/package.json +5 -1
- package/ps-license-manager.js +263 -0
- package/ps-pii-engine.cjs +823 -0
- package/ps-pii-engine.js +823 -0
- package/scrubber-core.cjs +1419 -11
package/ps-pii-engine.js
ADDED
|
@@ -0,0 +1,823 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* PrivacyScrubber Shared PII Engine
|
|
3
|
+
* Shared between Website Worker, Chrome Extension, and MCP Server.
|
|
4
|
+
* Responsible for running Regex checks, overlap resolution, and reverse scrubbing.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
(function() {
|
|
8
|
+
if (typeof window !== 'undefined' && window.PrivacyScrubberEngine) return;
|
|
9
|
+
|
|
10
|
+
// --- START DEFAULT RULES ---
|
|
11
|
+
/**
|
|
12
|
+
* PrivacyScrubber Unified PII Detection Rules
|
|
13
|
+
* Shared between Website Worker, Chrome Extension, and Build Pipeline.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
let DEVOPS_SECRETS = [
|
|
17
|
+
// Secrets & API Keys
|
|
18
|
+
{ name: 'AWS Credentials', type: 'SECRET', regex: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA)[A-Z0-9]{16}\b/g },
|
|
19
|
+
{ name: 'JSON Web Token (JWT)', type: 'SECRET', regex: /\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b/g },
|
|
20
|
+
{ name: 'API Token/Key (GitHub/Slack/NPM)', type: 'SECRET', regex: /\b(?:ghp|gho|ghu|ghs|ghr|glpat|npm|xox[baprs])[-_][A-Za-z0-9_]{10,}\b/g },
|
|
21
|
+
{ name: 'Stripe API Key', type: 'SECRET', regex: /\b(?:[rs]k)_(?:test|live)_[a-zA-Z0-9]{24,}\b/g },
|
|
22
|
+
{ name: 'Generic Secret/Key', type: 'SECRET', regex: /\b(sk|pk|secret|key|token|auth)(?:[-_][a-zA-Z0-9_-]{5,}|(?=[a-zA-Z0-9_-]{5,}\b)(?=[a-zA-Z_-]*[0-9])[a-zA-Z0-9_-]{5,})\b/gi },
|
|
23
|
+
{ name: 'Hash / Hex Key (32-64 chars)', type: 'SECRET', regex: /\b[a-fA-F0-9]{32,64}\b/g },
|
|
24
|
+
{ name: 'CVE Identifier', type: 'SECRET', regex: /\bCVE-\d{4}-\d{4,}\b/gi },
|
|
25
|
+
{ name: 'Cryptographic Hash', type: 'SECRET', regex: /\b(MD5|SHA1|SHA256)[:\s][a-f0-9]{32,64}\b/gi },
|
|
26
|
+
{ name: 'Database/API Secret', type: 'SECRET', regex: /\b(DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*[^\s"']+\b/gi },
|
|
27
|
+
{ name: 'Proprietary IP / Confidential', type: 'SECRET', regex: /\b(CONFIDENTIAL|PROPRIETARY|TRADE SECRET|DO NOT DISTRIBUTE|INTERNAL USE ONLY)\b/gi },
|
|
28
|
+
{ name: 'Private Cryptographic Key', type: 'SECRET', regex: /-----BEGIN (?:RSA |EC |PGP |DSA )?PRIVATE KEY-----/g }
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
let REGEX_RULES = [
|
|
32
|
+
...DEVOPS_SECRETS,
|
|
33
|
+
// Emails
|
|
34
|
+
{ type: 'EMAIL', regex: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g },
|
|
35
|
+
|
|
36
|
+
// Financial Data
|
|
37
|
+
{ type: 'FINANCIAL', regex: /(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[0-9,.]+\b/g },
|
|
38
|
+
{ type: 'FINANCIAL', regex: /\b[A-Z]{4}(?:AD|AE|AF|AG|AI|AL|AM|AO|AQ|AR|AS|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BJ|BL|BM|BN|BO|BQ|BR|BS|BT|BV|BW|BY|BZ|CA|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|CR|CU|CV|CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EE|EG|EH|ER|ES|ET|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|IO|IQ|IR|IS|IT|JE|JM|JO|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MF|MG|MH|MK|ML|MM|MN|MO|MP|MQ|MR|MS|MT|MU|MV|MW|MX|MY|MZ|NA|NC|NE|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|SS|ST|SV|SX|SY|SZ|TC|TD|TF|TG|TJ|TK|TL|TM|TN|TO|TR|TT|TV|TW|TZ|UA|UG|UM|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|YE|YT|ZA|ZM|ZW)[A-Z2-9][A-NP-Z0-9]([A-Z0-9]{3})?\b/g },
|
|
39
|
+
{ type: 'FINANCIAL', regex: /\b\d{9}\b/g },
|
|
40
|
+
{ type: 'FINANCIAL', regex: /\b[A-Z]{2}[0-9]{2}[a-zA-Z0-9]{4}[0-9]{7}[a-zA-Z0-9]{0,16}\b/g },
|
|
41
|
+
{ type: 'FINANCIAL', regex: /\bPORTFOLIO[-_][A-Z0-9]{5,}\b/gi },
|
|
42
|
+
{ type: 'FINANCIAL', regex: /\b(?:\d[ -]?){13,19}\b/g },
|
|
43
|
+
{ type: 'FINANCIAL', regex: /\b(?:1|3|bc1)[a-zA-HJ-NP-Z0-9]{25,39}\b/g },
|
|
44
|
+
{ type: 'FINANCIAL', regex: /\b0x[a-fA-F0-9]{40}\b/g },
|
|
45
|
+
|
|
46
|
+
// Legal & Court
|
|
47
|
+
{ type: 'LEGAL', regex: /\bCASE[-_][A-Z0-9]{4,}\b/gi },
|
|
48
|
+
{ type: 'LEGAL', regex: /\bMATTER[-_][A-Z0-9]{4,}\b/gi },
|
|
49
|
+
{ type: 'LEGAL', regex: /\b[A-Z]{2,4}[- ]?\d{2}[- ]?\d{4,}\b/g },
|
|
50
|
+
{ type: 'PRIVILEGE', regex: /ATTORNEY[- ]CLIENT[- ]PRIVILEGE/gi },
|
|
51
|
+
|
|
52
|
+
// Professional IDs
|
|
53
|
+
{ type: 'ID', regex: /\bEEID[ -]?\d{4,}\b/gi },
|
|
54
|
+
{ type: 'ID', regex: /\bRESUME[-_]?[A-Z0-9]{4,}\b/gi },
|
|
55
|
+
{ type: 'ID', regex: /\bLEAD[-_][A-Z0-9]{5,}\b/gi },
|
|
56
|
+
{ type: 'ID', regex: /\bCAMPAIGN[-_][A-Z0-9]{4,}\b/gi },
|
|
57
|
+
{ type: 'ID', regex: /\bDEAL[-_][A-Z0-9]{4,}\b/gi },
|
|
58
|
+
{ type: 'ID', regex: /\bENTITY[-_][0-9]{4,}\b/gi },
|
|
59
|
+
{ type: 'ID', regex: /\bOPPORTUNITY[-_][A-Z0-9]{5,}\b/gi },
|
|
60
|
+
{ type: 'ID', regex: /\bPROSPECT[-_][A-Z0-9]{5,}\b/gi },
|
|
61
|
+
{ type: 'ID', regex: /\bTICKET[-_][A-Z0-9]{5,}\b/gi },
|
|
62
|
+
{ type: 'ID', regex: /\bZENDESK[-_][0-9]{4,}\b/gi },
|
|
63
|
+
{ type: 'ID', regex: /\bEMP[-_]\d{3,}\b/gi },
|
|
64
|
+
{ type: 'ID', regex: /\bLIS[-]?\d{6,}\b/gi },
|
|
65
|
+
{ type: 'ID', regex: /\bPARCEL[-]?\d{5,}\b/gi },
|
|
66
|
+
{ type: 'ID', regex: /\bAGENT[-_][A-Z0-9]{4,}\b/gi },
|
|
67
|
+
{ type: 'ID', regex: /\bTASK[-_][A-Z0-9]{5,}\b/gi },
|
|
68
|
+
{ type: 'ID', regex: /\bSTUDENT[-_][0-9]{5,}\b/gi },
|
|
69
|
+
{ type: 'ID', regex: /\bCOURSE[-_][A-Z0-9]{4,}\b/gi },
|
|
70
|
+
{ type: 'ID', regex: /\bINSTANCE[-_]ID[-_][a-z0-9-]{10,}\b/gi },
|
|
71
|
+
{ type: 'ID', regex: /\bENV[-_][A-Z0-9]{3,}\b/gi },
|
|
72
|
+
{ type: 'PRIVACY', regex: /\bTENANT[-_]ID[-_][0-9]{4,}\b/gi },
|
|
73
|
+
|
|
74
|
+
// Addresses & Locations
|
|
75
|
+
{ type: 'ADDRESS', regex: /\b\d{1,6}\s+(?:[A-Z][a-zA-Z0-9.-]*\s+){1,3}(?:St|Street|Ave|Avenue|Blvd|Boulevard|Rd|Road|Ln|Lane|Dr|Drive|Way|Ct|Court|Pl|Place|Terrace|Pkwy|Parkway|Sq|Square)\b/gi },
|
|
76
|
+
{ type: 'LOCATION', regex: /\b[A-Z][a-zA-Z\s.-]{2,25},\s*[A-Z]{2}\b/g },
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
// PHI & Medical
|
|
80
|
+
{ type: 'PHI', regex: /\bMRN[ -]?\d{6,}\b/gi },
|
|
81
|
+
{ type: 'PHI', regex: /\b[A-TV-Z]\d{2}[. ]?\d[A-Z0-9]?\b/g },
|
|
82
|
+
{ type: 'PHI', regex: /\b[A-Z]{2,3}\d{6,8}\b/g },
|
|
83
|
+
{ type: 'PHI', regex: /\bNHS[ -]?\d{3}[ -]?\d{3}[ -]?\d{4}\b/gi },
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
// Copyrights
|
|
87
|
+
{ type: 'COPYRIGHT', regex: /\bPROJECT[-_][A-Z0-9]{5,}\b/gi },
|
|
88
|
+
{ type: 'COPYRIGHT', regex: /\b(DRAFT|ASSET|SCRIPT)[-_][0-9]{4,}\b/gi },
|
|
89
|
+
|
|
90
|
+
// General Privacy & Dates
|
|
91
|
+
{ type: 'PRIVACY', regex: /\b(GDPR|HIPAA|CCPA|SOC2)[-_]AUDIT[-_]\d{4}\b/gi },
|
|
92
|
+
{ type: 'PRIVACY', regex: /\bPOLICY[-_][A-Z0-9]{5,}\b/gi },
|
|
93
|
+
{ type: 'PRIVACY', regex: /\bGRADE[S]?\s*:\s*[A-DF][+-]?\b/gi },
|
|
94
|
+
{ type: 'PRIVACY', regex: /\b(DOB|BIRTHDAY)[:\s]*[0-9./-]{6,10}\b/gi },
|
|
95
|
+
{ type: 'PRIVACY', regex: /\b(PASSWORD|PWD|SECRET)[:\s]*[\S]{4,}\b/gi },
|
|
96
|
+
|
|
97
|
+
// Standard IDs (SSN, EIN, Passport, VAT)
|
|
98
|
+
{ type: 'ID', regex: /\b\d{3}-\d{2}-\d{4}\b/g },
|
|
99
|
+
{ type: 'ID', regex: /(?:\/|[A-Za-z]:\\)[\w\-. ]+(?:[\/\\][\w\-. ]+)+/g },
|
|
100
|
+
{ type: 'ID', regex: /\b[A-CEGHJ-PR-TW-Z]{1}[A-CEGHJ-NPR-TW-Z]{1}[0-9]{6}[A-DFM]{1}\b/gi },
|
|
101
|
+
{ type: 'ID', regex: /\b[A-Z]{2}[0-9]{6,12}\b/gi },
|
|
102
|
+
{ type: 'ID', regex: /[A-Z0-9<]{30,44}/g },
|
|
103
|
+
|
|
104
|
+
// IT / Technical IDs
|
|
105
|
+
{ type: 'IP', regex: /\b(?:\d{1,3}\.){3}\d{1,3}\b/g },
|
|
106
|
+
{ type: 'IP', regex: /\b(?:[a-fA-F0-9]{1,4}:){7}[a-fA-F0-9]{1,4}\b/g },
|
|
107
|
+
{ type: 'ID', regex: /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g },
|
|
108
|
+
{ type: 'ID', regex: /(?:\B\/|\b[a-zA-Z]:\\)(?:[\w.-]+[\/\\]?)+\b/g },
|
|
109
|
+
{ type: 'ID', regex: /\b\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}(?::\d{2})?)?\b/g },
|
|
110
|
+
{ type: 'ID', regex: /\b\d{4}-\d{2}-\d{2}\b/g },
|
|
111
|
+
{ type: 'ID', regex: /\b\d{2}\/\d{2}\/\d{4}\b/g },
|
|
112
|
+
|
|
113
|
+
// Phone Numbers — must come after IDs to avoid ID rules stealing phone matches
|
|
114
|
+
{ type: 'PHONE', regex: /\b(?:\+?\d{1,3}[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g },
|
|
115
|
+
{ type: 'PHONE', regex: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}/g },
|
|
116
|
+
{ type: 'PHONE', regex: /\+?[1-9]\d{1,3}[\s.-]\(?\d{1,4}\)?[\s.-]\d{2,4}[\s.-]\d{4}/g },
|
|
117
|
+
{ type: 'PHONE', regex: /(?:\+44\s?7\d{3}|\(?07\d{3}\)?)\s?\d{3}\s?\d{3}\b/g },
|
|
118
|
+
{ type: 'PHONE', regex: /\b(?:\d{3}[-.\s]\d{4}|\(\d{3}\)\s??\d{3}[-.\s]??\d{4}|\d{3}[-.\s]??\d{3}[-.\s]??\d{4})\b/g },
|
|
119
|
+
|
|
120
|
+
// Geolocation (Lat/Long)
|
|
121
|
+
{ type: 'LOCATION', regex: /\b-?\d{1,3}\.\d{4,6}[° ]?[NSns],\s*-?\d{1,3}\.\d{4,6}[° ]?[EWew]\b/g },
|
|
122
|
+
{ type: 'LOCATION', regex: /\b(LATITUDE|LONGITUDE)[:\s]-?\d{1,3}\.\d{4,10}\b/gi },
|
|
123
|
+
|
|
124
|
+
// Driver Licenses & State IDs
|
|
125
|
+
{ type: 'ID', regex: /\bDL[ -]?\d{6,12}\b/gi },
|
|
126
|
+
{ type: 'ID', regex: /\bDRIVER[S]?\s+LICENSE[ -]?\d{6,15}\b/gi },
|
|
127
|
+
|
|
128
|
+
// Generalized Name Detection (First [Middle] Last) — max 1 middle word to avoid grabbing job titles
|
|
129
|
+
// Middle word must be: a particle (van/de/etc.), a single initial (A.), or a capitalized word of ≥2 lowercase letters
|
|
130
|
+
{ type: 'NAME', isAggressiveName: true, regex: /(?<=^|[^\p{L}\p{N}_])\p{Lu}[\p{Ll}'-]*(?:\p{Lu}[\p{Ll}'-]*)?\p{L}(?:[ \t\xA0]+(?:\p{Lu}[\p{Ll}'-]{1,}(?:\p{Lu}[\p{Ll}'-]*)?\p{L}|\p{Lu}\.?|van|von|de|di|da|la|le|del|du|der|van[ \t\xA0]+de)){0,1}[ \t\xA0]+\p{Lu}[\p{Ll}'-]*(?:\p{Lu}[\p{Ll}'-]*)?\p{L}(?:'s)?(?=[^\p{L}\p{N}_]|$)(?![ \t\xA0]*:)/gu },
|
|
131
|
+
// All-Caps Names (2–3 words, each ≥3 chars) — prevents acronym chains like "GPA ROI AUM" (Unicode-safe)
|
|
132
|
+
{ type: 'NAME', regex: /(?<=^|[^\p{L}\p{N}_])\p{Lu}{3,}(?:[\p{Lu}'-]*\p{Lu})?(?:[ \t\xA0]+\p{Lu}{3,}(?:[\p{Lu}'-]*\p{Lu})?){1,2}(?:'s)?(?=[^\p{L}\p{N}_]|$)(?![ \t\xA0]*:)/gu },
|
|
133
|
+
// ALL-CAPS first + optional middle initial(s) with optional spaces + ALL-CAPS last name
|
|
134
|
+
// E.g. "KKKKK I. MMMMMM", "KKKKK I.MMMMMM", "KKKKKI.MMMMMM" (PDF extraction spacing anomalies)
|
|
135
|
+
{ type: 'NAME', regex: /(?<=^|[^\p{L}\p{N}_])\p{Lu}{2,}(?:[\p{Lu}'-]*\p{Lu})?(?:(?:[ \t\xA0]*\p{Lu}\.)+[ \t\xA0]*|[ \t\xA0]+)\p{Lu}{2,}(?:[\p{Lu}'-]*\p{Lu})?(?:'s)?(?=[^\p{L}\p{N}_]|$)(?![ \t\xA0]*:)/gu },
|
|
136
|
+
// ALL-CAPS first name + optional middle initial(s) with optional spaces + Mixed-Case last name
|
|
137
|
+
// E.g. "JOSEPH I. Casalvieri", "JOSEPH I.Casalvieri", "JOSEPHI.Casalvieri" (PDF extraction spacing anomalies)
|
|
138
|
+
{ type: 'NAME', regex: /(?<=^|[^\p{L}\p{N}_])\p{Lu}{2,}(?:[\p{Lu}'-]*\p{Lu})?(?:(?:[ \t\xA0]*\p{Lu}\.)+[ \t\xA0]*|[ \t\xA0]+)\p{Lu}\p{Ll}[\p{Ll}'-]*(?:\p{Lu}[\p{Ll}'-]*)?\p{L}(?:'s)?(?=[^\p{L}\p{N}_]|$)(?![ \t\xA0]*:)/gu },
|
|
139
|
+
// Single-Word Names with Honorifics (with or without period) (Unicode-safe)
|
|
140
|
+
{ type: 'NAME', regex: /(?<=^|[^\p{L}\p{N}_])(?:Mr|Mrs|Ms|Dr|Prof|Hon|Mr\.|Mrs\.|Ms\.|Dr\.|Prof\.|Hon\.)[ \t\xA0]+\p{Lu}[\p{Ll}'-]*(?:\p{Lu}[\p{Ll}'-]*)?\p{L}(?=[^\p{L}\p{N}_]|$)(?![ \t\xA0]*:)/gu },
|
|
141
|
+
|
|
142
|
+
// Contextual Names (Emergency Contacts, Spouses, Children, Tenants, etc.) - Uses capture group m[1]
|
|
143
|
+
{ type: 'NAME', isContextName: true, regex: /(?:Emergency Contact|Contact(?: Name)?|Spouse|Child|Parent|Guardian|Relationship|Kin|Tenant|Landlord|Buyer|Seller|Plaintiff|Defendant|Testator)[\s:]+([A-Z][a-z]*(?:\s+[A-Z][a-z]*)?)/g }
|
|
144
|
+
];
|
|
145
|
+
|
|
146
|
+
let PROFILE_RULES = {
|
|
147
|
+
general: [],
|
|
148
|
+
legal: [
|
|
149
|
+
{ type: 'LEGAL', regex: /\bCASE[-_][A-Z0-9]{4,}\b/gi },
|
|
150
|
+
{ type: 'LEGAL', regex: /\bMATTER[-_][A-Z0-9]{4,}\b/gi },
|
|
151
|
+
{ type: 'LEGAL', regex: /\b[A-Z]{2,4}[- ]?\d{2}[- ]?\d{4,}\b/g },
|
|
152
|
+
{ type: 'PRIVILEGE', regex: /ATTORNEY[- ]CLIENT[- ]PRIVILEGE/gi }
|
|
153
|
+
],
|
|
154
|
+
hr: [
|
|
155
|
+
{ type: 'NAME', isContextName: true, regex: /(?:Candidate|Applicant|Employee|Reporting To|Manager)[\s:]+([A-Z][a-z]*(?:\s+[A-Z][a-z]*)?)/g },
|
|
156
|
+
|
|
157
|
+
{ type: 'ID', regex: /\bEEID[ -]?\d{4,}\b/gi },
|
|
158
|
+
{ type: 'ID', regex: /\bEMP[-_]\d{3,}\b/gi },
|
|
159
|
+
{ type: 'ID', regex: /\bRESUME[-_]?[A-Z0-9]{4,}\b/gi },
|
|
160
|
+
{ type: 'PRIVACY', regex: /\\b(DOB|BIRTHDAY)[:\\s]*[0-9./-]{6,10}\\b/gi },
|
|
161
|
+
{ type: 'ADDRESS', regex: /\\b\\d{1,6}\\s+(?:[A-Z][a-zA-Z0-9.-]*\\s+){1,3}(?:St|Street|Ave|Avenue|Blvd|Boulevard|Rd|Road|Ln|Lane|Dr|Drive|Way|Ct|Court|Pl|Place|Terrace|Pkwy|Parkway|Sq|Square)\\b/gi }
|
|
162
|
+
],
|
|
163
|
+
finance: [
|
|
164
|
+
{ type: 'FINANCIAL', regex: /\b[A-Z]{4}(?:AD|AE|AF|AG|AI|AL|AM|AO|AQ|AR|AS|AT|AU|AW|AX|AZ|BA|BB|BD|BE|BF|BG|BH|BI|BJ|BL|BM|BN|BO|BQ|BR|BS|BT|BV|BW|BY|BZ|CA|CC|CD|CF|CG|CH|CI|CK|CL|CM|CN|CO|CR|CU|CV|CW|CX|CY|CZ|DE|DJ|DK|DM|DO|DZ|EC|EE|EG|EH|ER|ES|ET|FI|FJ|FK|FM|FO|FR|GA|GB|GD|GE|GF|GG|GH|GI|GL|GM|GN|GP|GQ|GR|GS|GT|GU|GW|GY|HK|HM|HN|HR|HT|HU|ID|IE|IL|IM|IN|IO|IQ|IR|IS|IT|JE|JM|JO|JP|KE|KG|KH|KI|KM|KN|KP|KR|KW|KY|KZ|LA|LB|LC|LI|LK|LR|LS|LT|LU|LV|LY|MA|MC|MD|ME|MF|MG|MH|MK|ML|MM|MN|MO|MP|MQ|MR|MS|MT|MU|MV|MW|MX|MY|MZ|NA|NC|NE|NF|NG|NI|NL|NO|NP|NR|NU|NZ|OM|PA|PE|PF|PG|PH|PK|PL|PM|PN|PR|PS|PT|PW|PY|QA|RE|RO|RS|RU|RW|SA|SB|SC|SD|SE|SG|SH|SI|SJ|SK|SL|SM|SN|SO|SR|SS|ST|SV|SX|SY|SZ|TC|TD|TF|TG|TJ|TK|TL|TM|TN|TO|TR|TT|TV|TW|TZ|UA|UG|UM|US|UY|UZ|VA|VC|VE|VG|VI|VN|VU|WF|WS|YE|YT|ZA|ZM|ZW)[A-Z2-9][A-NP-Z0-9]([A-Z0-9]{3})?\b/g },
|
|
165
|
+
{ type: 'FINANCIAL', regex: /\b[A-Z]{2}[0-9]{2}[a-zA-Z0-9]{4}[0-9]{7}[a-zA-Z0-9]{0,16}\b/g },
|
|
166
|
+
{ type: 'FINANCIAL', regex: /\bPORTFOLIO[-_][A-Z0-9]{5,}\b/gi }
|
|
167
|
+
],
|
|
168
|
+
medical: [
|
|
169
|
+
{ type: 'PHI', regex: /\b(?:MRN|Patient ID|Medical Record No|Patient No)[\s:#]*([A-Za-z0-9-]+)/gi },
|
|
170
|
+
{ type: 'PRIVACY', regex: /\b(DOB|Date of Birth)[:\s]*[0-9./-]{6,10}\b/gi },
|
|
171
|
+
|
|
172
|
+
{ type: 'PHI', regex: /\bMRN[ -]?\d{6,}\b/gi },
|
|
173
|
+
{ type: 'PHI', regex: /\b[A-TV-Z]\d{2}[. ]?\d[A-Z0-9]?\b/g },
|
|
174
|
+
{ type: 'PHI', regex: /\b[A-Z]{2,3}\d{6,8}\b/g },
|
|
175
|
+
{ type: 'PHI', regex: /\bNHS[ -]?\d{3}[ -]?\d{3}[ -]?\d{4}\b/gi }
|
|
176
|
+
],
|
|
177
|
+
security: [
|
|
178
|
+
{ type: 'SECRET', regex: /\b(?:INCIDENT|BREACH)[-_: ]?ID[-_: ]?[0-9]{4,10}\b/gi }
|
|
179
|
+
],
|
|
180
|
+
marketing: [
|
|
181
|
+
{ type: 'ID', regex: /\b(?:LEAD|PROSPECT)[-_: ]*[A-Z0-9_-]*[0-9][A-Z0-9_-]*\b/gi },
|
|
182
|
+
{ type: 'ID', regex: /\b(?:CAMPAIGN|CLID|GCLID|FBCLID)[-_:=]*[A-Za-z0-9_-]{10,}\b/gi },
|
|
183
|
+
{ type: 'FINANCIAL', regex: /(?<=\b(?:LTV|CAC)[\s:]*)(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[0-9,.]+\b/gi },
|
|
184
|
+
{ type: 'ID', regex: /\b(?:SEGMENT|COHORT)[-_: ]*[0-9]{4,8}\b/gi }
|
|
185
|
+
],
|
|
186
|
+
bizops: [
|
|
187
|
+
{ type: 'ID', regex: /\b(?:DEAL|KPI|METRIC)[-_: ]?[A-Z0-9]{4,}\b/gi },
|
|
188
|
+
{ type: 'ID', regex: /\b(?:ENTITY|VENDOR|PARTNER)[-_: ]?[0-9]{4,10}\b/gi },
|
|
189
|
+
{ type: 'FINANCIAL', regex: /\b(?:REVENUE|EBITDA|PROFIT|MARGIN)[-_: ]?(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)?[0-9,.]+[KM]?\b/gi },
|
|
190
|
+
{ type: 'SECRET', regex: /\b(?:NDA|M&A|MERGER)[-_: ]?[A-Z0-9]{4,}\b/gi }
|
|
191
|
+
],
|
|
192
|
+
sales: [
|
|
193
|
+
{ type: 'ID', regex: /\bOPPORTUNITY[-_: ]?[A-Z0-9]{5,}\b/gi },
|
|
194
|
+
{ type: 'ID', regex: /\b(?:DOCUSIGN|CONTRACT)[-_: ]?[0-9A-F]{8,32}\b/gi },
|
|
195
|
+
{ type: 'FINANCIAL', regex: /\b(?:ARR|MRR|QUOTA)[\s:]?(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)?[0-9,.]+[KM]?\b/gi },
|
|
196
|
+
{ type: 'ID', regex: /\b(?:SFDC|HUBSPOT)[-_: ]?[0-9A-Z]{15,18}\b/gi }
|
|
197
|
+
],
|
|
198
|
+
support: [
|
|
199
|
+
{ type: 'ID', regex: /\bTICKET[-_:# ]?[A-Z0-9]{5,10}\b/gi },
|
|
200
|
+
{ type: 'ID', regex: /\b(?:ZENDESK|INTERCOM|JIRA)[-_:# ]?[0-9]{4,10}\b/gi },
|
|
201
|
+
{ type: 'ID', regex: /\b(?:REFUND|RETURN|RMA)[-_:# ]?[A-Z0-9]{6,12}\b/gi },
|
|
202
|
+
{ type: 'ID', regex: /\b(?:LOYALTY|REWARDS)[-_:# ]?\d{8,12}\b/gi }
|
|
203
|
+
],
|
|
204
|
+
realestate: [
|
|
205
|
+
{ type: 'ID', regex: /\b(?:MLS|LIS)[- ]?\d{6,10}\b/gi },
|
|
206
|
+
{ type: 'ID', regex: /\bPARCEL[- ]?\d{5,15}\b/gi },
|
|
207
|
+
{ type: 'ID', regex: /\bTENANT[-_]ID[-_][0-9]{4,}\b/gi },
|
|
208
|
+
{ type: 'FINANCIAL', regex: /\b(?:RENT|LEASE|ESCROW)[\s:]?(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[0-9,]{3,}\b/gi },
|
|
209
|
+
{ type: 'SECRET', regex: /\b(?:GATE|DOOR|LOBBY)[-_ ](?:CODE|PIN)[\s:]?\d{4,6}\b/gi }
|
|
210
|
+
],
|
|
211
|
+
compliance: [
|
|
212
|
+
{ type: 'SECRET', regex: /\b(?:GDPR|HIPAA|CCPA|SOC2|ISO27001)[-_: ]?AUDIT[-_: ]?\d{4}\b/gi },
|
|
213
|
+
{ type: 'SECRET', regex: /\b(?:DPA|POLICY)[-_: ]?[A-Z0-9]{5,15}\b/gi },
|
|
214
|
+
{ type: 'ID', regex: /\b(?:SAR|DSAR)[-_\/: ]?[A-Z0-9-/]+\b/gi }
|
|
215
|
+
],
|
|
216
|
+
ccpa: [
|
|
217
|
+
{ type: 'ID', regex: /\bDL[ -]?\d{6,12}\b/gi },
|
|
218
|
+
{ type: 'LOCATION', regex: /\b-?\d{1,3}\.\d{4,6}[° ]?[NSns],\s*-?\d{1,3}\.\d{4,6}[° ]?[EWew]\b/g },
|
|
219
|
+
{ type: 'PRIVACY', regex: /\b(?:CCPA|CPRA)[-_: ]?OPT[-_ ]OUT\b/gi },
|
|
220
|
+
{ type: 'ID', regex: /\bACCOUNT[ -]?(?:ID|NUM|NUMBER)[:\s][A-Z0-9]{6,20}\b/gi }
|
|
221
|
+
],
|
|
222
|
+
engineering: [
|
|
223
|
+
{ type: 'SECRET', regex: /(?<=\b(?:DB|POSTGRES|REDIS|MYSQL|AWS|SECRET|PASSWORD|TOKEN|API|KEY)[A-Z0-9_]*\s*[:=]\s*["']?)[A-Za-z0-9_-]{10,}/gi },
|
|
224
|
+
{ type: 'ID', regex: /\b[a-z0-9._-]+\/[a-z0-9._-]+:[a-z0-9._-]+\b/g },
|
|
225
|
+
{ type: 'ID', regex: /\b[a-fA-F0-9]{40}\b/g },
|
|
226
|
+
{ type: 'ID', regex: /\b[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.svc\.cluster\.local\b/g }
|
|
227
|
+
],
|
|
228
|
+
agents: [
|
|
229
|
+
{ type: 'ID', regex: /\b(?:AGENT|VECTOR|EMBEDDING)[-_: ]?[A-Z0-9]{8,}\b/gi },
|
|
230
|
+
{ type: 'ID', regex: /\bTASK[-_: ]?[A-Z0-9]{5,15}\b/gi },
|
|
231
|
+
{ type: 'SECRET', regex: /\b(?:SYS_PROMPT|SYSTEM_PROMPT|OPENAI_API_KEY)[-_: ]?[A-Za-z0-9_-]{10,}\b/gi }
|
|
232
|
+
],
|
|
233
|
+
academic: [
|
|
234
|
+
{ type: 'ID', regex: /\b(?:STUDENT|ALUMNI)[-_:# ]?[0-9]{5,10}\b/gi },
|
|
235
|
+
{ type: 'ID', regex: /\bCOURSE[-_:# ]?[A-Z]{3,4}[ ]?[0-9]{3,4}\b/gi },
|
|
236
|
+
{ type: 'ID', regex: /\b(?:FERPA|IRB)[-_:# ]?[A-Z0-9]{5,10}\b/gi },
|
|
237
|
+
{ type: 'ID', regex: /\bGRADE[S]?[\s:][A-DF][+-]?\b/gi }
|
|
238
|
+
],
|
|
239
|
+
creative: [
|
|
240
|
+
{ type: 'SECRET', regex: /\b(?:PROJECT|DRAFT|ASSET|SCRIPT|IP)[-_: ]?[0-9]{4,10}\b/gi },
|
|
241
|
+
{ type: 'SECRET', regex: /\b(?:SPOILER|UNRELEASED|EMBARGOED)\b/gi },
|
|
242
|
+
{ type: 'NAME', regex: /\b(?:GHOSTWRITER|SOURCE)[:\s][A-Z][a-z]+ [A-Z][a-z]+\b/g }
|
|
243
|
+
],
|
|
244
|
+
tech: [
|
|
245
|
+
{ type: 'ID', regex: /\b(?:INSTANCE|NODE|CLUSTER)[-_]ID[-_][a-z0-9-]{10,}\b/gi },
|
|
246
|
+
{ type: 'ID', regex: /\bENV[-_: ]?(?:PROD|STAGING|DEV|TEST|QA)\b/gi },
|
|
247
|
+
{ type: 'SECRET', regex: /\b(?:CONFIG|KUBECONFIG|TFSTATE)[-_: ]?[A-Z0-9]{6,15}\b/gi }
|
|
248
|
+
],
|
|
249
|
+
personal: [
|
|
250
|
+
{ type: 'ID', regex: /\b(?:DOB|BIRTHDAY)[\s:]*[0-9./-]{6,10}\b/gi },
|
|
251
|
+
{ type: 'SECRET', regex: /\b(?:PASSWORD|PWD|SECRET|PIN)[\s:]*[\S]{4,20}\b/gi },
|
|
252
|
+
{ type: 'PHONE', regex: /\b(?:WIFE|HUSBAND|PARTNER|MOM|DAD)[\s:]+(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b/gi }
|
|
253
|
+
],
|
|
254
|
+
wealthmgmt: [
|
|
255
|
+
{ type: 'LEGAL', regex: /\b(?:THE\s+)?[A-Z][A-Z\s.'-]{4,}\s+(?:(?:REVOCABLE|IRREVOCABLE|LIVING|FAMILY|TESTAMENTARY|CHARITABLE|GENERATION-SKIPPING)\s+)*TRUST\b/g },
|
|
256
|
+
// Mixed-Caps name: All-Caps first name (3+ chars) + optional initials + Mixed-Case last name
|
|
257
|
+
// Handles PDF-rendered trust names where OCR/extraction produces e.g. "KKKKK I. Mmmmmmm"
|
|
258
|
+
{ type: 'NAME', regex: /(?<=^|[^\p{L}\p{N}_])\p{Lu}{3,}(?:[\p{Lu}'-]*\p{Lu})?(?:[ \t\xA0]+\p{Lu}\.)*[ \t\xA0]+\p{Lu}\p{Ll}[\p{Ll}'-]*(?:\p{Lu}[\p{Ll}'-]*)?\p{L}(?:'s)?(?=[^\p{L}\p{N}_]|$)/gu },
|
|
259
|
+
{ type: 'FINANCIAL', regex: /\b(?:ABA|Routing(?:\s+No)?|RTN)[:\s#]*\d{9}\b/gi },
|
|
260
|
+
{ type: 'FINANCIAL', regex: /\b(?:Account|Acct\.?)[:\s#]*\d{4}[-\s]?\d{4}[-\s]?\d{2,6}\b/gi },
|
|
261
|
+
{ type: 'FINANCIAL', regex: /\b(?:Policy|Contract)\s*(?:No\.?|Number|#)[:\s]+[A-Z0-9][A-Z0-9\-]{3,14}\b/gi },
|
|
262
|
+
{ type: 'ID', regex: /\bCRD\s*#?\s*\d{4,8}\b/gi },
|
|
263
|
+
{ type: 'FINANCIAL', regex: /\b(?:Annual\s+(?:Distribution|Withdrawal)|RMD|Required\s+Minimum\s+Distribution)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.][\d]{2})?\b/gi },
|
|
264
|
+
{ type: 'FINANCIAL', regex: /\b(?:Portfolio|Market\s+Value|Net\s+Worth|AUM|Total\s+Assets)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,.]+[KMB]?\b/gi },
|
|
265
|
+
{ type: 'FINANCIAL', regex: /\b(?:Roth\s+)?(?:IRA|401k|401\(k\)|403b|403\(b\)|SEP|SIMPLE)\s*(?:Account|Acct|Plan)?\s*(?:No|Number|#)?[:\s#]*[A-Z0-9]{4,15}\b/gi }
|
|
266
|
+
],
|
|
267
|
+
insurance: [
|
|
268
|
+
{ type: 'ID', regex: /\b(?:Claim|CLM)[:\s#-]*[A-Z0-9]{6,15}\b/gi },
|
|
269
|
+
{ type: 'ID', regex: /\b(?:Policy|POL)[:\s#-]*[A-Z]{0,4}[-]?\d{6,12}\b/gi },
|
|
270
|
+
{ type: 'FINANCIAL', regex: /\b(?:Loss|Claim\s+Amount|Settlement|Reserve|Indemnity)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
271
|
+
{ type: 'ID', regex: /\b(?:Adjuster\s+(?:ID|No)|Claim\s+Rep(?:\.|resentative)?)[:\s#]*[A-Z0-9]{4,12}\b/gi },
|
|
272
|
+
{ type: 'ID', regex: /\bNAIC[:\s#]*\d{5}\b/gi },
|
|
273
|
+
{ type: 'FINANCIAL', regex: /\b(?:Deductible|Premium|Coverage\s+Amount)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
274
|
+
{ type: 'ID', regex: /\b[A-HJ-NPR-Z0-9]{17}\b/g },
|
|
275
|
+
{ type: 'ID', regex: /\b(?:Insured|Named\s+Insured)[:\s]+[A-Z][a-z]+(?:\s+[A-Z][a-z]+)+/g },
|
|
276
|
+
{ type: 'ID', regex: /\b(?:Agent|Producer)\s*(?:Code|No|ID)[:\s#]*[A-Z0-9]{4,12}\b/gi }
|
|
277
|
+
],
|
|
278
|
+
accounting: [
|
|
279
|
+
{ type: 'ID', regex: /\b(?:EIN|FEIN|Tax\s+ID)[:\s#]*\d{2}-\d{7}\b/gi },
|
|
280
|
+
{ type: 'FINANCIAL', regex: /\b(?:AGI|Adjusted\s+Gross\s+Income|Taxable\s+Income|Total\s+Income)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
281
|
+
{ type: 'FINANCIAL', regex: /\bBox\s+\d{1,2}[a-z]?[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
282
|
+
{ type: 'ID', regex: /\b(?:Form|Schedule)\s+(?:1040|1040-SR|W-2|W-4|1099-[A-Z]{1,4}|K-1|941|990|4562)\b/gi },
|
|
283
|
+
{ type: 'FINANCIAL', regex: /\b(?:Federal|State)\s+(?:Tax\s+)?(?:Refund|Amount\s+Owed|Balance\s+Due)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
284
|
+
{ type: 'ID', regex: /\b(?:State\s+Tax\s+ID|SUI|UI\s+Account\s+No)[:\s#]*[A-Z]{0,3}[-]?\d{4,15}\b/gi },
|
|
285
|
+
{ type: 'FINANCIAL', regex: /\b(?:Net\s+Pay|Gross\s+Pay|Taxable\s+Wages)[:\s]+(?:[$€£¥₪₽₹]|(?:USD|EUR|GBP|CHF|ILS|RUB)\s?)[\d,]+(?:[.]\d{2})?\b/gi },
|
|
286
|
+
{ type: 'ID', regex: /\b(?:CAF|Practitioner\s+(?:PIN|ID)|PTIN)[:\s#]*[A-Z]?\d{6,9}\b/gi }
|
|
287
|
+
],
|
|
288
|
+
pharma: [
|
|
289
|
+
{ type: 'ID', regex: /\b(?:Subject|Patient|Participant)\s+(?:ID|No)[:\s#]*[A-Z0-9]{3,12}\b/gi },
|
|
290
|
+
{ type: 'ID', regex: /\b(?:Protocol|Study)\s*(?:No|Number|ID)[:\s#]*[A-Z0-9][-A-Z0-9]{3,15}\b/gi },
|
|
291
|
+
{ type: 'ID', regex: /\b(?:IND|NDA|BLA|ANDA)\s*(?:No|Number|#)?[:\s]*\d{3,6}\b/gi },
|
|
292
|
+
{ type: 'ID', regex: /\bIRB[:\s#]*[A-Z0-9]{4,12}\b/gi },
|
|
293
|
+
{ type: 'ID', regex: /\b(?:Site\s+(?:No|ID)|Investigator\s+Site)[:\s#]*\d{3,6}\b/gi },
|
|
294
|
+
{ type: 'ID', regex: /\b(?:Batch|Lot|Serial)\s*(?:No[.]?|Number|#)?[:\s#]*[A-Z0-9][A-Z0-9\-]{3,14}\b/gi },
|
|
295
|
+
{ type: 'PHI', regex: /\b(?:Dose|Dosage)[:\s]+\d+(?:\.\d+)?\s*(?:mg|mcg|mL|IU|units?)\b/gi },
|
|
296
|
+
{ type: 'ID', regex: /\b(?:CRF|eCRF|Case\s+Report\s+Form)\s*(?:No|Page|ID)?[:\s#]*[A-Z0-9]{2,10}\b/gi }
|
|
297
|
+
]
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
let NAME_STOP_LIST = new Set([
|
|
301
|
+
'case no', 'account no', 'client no', 'ref no', 'matter no',
|
|
302
|
+
'affected user', 'incident date', 'incident type', 'incident report',
|
|
303
|
+
'review period', 'review date', 'salary band', 'salary range',
|
|
304
|
+
'personal email', 'work email', 'business email',
|
|
305
|
+
'lead source', 'account exec', 'account executive',
|
|
306
|
+
'next appt', 'next appointment', 'last appt', 'chief complaint',
|
|
307
|
+
'portfolio value', 'portfolio manager',
|
|
308
|
+
'tax id', 'account number', 'employee id', 'social security',
|
|
309
|
+
'date of', 'date of birth', 'place of birth',
|
|
310
|
+
'hiring manager', 'direct report', 'team lead', 'team leader',
|
|
311
|
+
'job title', 'job function', 'pay grade',
|
|
312
|
+
'review type', 'performance review', 'annual review',
|
|
313
|
+
'invoice number', 'purchase order', 'order number',
|
|
314
|
+
'oak street', 'main street', 'high street', 'park avenue',
|
|
315
|
+
'first street', 'second street', 'third street',
|
|
316
|
+
'north avenue', 'south avenue', 'east side', 'west side',
|
|
317
|
+
'soc team', 'hr team', 'it team', 'qa team', 'ux team',
|
|
318
|
+
'new york', 'los angeles', 'san francisco', 'las vegas',
|
|
319
|
+
'united states', 'united kingdom', 'north america', 'south america',
|
|
320
|
+
'senior analyst', 'senior engineer', 'senior manager', 'senior consultant',
|
|
321
|
+
'junior analyst', 'junior engineer', 'junior developer',
|
|
322
|
+
'vice president', 'chief executive', 'chief officer',
|
|
323
|
+
'account manager', 'project manager', 'product manager',
|
|
324
|
+
'security officer', 'compliance officer',
|
|
325
|
+
'general hospital', 'medical center', 'urgent care', 'primary care',
|
|
326
|
+
'read more', 'learn more', 'click here', 'sign up', 'log in',
|
|
327
|
+
'privacy policy', 'terms of', 'terms of service',
|
|
328
|
+
'true positive', 'false positive', 'open source',
|
|
329
|
+
'private identifiers', 'data privacy', 'system instruction', 'system instructions', 'critical', 'instruction',
|
|
330
|
+
'user data', 'protected user', 'privacy scrubber', 'end of',
|
|
331
|
+
'thank you', 'best regards', 'kind regards', 'warm regards', 'yours truly', 'sincerely yours', 'good morning', 'good afternoon', 'good evening', 'hello world', 'hello there',
|
|
332
|
+
'supreme court', 'high court', 'district court', 'civil court', 'non disclosure', 'data protection', 'intellectual property', 'trade secret', 'force majeure', 'habeas corpus', 'amicus curiae', 'board member', 'board meeting', 'general assembly',
|
|
333
|
+
'first name', 'last name', 'middle name', 'full name', 'email address', 'phone number', 'cell phone', 'home phone', 'zip code', 'postal code', 'page number', 'section one', 'table contents', 'table of', 'figure one',
|
|
334
|
+
'marketing department', 'sales department', 'engineering team', 'product team', 'customer support', 'human resources', 'public relations',
|
|
335
|
+
'artificial intelligence', 'machine learning', 'deep learning', 'large language', 'operating system', 'source code', 'user interface', 'web browser', 'pull request', 'merge request', 'commit message', 'code review', 'cloud computing', 'database schema',
|
|
336
|
+
'blood pressure', 'heart rate', 'chief physician', 'treating physician', 'health care', 'healthcare provider', 'medical record',
|
|
337
|
+
'grade a', 'grade b', 'grade c', 'grade d', 'grade f',
|
|
338
|
+
'version 1', 'version 2', 'version 3', 'version 4', 'version 5',
|
|
339
|
+
'step 1', 'step 2', 'step 3', 'step 4', 'step 5',
|
|
340
|
+
'page 1', 'page 2', 'page 3', 'page 4', 'page 5',
|
|
341
|
+
'cs101', 'course cs101',
|
|
342
|
+
// Expanded Stop List (Common nouns that look like names)
|
|
343
|
+
'january', 'february', 'march', 'april', 'may', 'june', 'july', 'august', 'september', 'october', 'november', 'december',
|
|
344
|
+
'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday',
|
|
345
|
+
'yesterday', 'tomorrow', 'today', 'last week', 'next month', 'early morning', 'late night',
|
|
346
|
+
'microsoft office', 'google workspace', 'slack channel', 'zoom meeting', 'teams call',
|
|
347
|
+
'amazon prime', 'netflix show', 'youtube video', 'twitter post', 'linkedin profile',
|
|
348
|
+
'central park', 'white house', 'grand canyon', 'mount everest', 'pacific ocean', 'atlantic ocean',
|
|
349
|
+
'silicon valley', 'wall street', 'fifth avenue', 'times square', 'golden gate', 'empire state',
|
|
350
|
+
'statue liberty', 'tower bridge', 'big ben', 'eiffel tower', 'great wall',
|
|
351
|
+
'london bridge', 'san jose', 'las vegas', 'mexico city', 'hong kong', 'san diego',
|
|
352
|
+
'software development', 'user experience', 'front end', 'back end', 'full stack',
|
|
353
|
+
'web application', 'mobile app', 'desktop app', 'cloud services', 'data analytics',
|
|
354
|
+
'cyber security', 'network security', 'information security', 'incident response',
|
|
355
|
+
'disaster recovery', 'business continuity', 'risk management', 'quality assurance',
|
|
356
|
+
'user acceptance', 'beta test', 'release candidate', 'version control', 'continuous integration',
|
|
357
|
+
'continuous deployment', 'agile scrum', 'kanban board', 'sprint planning', 'daily standup',
|
|
358
|
+
'product backlog', 'user story', 'acceptance criteria', 'technical debt', 'codebase',
|
|
359
|
+
'repository', 'branch name', 'tag version', 'hot fix', 'patch note',
|
|
360
|
+
'legal counsel', 'general counsel', 'human capital', 'talent acquisition',
|
|
361
|
+
'employee benefits', 'payroll services', 'stock options', 'health insurance',
|
|
362
|
+
'retirement plan', 'vacation time', 'sick leave', 'performance bonus',
|
|
363
|
+
'commission structure', 'sales target', 'market share', 'brand identity',
|
|
364
|
+
'marketing campaign', 'ad spend', 'click rate', 'conversion rate',
|
|
365
|
+
'customer lifetime', 'churn rate', 'user engagement', 'social media',
|
|
366
|
+
'content strategy', 'search engine', 'local search', 'organic traffic',
|
|
367
|
+
'paid search', 'email marketing', 'press release', 'case study',
|
|
368
|
+
'white paper', 'user guide', 'help center', 'frequently asked',
|
|
369
|
+
'support ticket', 'service level', 'response time', 'resolution time',
|
|
370
|
+
'customer satisfaction', 'net promoter', 'user feedback', 'product feature',
|
|
371
|
+
'roadmap item', 'beta program', 'early access', 'invite only',
|
|
372
|
+
'project status', 'meeting notes', 'call transcript', 'action items', 'follow up', 'best practices', 'success stories',
|
|
373
|
+
'class name', 'function name', 'variable name', 'database table', 'schema name', 'index name', 'query result', 'error message', 'warning message', 'log entry', 'debug log', 'stack trace',
|
|
374
|
+
'staff member', 'team member', 'board member', 'board meeting', 'committee member', 'executive board',
|
|
375
|
+
'email us', 'contact us', 'about us', 'sign in', 'sign out',
|
|
376
|
+
'lighting', 'keyboard', 'creating', 'building', 'training', 'planning', 'starting', 'painting', 'printing', 'returned', 'released', 'required', 'accepted', 'imported', 'services', 'products', 'accounts', 'settings', 'partners', 'keywords', 'keystone', 'keyspace', 'keynotes', 'keychain'
|
|
377
|
+
, 'quarterly results', 'strategic planning', 'market research', 'customer base', 'privacy settings', 'account settings', 'security settings', 'download now', 'free trial', 'limited time', 'copyright protected', 'all rights', 'rights reserved', 'credit score', 'monthly rent', 'lease application', 'property address', 'reference number', 'additional identifier', 'lease agreement', 'hiring review', 'candidate name', 'privacy policy', 'terms of service', 'machine learning', 'artificial intelligence', 'generative ai', 'silicon valley', 'google cloud', 'amazon web', 'data science', 'operating system', 'software engineer', 'product manager', 'project manager', 'data analyst', 'gross margin', 'revenue growth', 'source code', 'version control', 'large language model']);
|
|
378
|
+
|
|
379
|
+
let JARGON_WORDS = new Set(['step', 'page', 'grade', 'version', 'course', 'class', 'follow', 'chapter', 'lesson', 'unit', 'marketing', 'manager', 'specialist', 'science', 'administration', 'university', 'skills', 'leadership', 'communication', 'working', 'proficiency', 'decision', 'driven', 'experience', 'summary', 'bachelor', 'ads', 'solutions', 'positioning', 'acquisition', 'strategy', 'research', 'database', 'forecast', 'interest', 'prepared', 'merchant', 'document', 'feedback', 'template', 'campaign', 'partners', 'settings', 'keystone', 'llm', 'gpt', 'chatgpt', 'openai', 'anthropic', 'claude', 'gemini', 'api', 'json', 'xml', 'html', 'css', 'javascript', 'python', 'policy', 'terms', 'conditions', 'release', 'sprint', 'deployment', 'cluster', 'instance', 'package', 'module', 'revenue', 'margin', 'gross', 'quarter', 'system', 'code', 'data', 'cloud', 'server', 'database', 'artificial', 'intelligence', 'learning', 'generative']);
|
|
380
|
+
|
|
381
|
+
let NOT_NAME_WORDS = new Set([
|
|
382
|
+
// Grammatical & Sentence Starters
|
|
383
|
+
'the', 'a', 'an', 'this', 'that', 'these', 'those', 'my', 'your', 'his', 'her', 'their', 'our', 'its', 'it', 'he', 'she', 'they', 'we', 'i', 'you', 'who', 'whom', 'which', 'what', 'whose', 'why', 'how', 'when', 'where', 'with', 'for', 'from', 'by', 'to', 'at', 'in', 'on', 'of', 'about', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below', 'and', 'but', 'or', 'so', 'yet',
|
|
384
|
+
// Greetings & Salutations
|
|
385
|
+
'hello', 'hi', 'hey', 'dear', 'greetings',
|
|
386
|
+
// Document & Resume Structure
|
|
387
|
+
'summary', 'experience', 'education', 'skills', 'languages', 'project', 'history', 'background', 'objective', 'profile', 'awards', 'honors', 'certifications', 'publications', 'interests', 'references',
|
|
388
|
+
// Business & Job Roles
|
|
389
|
+
'manager', 'director', 'specialist', 'analyst', 'engineer', 'developer', 'consultant', 'officer', 'representative', 'agent', 'lead', 'leader', 'president', 'coordinator', 'admin', 'administrator', 'executive', 'founder', 'partner', 'intern', 'trainee', 'advisor', 'head', 'vp', 'chief',
|
|
390
|
+
// Departments & Fields
|
|
391
|
+
'marketing', 'sales', 'engineering', 'finance', 'accounting', 'legal', 'operations', 'support', 'recruiting', 'talent', 'acquisition', 'compliance', 'security', 'technical', 'development', 'product', 'design', 'creative', 'strategy', 'planning', 'analytics', 'science', 'business', 'administration',
|
|
392
|
+
// Tools & Tech Concepts
|
|
393
|
+
'google', 'ads', 'analytics', 'meta', 'hubspot', 'crm', 'salesforce', 'wordpress', 'mailchimp', 'adobe', 'figma', 'canva', 'slack', 'zoom', 'teams', 'microsoft', 'office', 'excel', 'word', 'powerpoint', 'notion', 'jira', 'confluence', 'github', 'gitlab', 'aws', 'azure', 'cloud', 'database', 'sql', 'python', 'java', 'javascript', 'html', 'css', 'react', 'node', 'api', 'saas', 'b2b', 'b2c', 'url', 'domain', 'website', 'app', 'application', 'software', 'email', 'phone', 'contact', 'address',
|
|
394
|
+
|
|
395
|
+
// General Academic & Professional vocabulary
|
|
396
|
+
'bachelor', 'master', 'doctor', 'associate', 'degree', 'university', 'college', 'school', 'institute', 'academy', 'graduated', 'major', 'minor', 'gpa', 'cum', 'laude', 'honors', 'deans', 'list', 'scholarship',
|
|
397
|
+
// Quality & Adjectives
|
|
398
|
+
'results', 'driven', 'oriented', 'expert', 'professional', 'proven', 'track', 'record', 'creative', 'excellent', 'communication', 'verbal', 'written', 'native', 'fluent', 'bilingual', 'working', 'proficiency', 'strategic', 'interpersonal', 'teamwork', 'organizational', 'detail', 'analytical',
|
|
399
|
+
// Common Resume / Business Phrases
|
|
400
|
+
'results-driven', 'data-driven', 'customer-centric', 'detail-oriented', 'cross-functional', 'self-motivated', 'time-management', 'problem-solving', 'fast-paced', 'year-over-year',
|
|
401
|
+
// Legal & Trust terms
|
|
402
|
+
'trust', 'trustee', 'co-trustee', 'settlor', 'grantor', 'beneficiary', 'agreement', 'will', 'estate', 'witness', 'declaration', 'signatory', 'testator', 'notary', 'commission', 'county', 'state', 'court', 'article', 'section', 'paragraph', 'schedule', 'exhibit', 'amendment', 'addendum', 'power', 'attorney', 'guardian', 'executor', 'administrator', 'survivor', 'predecessor', 'successor', 'whereof', 'hereby', 'thereby', 'herein', 'therein', 'witnesseth', 'whereas', 'therefore', 'now', 'dated', 'effective',
|
|
403
|
+
// US States
|
|
404
|
+
'california', 'texas', 'florida', 'york', 'illinois', 'pennsylvania', 'ohio', 'georgia', 'michigan', 'carolina', 'virginia', 'washington', 'arizona', 'massachusetts', 'tennessee', 'indiana', 'maryland', 'missouri', 'wisconsin', 'colorado', 'minnesota', 'alabama', 'louisiana', 'kentucky', 'oregon', 'oklahoma', 'connecticut', 'utah', 'iowa', 'nevada', 'arkansas', 'mississippi', 'kansas', 'new mexico', 'nebraska', 'idaho', 'hawaii', 'maine', 'new hampshire', 'rhode island', 'montana', 'delaware', 'south dakota', 'north dakota', 'alaska', 'vermont', 'wyoming'
|
|
405
|
+
, 'f.3d', 'f.supp', 'u.s.c.', 'v.', 'plaintiff', 'defendant', 'v', 'u.s.', 'court', 'app.', 'reporter', 'cir.']);
|
|
406
|
+
|
|
407
|
+
const PROFILE_JARGON = {
|
|
408
|
+
medical: ['sleep', 'apnea', 'symptom', 'symptoms', 'trauma', 'hypertension', 'health', 'disease', 'condition', 'diagnosis', 'treatment', 'medication', 'dose', 'patient', 'clinic', 'surgery', 'therapy', 'alcohol', 'cannabis', 'blood', 'pressure', 'heart', 'rate', 'emergency', 'contact', 'relationship'],
|
|
409
|
+
realestate: ['escrow', 'tenant', 'landlord', 'lease', 'mortgage', 'appraisal', 'broker', 'property', 'zoning', 'parcel', 'rent', 'buyer', 'seller', 'agent', 'listing'],
|
|
410
|
+
legal: ['testator', 'notary', 'commission', 'county', 'court', 'affidavit', 'plaintiff', 'defendant', 'litigation', 'jurisdiction', 'agreement', 'contract', 'settlement', 'clause', 'article', 'section'],
|
|
411
|
+
hr: ['candidate', 'employee', 'payroll', 'benefits', 'salary', 'vacation', 'supervisor', 'subordinate', 'performance', 'appraisal', 'interview', 'resume', 'applicant'],
|
|
412
|
+
sales: ['prospect', 'opportunity', 'quota', 'pipeline', 'deal', 'revenue', 'forecast', 'lead', 'churn', 'client', 'customer']
|
|
413
|
+
};
|
|
414
|
+
// --- END DEFAULT RULES ---
|
|
415
|
+
|
|
416
|
+
function hydrateRegex(r) {
|
|
417
|
+
if (r && typeof r.regex === 'string') {
|
|
418
|
+
if (r.regex.startsWith('/')) {
|
|
419
|
+
try {
|
|
420
|
+
const lastSlash = r.regex.lastIndexOf('/');
|
|
421
|
+
const pattern = r.regex.substring(1, lastSlash);
|
|
422
|
+
const flags = r.regex.substring(lastSlash + 1);
|
|
423
|
+
return { ...r, regex: new RegExp(pattern, flags) };
|
|
424
|
+
} catch (e) {
|
|
425
|
+
console.error('Failed to hydrate regex:', r.regex, e);
|
|
426
|
+
return r;
|
|
427
|
+
}
|
|
428
|
+
} else {
|
|
429
|
+
return { ...r, regex: new RegExp(r.regex, 'gi') };
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return r;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function init(config) {
|
|
436
|
+
if (!config) return;
|
|
437
|
+
if (config.regexes) {
|
|
438
|
+
REGEX_RULES = config.regexes.map(hydrateRegex);
|
|
439
|
+
}
|
|
440
|
+
if (config.profiles) {
|
|
441
|
+
PROFILE_RULES = {};
|
|
442
|
+
for (const [profile, rules] of Object.entries(config.profiles)) {
|
|
443
|
+
PROFILE_RULES[profile] = rules.map(hydrateRegex);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
if (config.names) NAME_STOP_LIST = new Set(config.names);
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
const PROFILE_ALIAS_MAP = {
|
|
450
|
+
'medical': 'medical', 'healthcare': 'medical', 'health': 'medical', 'pharma': 'pharma',
|
|
451
|
+
'engineering': 'engineering', 'dev': 'engineering', 'tech': 'tech',
|
|
452
|
+
'finance': 'finance', 'bizops': 'bizops', 'sales': 'sales', 'wealthmgmt': 'wealthmgmt', 'insurance': 'insurance', 'accounting': 'accounting',
|
|
453
|
+
'legal': 'legal', 'compliance': 'compliance', 'ccpa': 'ccpa',
|
|
454
|
+
'hr': 'hr', 'security': 'security', 'marketing': 'marketing', 'support': 'support',
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
function getActiveRules(activeProfile) {
|
|
458
|
+
let activeRules = [...REGEX_RULES];
|
|
459
|
+
if (activeProfile && activeProfile.toLowerCase() !== 'general') {
|
|
460
|
+
const canonicalProfile = PROFILE_ALIAS_MAP[activeProfile.toLowerCase()] || 'general';
|
|
461
|
+
if (canonicalProfile !== 'general' && PROFILE_RULES[canonicalProfile]) {
|
|
462
|
+
activeRules = activeRules.concat(PROFILE_RULES[canonicalProfile]);
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
return activeRules;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function stitchOrphanedNameLines(text, profile) {
|
|
469
|
+
if (profile === 'medical') {
|
|
470
|
+
text = text.replace(/Patient Name:\s*\n+([A-Z][a-zA-Z]+\s[A-Z][a-zA-Z]+)/g, 'Patient Name: $1');
|
|
471
|
+
} else if (profile === 'legal') {
|
|
472
|
+
text = text.replace(/Defendant:\s*\n+([A-Z][a-zA-Z]+\s[A-Z][a-zA-Z]+)/g, 'Defendant: $1');
|
|
473
|
+
}
|
|
474
|
+
return text;
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
function detectMatches(text, activeProfile = 'general', customRules = [], nlpNames = [], enabledEntities = null, mlEntities = []) {
|
|
478
|
+
let textToProcess = text.replace(/[\u200b\u200c\u200d\ufeff]/g, '');
|
|
479
|
+
textToProcess = textToProcess.replace(/([a-zA-Z])(Email|Phone|Mobile|Tel|Address|IP|ID|URL|SSN|Date):/g, '$1 $2:');
|
|
480
|
+
|
|
481
|
+
let matches = [];
|
|
482
|
+
|
|
483
|
+
if (customRules && customRules.length > 0) {
|
|
484
|
+
const sorted = [...customRules].sort((a, b) => {
|
|
485
|
+
const patternA = typeof a === 'string' ? a : a.pattern;
|
|
486
|
+
const patternB = typeof b === 'string' ? b : b.pattern;
|
|
487
|
+
return patternB.length - patternA.length;
|
|
488
|
+
});
|
|
489
|
+
|
|
490
|
+
sorted.forEach(cr => {
|
|
491
|
+
const pattern = typeof cr === 'string' ? cr : cr.pattern;
|
|
492
|
+
const label = typeof cr === 'string' ? 'CUSTOM' : (cr.label || 'CUSTOM');
|
|
493
|
+
|
|
494
|
+
let rx;
|
|
495
|
+
try {
|
|
496
|
+
const isExact = !/(\^|\$|\\[bBdDwWsS]|\[|\(|\{|\*|\+|\|)/.test(pattern);
|
|
497
|
+
if (isExact) {
|
|
498
|
+
const safe = pattern.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
499
|
+
rx = new RegExp(`(?:^|\\b)${safe}(?:\\b|$)`, 'gi');
|
|
500
|
+
} else {
|
|
501
|
+
rx = new RegExp(pattern, 'gi');
|
|
502
|
+
}
|
|
503
|
+
} catch (e) {
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
if (rx.test('')) return;
|
|
507
|
+
rx.lastIndex = 0;
|
|
508
|
+
let m;
|
|
509
|
+
while ((m = rx.exec(textToProcess)) !== null) {
|
|
510
|
+
matches.push({ start: m.index, end: m.index + m[0].length, value: m[0], type: 'CUSTOM', customLabel: label });
|
|
511
|
+
}
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const activeRules = getActiveRules(activeProfile);
|
|
516
|
+
const currentJargon = PROFILE_JARGON[PROFILE_ALIAS_MAP[activeProfile?.toLowerCase()]] ? new Set([...JARGON_WORDS, ...PROFILE_JARGON[PROFILE_ALIAS_MAP[activeProfile?.toLowerCase()]]]) : JARGON_WORDS;
|
|
517
|
+
|
|
518
|
+
activeRules.forEach(rule => {
|
|
519
|
+
if (enabledEntities && enabledEntities.indexOf(rule.type) === -1) return;
|
|
520
|
+
if (!rule.regex) return;
|
|
521
|
+
|
|
522
|
+
rule.regex.lastIndex = 0;
|
|
523
|
+
let m;
|
|
524
|
+
while ((m = rule.regex.exec(textToProcess)) !== null) {
|
|
525
|
+
let matchedText = m[0];
|
|
526
|
+
let start = m.index;
|
|
527
|
+
|
|
528
|
+
if (rule.isContextName && m[1]) {
|
|
529
|
+
matchedText = m[1];
|
|
530
|
+
start = m.index + m[0].lastIndexOf(m[1]);
|
|
531
|
+
}
|
|
532
|
+
const end = start + matchedText.length;
|
|
533
|
+
|
|
534
|
+
if (rule.type !== 'NAME' && rule.type !== 'ADDRESS') {
|
|
535
|
+
matches.push({ start, end, value: matchedText, type: rule.type });
|
|
536
|
+
} else {
|
|
537
|
+
let val = matchedText.toLowerCase().trim();
|
|
538
|
+
if (NAME_STOP_LIST.has(val)) continue;
|
|
539
|
+
|
|
540
|
+
if (!rule.isContextName && rule.type === 'NAME') {
|
|
541
|
+
let words = val.split(/[ \t\xA0]+/);
|
|
542
|
+
let origWords = matchedText.split(/[ \t\xA0]+/);
|
|
543
|
+
while (words.length > 2 && (currentJargon.has(words[0]) || NOT_NAME_WORDS.has(words[0]))) {
|
|
544
|
+
origWords.shift();
|
|
545
|
+
words.shift();
|
|
546
|
+
const nextStart = matchedText.indexOf(origWords[0]);
|
|
547
|
+
start += nextStart;
|
|
548
|
+
matchedText = matchedText.substring(nextStart);
|
|
549
|
+
val = matchedText.toLowerCase().trim();
|
|
550
|
+
}
|
|
551
|
+
if (words.some(w => currentJargon.has(w) || NOT_NAME_WORDS.has(w))) continue;
|
|
552
|
+
} else if (rule.type === 'ADDRESS') {
|
|
553
|
+
const words = val.split(/\s+/);
|
|
554
|
+
if (words.some(w => currentJargon.has(w))) continue;
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
matches.push({ start, end: start + matchedText.length, value: matchedText, type: rule.type });
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
if (nlpNames && nlpNames.length > 0) {
|
|
563
|
+
nlpNames.forEach(name => {
|
|
564
|
+
const safeName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
565
|
+
const rx = new RegExp(`(?<=^|[^\\p{L}\\p{N}_])${safeName}(?=[^\\p{L}\\p{N}_]|$)`, 'gu');
|
|
566
|
+
let m;
|
|
567
|
+
while ((m = rx.exec(textToProcess)) !== null) {
|
|
568
|
+
matches.push({ start: m.index, end: m.index + m[0].length, value: m[0], type: 'NAME' });
|
|
569
|
+
}
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
if (mlEntities && mlEntities.length > 0) {
|
|
574
|
+
mlEntities.forEach(item => {
|
|
575
|
+
const safeName = item.word.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
576
|
+
const rx = new RegExp(`(?<=^|[^\\p{L}\\p{N}_])${safeName}(?=[^\\p{L}\\p{N}_]|$)`, 'giu');
|
|
577
|
+
let m;
|
|
578
|
+
while ((m = rx.exec(textToProcess)) !== null) {
|
|
579
|
+
matches.push({ start: m.index, end: m.index + m[0].length, value: m[0], type: item.type });
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const learnedNames = new Set();
|
|
585
|
+
matches.forEach(m => {
|
|
586
|
+
if (m.type === 'NAME') {
|
|
587
|
+
const nameWords = m.value.split(/[^\p{L}'-]+/u);
|
|
588
|
+
nameWords.forEach(w => {
|
|
589
|
+
if (w && w.length >= 2 && /^\p{Lu}/u.test(w)) {
|
|
590
|
+
const wl = w.toLowerCase();
|
|
591
|
+
if (!NOT_NAME_WORDS.has(wl) && !JARGON_WORDS.has(wl) && !NAME_STOP_LIST.has(wl)) {
|
|
592
|
+
learnedNames.add(w);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
});
|
|
598
|
+
|
|
599
|
+
if (learnedNames.size > 0) {
|
|
600
|
+
learnedNames.forEach(name => {
|
|
601
|
+
const safeName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
602
|
+
const rx = new RegExp(`(?<=^|[^\\p{L}\\p{N}_])${safeName}(?=[^\\p{L}\\p{N}_]|$)`, 'gu');
|
|
603
|
+
let m;
|
|
604
|
+
while ((m = rx.exec(textToProcess)) !== null) {
|
|
605
|
+
matches.push({ start: m.index, end: m.index + m[0].length, value: m[0], type: 'NAME' });
|
|
606
|
+
}
|
|
607
|
+
});
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
matches.sort((a, b) => a.start - b.start || b.end - a.end);
|
|
611
|
+
let filtered = [];
|
|
612
|
+
let lastEnd = 0;
|
|
613
|
+
matches.forEach(m => {
|
|
614
|
+
if (m.start >= lastEnd) { filtered.push(m); lastEnd = m.end; }
|
|
615
|
+
});
|
|
616
|
+
|
|
617
|
+
return { filteredMatches: filtered, processedText: textToProcess };
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
const LABEL_ALIASES = {
|
|
621
|
+
NAME: ['NAME', 'NAMES', 'USERNAME', 'USER_NAME', 'CLIENTNAME', 'CLIENT_NAME', 'CANDIDATE_NAME', 'FULL_NAME', 'FIRSTNAME', 'FIRST_NAME', 'LASTNAME', 'LAST_NAME', 'SURNAME', 'ИМЯ', 'ИМЕНА', 'ПОЛЬЗОВАТЕЛЬ', 'ФИО', 'КЛИЕНТ', 'NOMBRE', 'NOMBRES', 'USUARIO', 'CLIENTE', 'NOM', 'NOMS', 'UTILISATEUR', 'NAME', 'NAMEN', 'BENUTZER', 'KUNDE', 'NOME', 'COGNOME', 'UTENTE', 'NAAM', 'GEBRUIKER', 'KLANT'],
|
|
622
|
+
EMAIL: ['EMAIL', 'EMAILS', 'EMAILADDR', 'EMAIL_ADDR', 'EMAILADDRESS', 'EMAIL_ADDRESS', 'EMAIL_ADR', 'MAIL', 'MAILS', 'ПОЧТА', 'ЭЛ_ПОЧТА', 'АДРЕС_ПОЧТЫ', 'МЕЙЛ', 'МАЙЛ', 'CORREO', 'COURRIEL', 'CORREO_ELECTRONICO', 'MEL'],
|
|
623
|
+
PHONE: ['PHONE', 'PHONES', 'PHONENUM', 'PHONE_NUM', 'PHONENUMBER', 'PHONE_NUMBER', 'TEL', 'TELS', 'TELEPHONE', 'TELEPHONES', 'MOBILE', 'CELL', 'ТЕЛЕФОН', 'ТЕЛЕФОНЫ', 'НОМЕР_ТЕЛЕФОНА', 'НОМЕР', 'MOVIL', 'PORTABLE', 'HANDY', 'TELEFONI', 'CELLULARE'],
|
|
624
|
+
ID: ['ID', 'IDS', 'IDNUM', 'ID_NUM', 'IDNUMBER', 'ID_NUMBER', 'IDENTIFIER', 'IDENTIFIERS', 'PASSPORT', 'SSN', 'EIN', 'TAXID', 'TAX_ID', 'LICENSE', 'LICENSE_PLATE', 'ИД', 'ИДЕНТИФИКАТОР', 'ПАСПОРТ', 'СНИЛС', 'ИНН', 'IDENTIFICADOR', 'PASAPORTE', 'IDENTIFIANT', 'PASSEPORT', 'IDENTIFIKATOR', 'PASS', 'IDENTIFICATORE', 'PASSAPORTO'],
|
|
625
|
+
FINANCIAL: ['FINANCIAL', 'FINANCIALS', 'MONEY', 'AMOUNT', 'PRICE', 'COST', 'CARD', 'CREDITCARD', 'DEBITCARD', 'ACCOUNT', 'IBAN', 'BIC', 'ДЕНЬГИ', 'СУММА', 'КАРТА', 'СЧЕТ', 'БАНК', 'DINERO', 'CANTIDAD', 'TARJETA', 'CUENTA', 'ARGENT', 'MONTANT', 'COMPTE', 'GELD', 'BETRAG', 'KONTO'],
|
|
626
|
+
ADDRESS: ['ADDRESS', 'ADDRESSES', 'STREET', 'STREET_ADDRESS', 'CITY', 'STATE', 'ZIP', 'ZIPCODE', 'ZIP_CODE', 'COUNTRY', 'LOCATION', 'АДРЕС', 'АДРЕСА', 'УЛИЦА', 'ГОРОД', 'СТРАНА', 'DIRECCION', 'DIRECCIONES', 'CALLE', 'CIUDAD', 'PAIS', 'ADRESSE', 'ADRESSES', 'RUE', 'VILLE', 'STRASSE', 'STADT', 'LAND'],
|
|
627
|
+
DATE: ['DATE', 'DATES', 'BIRTHDAY', 'DOB', 'ДАТА', 'ДАТЫ', 'ДЕНЬ_РОЖДЕНИЯ', 'FECHA', 'FECHAS', 'CUMPLEANOS', 'ANNIVERSAIRE', 'DATUM', 'DATEN', 'GEBURTSTAG'],
|
|
628
|
+
PHI: ['PHI', 'MRN', 'NHS', 'HEALTH', 'MEDICAL', 'PATIENT', 'МЕД', 'ПАЦИЕНТ', 'PACIENTE'],
|
|
629
|
+
SECRET: ['SECRET', 'SECRETS', 'KEY', 'KEYS', 'TOKEN', 'TOKENS', 'PASSWORD', 'PASSWORDS', 'AUTH', 'APIKEY', 'API_KEY', 'КЛЮЧ', 'КЛЮЧИ', 'ПАРОЛЬ', 'ПАРОЛИ', 'ТОКЕН', 'CLAVE', 'CONTRASENA', 'CLE', 'MOT_DE_PASSE', 'SCHLUESSEL', 'PASSWORT'],
|
|
630
|
+
CUSTOM: ['CUSTOM', 'CUSTOMS', 'RULE', 'RULES', 'КАСТОМ', 'ПРАВИЛО']
|
|
631
|
+
};
|
|
632
|
+
|
|
633
|
+
function getLabelAliases(label) {
|
|
634
|
+
const upper = label.toUpperCase();
|
|
635
|
+
if (LABEL_ALIASES[upper]) return LABEL_ALIASES[upper];
|
|
636
|
+
const aliases = new Set([label, upper, label.toLowerCase()]);
|
|
637
|
+
aliases.add(label.replace(/_/g, ' '));
|
|
638
|
+
aliases.add(label.replace(/_/g, '-'));
|
|
639
|
+
aliases.add(label.replace(/ /g, '_'));
|
|
640
|
+
aliases.add(label.replace(/-/g, '_'));
|
|
641
|
+
return Array.from(aliases);
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
function buildRestorationRegexAndRules(tokenMap) {
|
|
645
|
+
const ObjectKeys = Object.keys(tokenMap);
|
|
646
|
+
if (ObjectKeys.length === 0) return { compositeRegex: null, looseRules: [] };
|
|
647
|
+
|
|
648
|
+
const sortedKeys = [...ObjectKeys].sort((a, b) => {
|
|
649
|
+
const innerA = a.replace(/^\[|\]$/g, '');
|
|
650
|
+
const innerB = b.replace(/^\[|\]$/g, '');
|
|
651
|
+
const matchA = innerA.match(/^([A-Za-z_0-9]+?)[-_]?(\d+)$/);
|
|
652
|
+
const matchB = innerB.match(/^([A-Za-z_0-9]+?)[-_]?(\d+)$/);
|
|
653
|
+
if (matchA && matchB) {
|
|
654
|
+
const idxA = parseInt(matchA[2], 10);
|
|
655
|
+
const idxB = parseInt(matchB[2], 10);
|
|
656
|
+
const labelA = matchA[1];
|
|
657
|
+
const labelB = matchB[1];
|
|
658
|
+
if (idxA !== idxB) return idxB - idxA;
|
|
659
|
+
if (labelA.length !== labelB.length) return labelB.length - labelA.length;
|
|
660
|
+
}
|
|
661
|
+
return b.length - a.length;
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
const looseRules = [];
|
|
665
|
+
const regexParts = [];
|
|
666
|
+
|
|
667
|
+
sortedKeys.forEach(k => {
|
|
668
|
+
const inner = k.replace(/^\[|\]$/g, '');
|
|
669
|
+
const match = inner.match(/^([A-Za-z_0-9]+?)[-_]?(\d+)$/);
|
|
670
|
+
if (match) {
|
|
671
|
+
const label = match[1];
|
|
672
|
+
const baseIndex = parseInt(match[2], 10);
|
|
673
|
+
const aliases = getLabelAliases(label);
|
|
674
|
+
|
|
675
|
+
const escapedAliases = aliases.map(a => a.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
676
|
+
const loosePattern = `\\[\\s*(?:${escapedAliases.join('|')})[-_\\s]*0*${baseIndex}\\s*\\]|(?<![A-Za-z0-9\\u0400-\\u04FF_])(?:${escapedAliases.join('|')})[-_\\s]*0*${baseIndex}(?![A-Za-z0-9\\u0400-\\u04FF_])`;
|
|
677
|
+
looseRules.push({ token: k, pattern: loosePattern });
|
|
678
|
+
}
|
|
679
|
+
regexParts.push(k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'));
|
|
680
|
+
});
|
|
681
|
+
|
|
682
|
+
let compositeRegex = null;
|
|
683
|
+
if (regexParts.length > 0) {
|
|
684
|
+
compositeRegex = new RegExp(`(?:\\b|\\[)?(?:(?:(?<!\\w)|(?<=\\s))(?:${regexParts.join('|')})(?:(?!\\w)|(?=\\s)))(?:\\b|\\])?`, 'g');
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
return { compositeRegex, looseRules };
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
function cleanAIPromptPrefix(text) {
|
|
691
|
+
if (!text) return "";
|
|
692
|
+
let cleaned = text.replace(/^\s*(?:Claude responded|Claude|ChatGPT|Gemini|Grok|DeepSeek|Kimi|Copilot|Assistant|User)\s*(?::|\bsaid\b|\bresponded\b|(?=\s))\s*/i, "");
|
|
693
|
+
cleaned = cleaned.replace(/^(?:Here (?:is|are) (?:the )?(?:redacted|scrubbed|sanitized|processed|clean|updated|modified) (?:text|output|version|data).*?[:\n]+|\*\*Scrubbed Text\*\*[:\n]+|### Scrubbed Text[:\n]+)/i, '');
|
|
694
|
+
cleaned = cleaned.replace(/^\s*Edit\s*\n+/i, "");
|
|
695
|
+
cleaned = cleaned.replace(/\s*\bEdit\s+in\s+a\s+page\b\s*$/i, "");
|
|
696
|
+
return cleaned.trim();
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
function unscrubText(text, sessionMap) {
|
|
700
|
+
let restoredCount = 0;
|
|
701
|
+
let result = text;
|
|
702
|
+
const tokens = Object.keys(sessionMap);
|
|
703
|
+
if (tokens.length === 0) return { text: result, count: 0 };
|
|
704
|
+
|
|
705
|
+
result = cleanAIPromptPrefix(result);
|
|
706
|
+
|
|
707
|
+
const { compositeRegex, looseRules } = buildRestorationRegexAndRules(sessionMap);
|
|
708
|
+
|
|
709
|
+
if (compositeRegex) {
|
|
710
|
+
result = result.replace(compositeRegex, (match) => {
|
|
711
|
+
const cleanMatch = match.replace(/^\[|\]$/g, '').trim();
|
|
712
|
+
const token = `[${cleanMatch.replace(/^\[|\]$/g, '')}]`;
|
|
713
|
+
if (sessionMap[token]) {
|
|
714
|
+
restoredCount++;
|
|
715
|
+
return sessionMap[token];
|
|
716
|
+
}
|
|
717
|
+
if (sessionMap[cleanMatch]) {
|
|
718
|
+
restoredCount++;
|
|
719
|
+
return sessionMap[cleanMatch];
|
|
720
|
+
}
|
|
721
|
+
return match;
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
looseRules.forEach(rule => {
|
|
726
|
+
const rx = new RegExp(rule.pattern, 'gi');
|
|
727
|
+
result = result.replace(rx, (match) => {
|
|
728
|
+
restoredCount++;
|
|
729
|
+
return sessionMap[rule.token];
|
|
730
|
+
});
|
|
731
|
+
});
|
|
732
|
+
|
|
733
|
+
return { text: result, count: restoredCount };
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
function escapeHTML(str) {
|
|
737
|
+
if (!str) return '';
|
|
738
|
+
return String(str)
|
|
739
|
+
.replace(/&/g, '&')
|
|
740
|
+
.replace(/</g, '<')
|
|
741
|
+
.replace(/>/g, '>')
|
|
742
|
+
.replace(/"/g, '"')
|
|
743
|
+
.replace(/'/g, ''');
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
function unscrubTextAsHTML(text, sessionMap) {
|
|
747
|
+
let restoredCount = 0;
|
|
748
|
+
let cleaned = cleanAIPromptPrefix(text || '');
|
|
749
|
+
let result = escapeHTML(cleaned);
|
|
750
|
+
const tokens = Object.keys(sessionMap || {});
|
|
751
|
+
if (tokens.length === 0) return { text: result, count: 0 };
|
|
752
|
+
|
|
753
|
+
const { compositeRegex, looseRules } = buildRestorationRegexAndRules(sessionMap);
|
|
754
|
+
|
|
755
|
+
if (compositeRegex) {
|
|
756
|
+
result = result.replace(compositeRegex, (match) => {
|
|
757
|
+
const cleanMatch = match.replace(/^\[|\]$/g, '').trim();
|
|
758
|
+
const token = `[${cleanMatch.replace(/^\[|\]$/g, '')}]`;
|
|
759
|
+
if (sessionMap[token]) {
|
|
760
|
+
restoredCount++;
|
|
761
|
+
const escapedVal = escapeHTML(sessionMap[token]);
|
|
762
|
+
return `<span class="ps-restored-data entity-tag entity-revealed" title="Original Token: ${escapeHTML(token)}">${escapedVal}</span>`;
|
|
763
|
+
}
|
|
764
|
+
if (sessionMap[cleanMatch]) {
|
|
765
|
+
restoredCount++;
|
|
766
|
+
const escapedVal = escapeHTML(sessionMap[cleanMatch]);
|
|
767
|
+
return `<span class="ps-restored-data entity-tag entity-revealed" title="Original Token: ${escapeHTML(cleanMatch)}">${escapedVal}</span>`;
|
|
768
|
+
}
|
|
769
|
+
return match;
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
looseRules.forEach(rule => {
|
|
774
|
+
const rx = new RegExp(rule.pattern, 'gi');
|
|
775
|
+
result = result.replace(rx, (match, offset, fullStr) => {
|
|
776
|
+
const before = fullStr.substring(0, offset);
|
|
777
|
+
const lastOpen = before.lastIndexOf('<');
|
|
778
|
+
const lastClose = before.lastIndexOf('>');
|
|
779
|
+
if (lastOpen > lastClose) return match;
|
|
780
|
+
|
|
781
|
+
restoredCount++;
|
|
782
|
+
const escapedVal = escapeHTML(sessionMap[rule.token]);
|
|
783
|
+
return `<span class="ps-restored-data entity-tag entity-revealed" title="Original Token: ${escapeHTML(rule.token)} (Fuzzy Match)">${escapedVal}</span>`;
|
|
784
|
+
});
|
|
785
|
+
});
|
|
786
|
+
|
|
787
|
+
return { text: result, count: restoredCount };
|
|
788
|
+
}
|
|
789
|
+
|
|
790
|
+
const PrivacyScrubberEngine = {
|
|
791
|
+
init,
|
|
792
|
+
hydrateRegex,
|
|
793
|
+
getActiveRules,
|
|
794
|
+
stitchOrphanedNameLines,
|
|
795
|
+
detectMatches,
|
|
796
|
+
getRulesMap: () => ({ REGEX_RULES, PROFILE_RULES }),
|
|
797
|
+
getDevopsRules: () => DEVOPS_SECRETS,
|
|
798
|
+
LABEL_ALIASES,
|
|
799
|
+
getLabelAliases,
|
|
800
|
+
PROFILE_ALIAS_MAP,
|
|
801
|
+
buildRestorationRegexAndRules,
|
|
802
|
+
unscrubText,
|
|
803
|
+
unscrubTextAsHTML,
|
|
804
|
+
cleanAIPromptPrefix,
|
|
805
|
+
// Expose underlying constants for Node scripts & build pipeline proxy
|
|
806
|
+
REGEX_RULES,
|
|
807
|
+
PROFILE_RULES,
|
|
808
|
+
NAME_STOP_LIST,
|
|
809
|
+
JARGON_WORDS,
|
|
810
|
+
NOT_NAME_WORDS,
|
|
811
|
+
PROFILE_JARGON,
|
|
812
|
+
};
|
|
813
|
+
|
|
814
|
+
if (typeof exports !== 'undefined') {
|
|
815
|
+
module.exports = PrivacyScrubberEngine;
|
|
816
|
+
}
|
|
817
|
+
if (typeof self !== 'undefined') {
|
|
818
|
+
self.PrivacyScrubberEngine = PrivacyScrubberEngine;
|
|
819
|
+
}
|
|
820
|
+
if (typeof window !== 'undefined') {
|
|
821
|
+
window.PrivacyScrubberEngine = PrivacyScrubberEngine;
|
|
822
|
+
}
|
|
823
|
+
})();
|