@link-foundation/sensitive-data-sanitizer 0.0.0-stage → 0.11.31

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +530 -0
  2. package/LICENSE +24 -0
  3. package/README.md +184 -2
  4. package/bin/legacy-example-package-name.js +95 -0
  5. package/bin/sensitive-data-sanitizer.js +430 -0
  6. package/bin/stream.js +56 -0
  7. package/docs/API.md +336 -0
  8. package/docs/COVERAGE.md +136 -0
  9. package/docs/HISTORY-REMEDIATION.md +41 -0
  10. package/examples/combined-scanners.mjs +23 -0
  11. package/examples/presidio-bridge.py +32 -0
  12. package/examples/realistic-fakes.mjs +18 -0
  13. package/examples/sanitize-session.mjs +27 -0
  14. package/examples/shared-tool-logs.mjs +21 -0
  15. package/examples/stream-jsonl.mjs +17 -0
  16. package/package.json +124 -4
  17. package/src/adapters.js +204 -0
  18. package/src/arithmetic.js +11 -0
  19. package/src/confidence.js +65 -0
  20. package/src/context.js +424 -0
  21. package/src/credentials.js +54 -0
  22. package/src/detection.js +293 -0
  23. package/src/encoded.js +233 -0
  24. package/src/engines.js +348 -0
  25. package/src/entities.js +166 -0
  26. package/src/escape-spans.js +20 -0
  27. package/src/eslint-plugin.d.ts +10 -0
  28. package/src/eslint-plugin.js +193 -0
  29. package/src/external.js +169 -0
  30. package/src/fake-names.js +222 -0
  31. package/src/fake.js +524 -0
  32. package/src/files.js +131 -0
  33. package/src/gitleaks.js +194 -0
  34. package/src/history-rewrite.js +339 -0
  35. package/src/history.js +103 -0
  36. package/src/identity.js +274 -0
  37. package/src/index.d.ts +321 -0
  38. package/src/index.js +40 -0
  39. package/src/jsonl-stream.js +198 -0
  40. package/src/keywords.js +45 -0
  41. package/src/known.js +164 -0
  42. package/src/legacy.d.ts +3 -0
  43. package/src/legacy.js +2 -0
  44. package/src/locales.js +217 -0
  45. package/src/log-bytes.js +85 -0
  46. package/src/names.js +65 -0
  47. package/src/national-checks.js +129 -0
  48. package/src/national.js +128 -0
  49. package/src/outbound.js +28 -0
  50. package/src/personal-context.js +85 -0
  51. package/src/personal.js +248 -0
  52. package/src/projection.js +126 -0
  53. package/src/propagation.js +163 -0
  54. package/src/public-verifier.js +77 -0
  55. package/src/public.js +153 -0
  56. package/src/record-worker.js +20 -0
  57. package/src/rules.js +210 -0
  58. package/src/sanitizer.js +794 -0
  59. package/src/specimens.js +34 -0
  60. package/src/stream-identity.js +129 -0
  61. package/src/stream-worker.js +18 -0
  62. package/src/stream.js +449 -0
  63. package/src/structured.js +278 -0
  64. package/src/transforms.js +192 -0
  65. package/src/transliteration.js +75 -0
  66. package/src/utf8.js +60 -0
  67. package/src/vendor/entities/aws.json +42 -0
  68. package/src/vendor/entities/azure.json +182 -0
  69. package/src/vendor/entities/google.json +219 -0
  70. package/src/vendor/gitleaks/LICENSE +21 -0
  71. package/src/vendor/gitleaks/provenance.json +8 -0
  72. package/src/vendor/gitleaks/rules-0.json +268 -0
  73. package/src/vendor/gitleaks/rules-1.json +1794 -0
  74. package/src/vendor/gitleaks/rules-2.json +292 -0
  75. package/src/vendor/gitleaks/rules-3.json +263 -0
  76. package/src/vendor/gitleaks/rules-4.json +277 -0
  77. package/src/vendor/gitleaks/rules-5.json +154 -0
  78. package/src/worker-options.js +27 -0
@@ -0,0 +1,274 @@
1
+ import { collectMatches } from './detection.js';
2
+
3
+ export const identityTypes = new Set([
4
+ 'SSN',
5
+ 'US_SSN',
6
+ 'BRAZIL_CPF_NUMBER',
7
+ 'INDIA_PAN_INDIVIDUAL',
8
+ 'UK_NATIONAL_INSURANCE_NUMBER',
9
+ 'SPAIN_DNI_NUMBER',
10
+ 'US_HEALTHCARE_NPI',
11
+ 'DRIVER_LICENSE',
12
+ 'DRIVERS_LICENSE',
13
+ 'NATIONAL_ID',
14
+ 'CHINA_RESIDENT_ID_NUMBER',
15
+ 'HONG_KONG_ID_NUMBER',
16
+ 'SOUTH_AFRICA_ID_NUMBER',
17
+ 'TAIWAN_ID_NUMBER',
18
+ 'TURKEY_ID_NUMBER',
19
+ 'GOVERNMENT_ID',
20
+ 'DOD_ID_NUMBER',
21
+ 'JAPAN_INDIVIDUAL_NUMBER',
22
+ 'INDIA_GST_INDIVIDUAL',
23
+ 'US_MEDICARE_BENEFICIARY_ID_NUMBER',
24
+ 'ID',
25
+ 'PASSPORT_NUMBER',
26
+ 'PASSPORT_MRZ',
27
+ 'BOOKING_REFERENCE',
28
+ 'TICKET_NUMBER',
29
+ 'VISA_NUMBER',
30
+ 'UK_NHS',
31
+ 'KR_RRN',
32
+ 'IT_FISCAL_CODE',
33
+ 'SG_NRIC_FIN',
34
+ 'PL_PESEL',
35
+ 'SE_PERSONNUMMER',
36
+ 'ZA_ID_NUMBER',
37
+ 'AU_TFN',
38
+ 'CA_SIN',
39
+ 'RU_SNILS',
40
+ 'RU_INN',
41
+ ]);
42
+ export function isIdentityType(type) {
43
+ return (
44
+ identityTypes.has(type) ||
45
+ /PASSPORT|(?:DRIVERS?|DRIVING)_?LICENSE|IDENTITY|NATIONAL_?(?:ID|IDENTIFICATION)|PERSONAL_?(?:ID|IDENTIFICATION)|UNIQUE_?IDENTIFICATION|SOCIAL_?(?:SECURITY|INSURANCE|IDENTIFICATION)|RESIDENT_?REGISTRATION|(?:TAX|TAXPAYER).*?(?:IDENTIFICATION|NUMBER|REFERENCE)|AADHAAR|FISCAL_CODE|PESEL|RRN|NHS|SNILS|INN_NUMBER/.test(
46
+ type
47
+ )
48
+ );
49
+ }
50
+ const personal = (type, rule, confidence = 0.85) => ({
51
+ type,
52
+ rule,
53
+ category: 'personal',
54
+ confidence,
55
+ });
56
+ const passportContext =
57
+ /(?<![\p{L}\p{N}_])(?:passports?|series and number|document number|паспорт[\p{L}]*|загран[\p{L}]*|серия|номер|pasaporte|Reisepass|passeport|passaporto|护照|パスポート)(?![\p{L}\p{N}_])/iu;
58
+
59
+ export function mrzCheck(value) {
60
+ let sum = 0;
61
+ for (let i = 0; i < value.length; i++) {
62
+ const char = value[i];
63
+ const digit =
64
+ char === '<'
65
+ ? 0
66
+ : /\d/.test(char)
67
+ ? Number(char)
68
+ : char.charCodeAt(0) - 55;
69
+ sum += digit * [7, 3, 1][i % 3];
70
+ }
71
+ return String(sum % 10);
72
+ }
73
+ function mrzConfidence(value) {
74
+ const compact = value.replace(/[ \t\r\n]/g, '');
75
+ const match =
76
+ /^([A-Z0-9<]{9})([\d<])[A-Z<]{3}(\d{6})(\d)[MF<](\d{6})([\d<])/.exec(
77
+ compact
78
+ );
79
+ if (match) {
80
+ return mrzCheck(match[1]) === match[2] &&
81
+ mrzCheck(match[3]) === match[4] &&
82
+ mrzCheck(match[5]) === match[6]
83
+ ? 0.99
84
+ : 0.85;
85
+ }
86
+ const td1Header = /^[IAC][A-Z<][A-Z<]{3}([A-Z0-9<]{9})([\d<])/.exec(compact);
87
+ if (td1Header) {
88
+ return mrzCheck(td1Header[1]) === td1Header[2] ? 0.99 : 0.85;
89
+ }
90
+ const td1Data = /^([\d<]{6})(\d)[MF<]([\d<]{6})(\d)[A-Z<]{3}/.exec(compact);
91
+ return td1Data &&
92
+ mrzCheck(td1Data[1]) === td1Data[2] &&
93
+ mrzCheck(td1Data[3]) === td1Data[4]
94
+ ? 0.99
95
+ : 0.85;
96
+ }
97
+ function detectMrz(text, emit) {
98
+ // Each candidate has a finite span. Checks improve confidence, never gate
99
+ // redaction: OCR damage and truncated personal data remain sensitive.
100
+ const patterns = [
101
+ /(?<![A-Za-z0-9_])[PIACV][A-Z<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}[A-Z0-9<][A-Z0-9< \t\r\n]{8,100}/g,
102
+ /(?<![A-Za-z0-9_])[A-Z]{2,30}<<[A-Z< \t\r\n]{2,72}/g,
103
+ /(?<![A-Za-z0-9_])[A-Z0-9<]{9}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}[\d<]{6}[\d<][ \t\r\n]{0,4}[MF<][ \t\r\n]{0,4}[\d<]{6}[\d<][A-Z0-9< \t\r\n]{0,40}/g,
104
+ /(?<![A-Za-z0-9_])[\d<]{6}[\d<][ \t\r\n]{0,4}[MF<][ \t\r\n]{0,4}[\d<]{6}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[A-Z0-9< \t\r\n]{4,20}/g,
105
+ /(?<![A-Za-z0-9_])[A-Z0-9<]{9}[\d<][ \t\r\n]{0,4}[A-Z<]{3}[ \t\r\n]{0,4}\d{6}(?:\d[ \t\r\n]{0,4}[MF<](?:[ \t\r\n]{0,4}[\d<]{1,7})?)?/g,
106
+ ];
107
+ patterns.forEach((pattern, index) => {
108
+ for (const match of text.matchAll(pattern)) {
109
+ const value = (
110
+ /[a-z]/.test(text[match.index + match[0].length] ?? '')
111
+ ? match[0].replace(/(?:[ \t\r\n]+)[A-Z]{1,6}$/, '')
112
+ : match[0]
113
+ ).trimEnd(),
114
+ compact = value.replace(/[ \t\r\n]/g, '');
115
+ const validShape =
116
+ index === 0
117
+ ? /^[PIACV][A-Z<][A-Z<]{3}(?:[A-Z]{2,}<<[A-Z<]+|[A-Z0-9<]{9}[\d<]<{2,})/.test(
118
+ compact
119
+ )
120
+ : index === 1
121
+ ? /^[A-Z]{2,}<<[A-Z]+</.test(compact)
122
+ : index === 3
123
+ ? /\d/.test(compact.slice(0, 7))
124
+ : /[A-Z0-9]/.test(compact.slice(0, 9));
125
+ if (validShape) {
126
+ emit({
127
+ start: match.index,
128
+ end: match.index + value.length,
129
+ ...personal('PASSPORT_MRZ', 'mrz', mrzConfidence(value)),
130
+ });
131
+ }
132
+ }
133
+ });
134
+ }
135
+ function numericNeighbour(text, start, end) {
136
+ return (
137
+ /\d[ ()+.-]{0,4}$/.test(text.slice(Math.max(0, start - 5), start)) ||
138
+ /^[ ()+.-]{0,4}\d/.test(text.slice(end, end + 5))
139
+ );
140
+ }
141
+ function detectPassports(text, emit) {
142
+ const patterns = [
143
+ /(?<![\p{L}\p{N}_])(?:\d{2}[ -]\d{2}[ -]\d{6}|\d{4}[ -]\d{6}|\d{2}[ -]\d{7})(?![\p{L}\p{N}_])/gu,
144
+ /(?<![\p{L}\p{N}_])(?:\d{9}|[CFGHJK][0-9CFGHJKLMNPRTVWXYZ]{8}|[A-Z]\d{7}|M\d{3}[A-Z]\d{4}|[A-Z]{2}\d{7})(?![\p{L}\p{N}_])/gu,
145
+ /(?<![\p{L}\p{N}_])[A-Z0-9<]{9}\d(?![\p{L}\p{N}_])/gu,
146
+ ];
147
+ patterns.forEach((pattern, index) => {
148
+ for (const match of text.matchAll(pattern)) {
149
+ const value = match[0],
150
+ start = match.index,
151
+ end = start + value.length;
152
+ const context = text.slice(
153
+ Math.max(0, start - 80),
154
+ Math.min(text.length, end + 80)
155
+ );
156
+ const checked =
157
+ index === 2 &&
158
+ /[A-Z]/.test(value) &&
159
+ mrzCheck(value.slice(0, 9)) === value[9];
160
+ if ((index === 2 && !checked) || numericNeighbour(text, start, end)) {
161
+ continue;
162
+ }
163
+ if (
164
+ /^\d+$/.test(value) &&
165
+ /(?:timestamp|epoch|version|commit|thread|call)[ :=_-]*$/i.test(
166
+ text.slice(Math.max(0, start - 24), start)
167
+ )
168
+ ) {
169
+ continue;
170
+ }
171
+ emit({
172
+ start,
173
+ end,
174
+ ...personal(
175
+ 'PASSPORT_NUMBER',
176
+ 'passport-format',
177
+ checked ? 0.99 : passportContext.test(context) ? 0.9 : 0.4
178
+ ),
179
+ });
180
+ }
181
+ });
182
+ }
183
+ const booking =
184
+ /(?<![\p{L}\p{N}_])(?:booking(?:[ _-]+(?:reference|code))?|reservation(?:[ _-]+code)?|confirmation(?:[ _-]+code)?|бронь|record[ _-]+locator|PNR|код брони|номер брони|бронирование)[ \t]*[:=:]?[ \t]*[`"']?([A-Z0-9]{4,8})(?![\p{L}\p{N}_])/giu;
185
+ const ticket =
186
+ /(?<![\p{L}\p{N}_])(?:e[ -]?ticket|ticket[ _-]*(?:number|no)?|номер билета|электронный билет)[ \t]*[:=:]?[ \t]*[`"']?(\d{3}[ -]?\d{10})(?![\p{L}\p{N}_])/giu;
187
+ const visa =
188
+ /(?<![\p{L}\p{N}_])(?:e[ -]?visa|visa|номер визы|виза)(?:[ \t_-]+(?:registration|application|number|no|code|номер|код)){0,3}[ \t]*[:=:]?[ \t]*[`"']?([A-Z0-9][A-Z0-9-]{4,31})(?![\p{L}\p{N}_])/giu;
189
+ const placeholder =
190
+ /^(?:NONE|UNKNOWN|NULL|UNDEFINED|REDACTED|EXAMPLE|SAMPLE|PENDING|HOTEL|BOOKING|REFERENCE|REGISTRATION|APPLICATION|NUMBER|CODE|FAILED|ERROR|HTTP|OK|CANCELLED|CONFIRMED|SUCCESS|READY|ACTIVE|CLOSED)$/i;
191
+ function detectTravel(text, emit) {
192
+ collectMatches(
193
+ text,
194
+ booking,
195
+ emit,
196
+ personal('BOOKING_REFERENCE', 'booking-context'),
197
+ 1,
198
+ (v) => !placeholder.test(v) && (v === v.toUpperCase() || /\d/.test(v))
199
+ );
200
+ collectMatches(
201
+ text,
202
+ /(?:AirIndia|Air-?India|AirFrance|BritishAirways|Lufthansa|Emirates|Qatar|Delta|United|Ryanair|Aeroflot|Аэрофлот)[-_]([A-Z0-9]{5,8})(?=[-_.])/g,
203
+ emit,
204
+ personal('BOOKING_REFERENCE', 'booking-filename', 0.9),
205
+ 1,
206
+ (v) => !placeholder.test(v)
207
+ );
208
+ collectMatches(
209
+ text,
210
+ ticket,
211
+ emit,
212
+ personal('TICKET_NUMBER', 'ticket-context'),
213
+ 1
214
+ );
215
+ collectMatches(
216
+ text,
217
+ visa,
218
+ emit,
219
+ personal('VISA_NUMBER', 'visa-context'),
220
+ 1,
221
+ (v) => !placeholder.test(v) && /\d/.test(v)
222
+ );
223
+ }
224
+
225
+ export function detectIdentity(text, emit) {
226
+ detectMrz(text, emit);
227
+ detectPassports(text, emit);
228
+ detectTravel(text, emit);
229
+ }
230
+
231
+ export function fakeIdentityValue(value, type, options) {
232
+ if (!isIdentityType(type)) {
233
+ return false;
234
+ }
235
+ if (options.fakeValues?.includes(value)) {
236
+ return true;
237
+ }
238
+ if (options.fakeIdentity !== 'specimen-and-synthetic') {
239
+ return false;
240
+ }
241
+ const compact = value.replace(/\s/g, '').toUpperCase();
242
+ if (/^(?:SPECIMEN|ОБРАЗЕЦ|MUSTERMANN)$/.test(compact)) {
243
+ return true;
244
+ }
245
+ if (type === 'PASSPORT_MRZ') {
246
+ if (
247
+ /^[PIACV][A-Z<]UTO/.test(compact) ||
248
+ /^(?:UTO)?ERIKSSON<<ANNA<MARIA</.test(compact) ||
249
+ /^L898902C3[\d<]UTO/.test(compact) ||
250
+ /^[\d<]{7}[MF<][\d<]{7}UTO/.test(compact)
251
+ ) {
252
+ return true;
253
+ }
254
+ const document = /^[PIACV][A-Z<][A-Z<]{3}/.test(compact)
255
+ ? compact.slice(5, 14)
256
+ : compact.slice(0, 9);
257
+ return (
258
+ /^(?:C01X00T47|T22000129)$/.test(document) || syntheticDocument(document)
259
+ );
260
+ }
261
+ return (
262
+ /^(?:L898902C3(?:6)?|D23145890(?:7)?|C01X00T47|T22000129)$/.test(compact) ||
263
+ syntheticDocument(compact)
264
+ );
265
+ }
266
+ function syntheticDocument(value) {
267
+ const digits = value.replace(/[ \t-]/g, '');
268
+ return (
269
+ (digits.length >= 5 && /^(\d)\1+$/.test(digits)) ||
270
+ /012345|123456|234567|345678|456789|987654|876543|765432|654321|543210/.test(
271
+ digits
272
+ )
273
+ );
274
+ }
package/src/index.d.ts ADDED
@@ -0,0 +1,321 @@
1
+ export type Category = 'credential' | 'personal';
2
+ /** UTF-16 offsets, zero-based start and exclusive end; never contains source values. */
3
+ export interface Finding {
4
+ start: number;
5
+ end: number;
6
+ category: Category;
7
+ type: string;
8
+ rule: string;
9
+ /** Detector confidence, not a cross-engine probability calibration. */
10
+ confidence?: number;
11
+ likelihood?:
12
+ | 'VERY_UNLIKELY'
13
+ | 'UNLIKELY'
14
+ | 'POSSIBLE'
15
+ | 'LIKELY'
16
+ | 'VERY_LIKELY';
17
+ /** One-based line and UTF-16 column, populated by inspect. */
18
+ line?: number;
19
+ column?: number;
20
+ /** Reviewed/generated fake exemption; generated credentials require explicit opt-in. */
21
+ kept?: 'fake';
22
+ /** Returned finding replaced with, or recognized as, a generated fake. */
23
+ faked?: true;
24
+ }
25
+ export interface PersonalValue {
26
+ type: string;
27
+ value: string;
28
+ }
29
+ export interface PublicEntity {
30
+ type: 'PERSON' | 'ORGANIZATION' | 'EMAIL';
31
+ value: string;
32
+ /** Organization-controlled HTTPS evidence, reviewed by the policy author. */
33
+ source: string;
34
+ /** ISO calendar date. The library does not fetch the evidence. */
35
+ reviewedAt: string;
36
+ }
37
+ export interface Detector {
38
+ id?: string;
39
+ /** Required detectors must throw on failure and return offsets for this exact input. */
40
+ detect(text: string): Finding[] | Promise<Finding[]>;
41
+ }
42
+ export interface SanitizerOptions {
43
+ /** Preserve all personal/account data; credential findings still redact. */
44
+ credentialsOnly?: boolean;
45
+ profile?: 'publication';
46
+ confirmedPersonal?: Array<{ value: string; type: string }>;
47
+ knownSecrets?: string[];
48
+ knownPersonal?: PersonalValue[];
49
+ publicEntities?: PublicEntity[];
50
+ /** Built-in exact public entities/resolvers and role mail on known domains. */
51
+ publicKnowledge?: boolean;
52
+ findings?: Finding[];
53
+ transformation?: Transformation;
54
+ transformations?: Record<string, Transformation>;
55
+ preserveEncoding?: boolean;
56
+ /** First/last two characters and a fixed *** marker for identity values >=5. */
57
+ identityMask?: boolean;
58
+ /** Explicit opt-in. Words such as test/fake/example never grant exemptions. */
59
+ fakeIdentity?: false | 'specimen-and-synthetic';
60
+ /** Reviewed exact fake identity values, maintained outside published text. */
61
+ fakeValues?: string[];
62
+ /** Scan JSON string keys/values in bounded batches and preserve parseability. */
63
+ structured?: 'json' | 'jsonl';
64
+ /** Property names whose metadata heuristics may be suppressed. */
65
+ structuralFields?: string[];
66
+ /** Personal replacement threshold, default 0.5. Inspect retains lower scores. */
67
+ minConfidence?: number;
68
+ /** Alias for minConfidence; specifying both requires equal values. */
69
+ threshold?: number;
70
+ /** Explicit network/callback verification; credentials never qualify. */
71
+ verifyPublic?: (candidate: {
72
+ type: PublicEntity['type'];
73
+ value: string;
74
+ }) => Promise<PublicEntity | undefined>;
75
+
76
+ paranoid?: boolean;
77
+ decode?: boolean;
78
+ maxInputLength?: number;
79
+ maxFindings?: number;
80
+ debug?: (event: { event: 'detected'; count: number }) => void;
81
+ /** Used by createSanitizer; synchronous functions use native rules only. */
82
+ detectors?: Detector[];
83
+ /** Required by default in createSanitizer. */
84
+ secretlint?: boolean;
85
+ }
86
+ export interface SanitizedText {
87
+ text: string;
88
+ findings: Finding[];
89
+ /** Number of disjoint replacements after overlapping detections are merged. */
90
+ redactions: number;
91
+ }
92
+ export interface Sanitizer {
93
+ inspect(text: string): Promise<Finding[]>;
94
+ sanitize(text: string): Promise<SanitizedText>;
95
+ }
96
+ export interface JsonlSanitizer extends Sanitizer {
97
+ sanitizeJsonl(text: string): Promise<SanitizedText>;
98
+ }
99
+ export declare const REDACTED: '[REDACTED]';
100
+ export declare function inspect(
101
+ text: string,
102
+ options?: SanitizerOptions
103
+ ): Finding[];
104
+ export declare function sanitize(
105
+ text: string,
106
+ options?: SanitizerOptions
107
+ ): SanitizedText;
108
+ /** Apply validated findings; use sanitize for automatic residual verification. */
109
+ export declare function redact(
110
+ text: string,
111
+ findings: Finding[],
112
+ options?: SanitizerOptions
113
+ ): string;
114
+ export declare function createSanitizer(
115
+ options?: SanitizerOptions
116
+ ): JsonlSanitizer;
117
+ export declare function entropy(value: string): number;
118
+ export declare function luhn(value: string): boolean;
119
+ export declare function codePointRange(
120
+ text: string,
121
+ start: number,
122
+ end: number
123
+ ): { start: number; end: number };
124
+ export declare function byteRange(
125
+ text: string,
126
+ start: number,
127
+ end: number
128
+ ): { start: number; end: number };
129
+ export declare function fromGitleaks(
130
+ text: string,
131
+ report: Array<{ Secret: string; [key: string]: unknown }>
132
+ ): Finding[];
133
+ export declare function fromTrufflehog(
134
+ text: string,
135
+ report: Array<{ Raw: string; RawV2?: string; [key: string]: unknown }>
136
+ ): Finding[];
137
+ export declare function fromDetectSecrets(
138
+ text: string,
139
+ report: {
140
+ results: Record<
141
+ string,
142
+ Array<{ line_number: number; [key: string]: unknown }>
143
+ >;
144
+ },
145
+ filename: string
146
+ ): Finding[];
147
+ export declare function fromPresidio(
148
+ text: string,
149
+ report: Array<{
150
+ start: number;
151
+ end: number;
152
+ entity_type: string;
153
+ [key: string]: unknown;
154
+ }>
155
+ ): Finding[];
156
+ export declare function fromOffsetReport(
157
+ text: string,
158
+ entries: Array<{
159
+ start: number;
160
+ end: number;
161
+ category: Category;
162
+ type: string;
163
+ [key: string]: unknown;
164
+ }>,
165
+ options: { unit: 'utf16' | 'codepoint' | 'byte'; rule: string }
166
+ ): Finding[];
167
+ export declare function knownSecretsFromEnv(
168
+ env?: Record<string, string | undefined>,
169
+ names?: string[]
170
+ ): string[];
171
+ /** Reversible UTF-8 view: invalid bytes use unpaired U+DC80..U+DCFF. */
172
+ export declare function decodeLogBytes(bytes: Uint8Array): string;
173
+ /** Encode sanitized text while restoring surrogate-escaped invalid bytes. */
174
+ export declare function encodeLogText(text: string): Uint8Array;
175
+ export declare function knownSecretsFromGitHubConfig(options?: {
176
+ path?: string;
177
+ }): Promise<string[]>;
178
+ export declare function personalVariants(
179
+ entries: PersonalValue[]
180
+ ): PersonalValue[];
181
+ export interface LocalScannerOptions {
182
+ command?: string;
183
+ timeoutMs?: number;
184
+ maxOutputBytes?: number;
185
+ }
186
+ export declare function createGitleaksDetector(
187
+ options?: LocalScannerOptions
188
+ ): Detector;
189
+ export declare function createTrufflehogDetector(
190
+ options?: LocalScannerOptions
191
+ ): Detector;
192
+ export declare function createPresidioDetector(
193
+ options: LocalScannerOptions & { args: string[] }
194
+ ): Detector;
195
+ export interface HistoryAudit {
196
+ scanned: number;
197
+ complete: boolean;
198
+ findings: Array<{
199
+ object: string;
200
+ type: 'blob' | 'commit' | 'tag';
201
+ findings: Finding[];
202
+ }>;
203
+ skipped: Array<{ object: string; reason: string }>;
204
+ }
205
+ export declare function auditGitHistory(
206
+ repository: string,
207
+ options?: {
208
+ sanitizer?: Pick<Sanitizer, 'inspect'>;
209
+ maxObjects?: number;
210
+ maxBytes?: number;
211
+ maxTotalBytes?: number;
212
+ }
213
+ ): Promise<HistoryAudit>;
214
+ export type Transformation =
215
+ | { mode: 'redact' | 'hive-mask' }
216
+ | {
217
+ mode: 'mask';
218
+ keepStart?: number;
219
+ keepEnd?: number;
220
+ marker?: string;
221
+ minLength?: number;
222
+ }
223
+ | { mode: 'pseudonym' | 'format-preserving'; key: string }
224
+ | {
225
+ mode: 'fake';
226
+ /** At least 16 UTF-8 bytes; keep this pseudonymisation key private. */
227
+ key: string;
228
+ /** Emit visibly invalid fake credentials; default is full redaction. */
229
+ credentials?: boolean;
230
+ /** Keep original MRZ issuer/nationality unless Utopia is requested. */
231
+ mrzCountry?: 'UTO';
232
+ }
233
+ | { mode: 'date-shift'; days: number }
234
+ | { mode: 'bucket'; size: number };
235
+ export interface StreamOptions extends SanitizerOptions {
236
+ /** Atomic file APIs only: spool input privately, collect, then redact. */
237
+ twoPass?: boolean;
238
+ maxRegistryValues?: number;
239
+ sanitizerOptions?: SanitizerOptions;
240
+ sanitizer?: Sanitizer;
241
+ maxRecordBytes?: number;
242
+ batchBytes?: number;
243
+ maxTotalBytes?: number;
244
+ worker?: boolean;
245
+ /** JSONL record workers, 1–16; default 2, or 1 for callbacks/Deno. */
246
+ workers?: number;
247
+ workerHeapMb?: number;
248
+ workerTimeoutMs?: number;
249
+ replace?: boolean;
250
+ }
251
+ export declare function sanitizeStream(
252
+ source: AsyncIterable<string | Uint8Array> | Iterable<string | Uint8Array>,
253
+ options?: StreamOptions
254
+ ): AsyncGenerator<string>;
255
+ export declare function sanitizeStreamToFile(
256
+ source: AsyncIterable<string | Uint8Array> | Iterable<string | Uint8Array>,
257
+ path: string,
258
+ options?: StreamOptions
259
+ ): Promise<{ outputBytes: number }>;
260
+ export declare function sanitizeFileToFile(
261
+ source: string,
262
+ target: string,
263
+ options?: StreamOptions
264
+ ): Promise<{ inputBytes: number; outputBytes: number }>;
265
+ export declare function sanitizeFileBounded(
266
+ source: string,
267
+ target: string,
268
+ options?: StreamOptions
269
+ ): Promise<{ inputBytes: number; outputBytes: number }>;
270
+ export declare const entityCatalogs: Record<
271
+ 'google' | 'azure' | 'aws',
272
+ { source: string; sha256: string; names: string[] }
273
+ >;
274
+ export declare function createWikidataVerifier(options?: {
275
+ fetch?: typeof fetch;
276
+ language?: string;
277
+ timeoutMs?: number;
278
+ maxEntries?: number;
279
+ }): NonNullable<SanitizerOptions['verifyPublic']>;
280
+ export declare function knownSecretsFromGitHubAuth(options?: {
281
+ command?: string;
282
+ hostname?: string;
283
+ timeoutMs?: number;
284
+ }): Promise<string[]>;
285
+ export interface OutboundOptions {
286
+ sanitizer?: Pick<Sanitizer, 'sanitize'>;
287
+ sanitizerOptions?: SanitizerOptions;
288
+ }
289
+ export declare function sanitizePayload<T>(
290
+ value: T,
291
+ options?: OutboundOptions
292
+ ): Promise<T>;
293
+ export declare function createSentryBeforeSend<T>(
294
+ options?: OutboundOptions
295
+ ): (event: T) => Promise<T | null>;
296
+ export declare function createOutboundSanitizer<T, R>(
297
+ publish: (payload: T) => R | Promise<R>,
298
+ options?: OutboundOptions
299
+ ): (payload: T) => Promise<R>;
300
+ export interface HistoryRewrite {
301
+ applied: boolean;
302
+ repository: string;
303
+ before: HistoryAudit;
304
+ after?: HistoryAudit;
305
+ changedPaths: number;
306
+ changedRefs: number;
307
+ }
308
+ export declare function rewriteGitHistory(
309
+ source: string,
310
+ destination: string,
311
+ options?: {
312
+ apply?: boolean;
313
+ sanitizer?: Sanitizer;
314
+ sanitizerOptions?: SanitizerOptions;
315
+ filterRepoCommand?: string;
316
+ maxPaths?: number;
317
+ maxObjects?: number;
318
+ maxBytes?: number;
319
+ maxTotalBytes?: number;
320
+ }
321
+ ): Promise<HistoryRewrite>;
package/src/index.js ADDED
@@ -0,0 +1,40 @@
1
+ /** Sensitive text detection and verified publication APIs. */
2
+
3
+ export { inspect, sanitize, redact } from './sanitizer.js';
4
+ export { createSanitizer } from './engines.js';
5
+ export { REDACTED, entropy, luhn } from './detection.js';
6
+ export { codePointRange } from './projection.js';
7
+ export {
8
+ fromGitleaks,
9
+ fromTrufflehog,
10
+ fromDetectSecrets,
11
+ fromPresidio,
12
+ byteRange,
13
+ fromOffsetReport,
14
+ } from './adapters.js';
15
+ export { knownSecretsFromEnv, personalVariants } from './known.js';
16
+ export { auditGitHistory } from './history.js';
17
+ export {
18
+ createGitleaksDetector,
19
+ createTrufflehogDetector,
20
+ createPresidioDetector,
21
+ } from './external.js';
22
+
23
+ export { entityCatalogs } from './entities.js';
24
+ export {
25
+ sanitizeStream,
26
+ sanitizeStreamToFile,
27
+ sanitizeFileToFile,
28
+ sanitizeFileBounded,
29
+ } from './stream.js';
30
+
31
+ export { createWikidataVerifier } from './public-verifier.js';
32
+ export {
33
+ sanitizePayload,
34
+ createSentryBeforeSend,
35
+ createOutboundSanitizer,
36
+ } from './outbound.js';
37
+ export { knownSecretsFromGitHubAuth } from './known.js';
38
+ export { knownSecretsFromGitHubConfig } from './known.js';
39
+ export { decodeLogBytes, encodeLogText } from './log-bytes.js';
40
+ export { rewriteGitHistory } from './history-rewrite.js';