kaloko 0.12.0 → 0.12.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/package.json +1 -1
- package/schemas/config.v1.json +6 -4
- package/skills/kaloko/reference/acceptance.md +2 -2
- package/src/cli.mjs +11 -7
- package/src/lib/build.mjs +1 -1
- package/src/lib/capture.mjs +88 -44
- package/src/lib/changelog-core.mjs +3 -1
- package/src/lib/evaluate.mjs +12 -9
- package/src/lib/htmlview.mjs +3 -1
- package/src/lib/keyboard.mjs +4 -2
- package/src/lib/pii.mjs +529 -212
- package/src/lib/recording.mjs +12 -3
- package/src/lib/triage-core.mjs +3 -1
- package/src/lib/triage.mjs +17 -5
- package/src/lib/video.mjs +2 -0
- package/viewer/index.html +82 -19
package/src/lib/pii.mjs
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
// Personal data in captures (#64): one mask for every output of a step. The mask is applied to the page itself for the
|
|
2
|
-
// moment of the capture (maskInPage): the screenshot, the stored HTML, the page outline for the evaluators
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// (
|
|
6
|
-
//
|
|
7
|
-
//
|
|
2
|
+
// moment of the capture (maskInPage): the screenshot, the stored HTML, the page outline for the evaluators and the HTML
|
|
3
|
+
// view all see placeholders ("[hidden]") or consistent fake values (pseudonymize), and the page gets its own text back
|
|
4
|
+
// right after. Attributes nobody sees (data-*, content, links) are not touched in the live page: their masked values
|
|
5
|
+
// are applied to the stored HTML (maskHtml). Recordings get an overlay that covers the same places while the walk runs
|
|
6
|
+
// (maskPainter). Every value masked is remembered for the run (rememberPii, kept in the run folder for the next
|
|
7
|
+
// process), so traces, console lines, capture records and later steps replace it too. `kaloko share` scans what it is
|
|
8
|
+
// about to upload (piiFindings): a run that asked for hiding personal data is refused with unmasked e-mails, phone
|
|
9
|
+
// numbers, birth numbers or IBANs in it (unless --allow-pii); any other run gets a warning.
|
|
8
10
|
//
|
|
9
11
|
// piiKit, maskInPage and maskPainter run inside the browser: keep them self-contained (no outer references).
|
|
10
|
-
import { readFile } from 'node:fs/promises'
|
|
12
|
+
import { readFile, writeFile } from 'node:fs/promises'
|
|
11
13
|
import { join } from 'node:path'
|
|
12
14
|
import { UserError } from './util.mjs'
|
|
13
15
|
|
|
14
16
|
/**
|
|
15
17
|
* Detection, placeholders and fake values. Self-contained: Node calls it directly, the page gets it serialised.
|
|
16
|
-
* find(text, { kinds, allow }) → [{ kind, start, end, value }] in text order, without overlaps.
|
|
18
|
+
* find(text, { kinds, allow, context }) → [{ kind, start, end, value, plain? }] in text order, without overlaps.
|
|
19
|
+
* `context` is the text just before (a label in the previous element); `plain` is the decoded value of a hit in
|
|
20
|
+
* percent-encoded text (?email=jan%40firma.cz).
|
|
17
21
|
*/
|
|
18
22
|
export function piiKit () {
|
|
19
23
|
const KINDS = ['email', 'phone', 'birth_number', 'account', 'iban', 'birth_date', 'address', 'name']
|
|
@@ -22,10 +26,12 @@ export function piiKit () {
|
|
|
22
26
|
const digits = (s) => String(s).replace(/\D/g, '')
|
|
23
27
|
const wild = (p) => new RegExp('^' + String(p).trim().split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$', 'i')
|
|
24
28
|
const allowed = (value, allow) => (allow || []).some((p) => wild(p).test(value.trim()) || (digits(p).length >= 6 && !String(p).includes('*') && digits(p) === digits(value)))
|
|
29
|
+
const DAYS = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
|
|
25
30
|
const birthNumberOk = (yy, mm, dd, tail) => {
|
|
26
31
|
let m = Number(mm); const d = Number(dd)
|
|
27
|
-
|
|
28
|
-
if (m
|
|
32
|
+
// women +50; since 2004 also +20 (men) and +70 (women), only in the ten-digit form
|
|
33
|
+
if (m > 70 && tail.length === 4) m -= 70; else if (m > 50) m -= 50; else if (m > 20 && tail.length === 4) m -= 20
|
|
34
|
+
if (m < 1 || m > 12 || d < 1 || d > DAYS[m - 1]) return false
|
|
29
35
|
if (tail.length === 3) return Number(yy) < 54
|
|
30
36
|
const n = Number(`${yy}${mm}${dd}${tail}`)
|
|
31
37
|
return n % 11 === 0 || (Math.floor(n / 10) % 11 === 10 && n % 10 === 0)
|
|
@@ -42,29 +48,48 @@ export function piiKit () {
|
|
|
42
48
|
for (const c of r) m = (m * 10 + Number(c)) % 97
|
|
43
49
|
return m === 1
|
|
44
50
|
}
|
|
45
|
-
|
|
51
|
+
// an amount, not a phone number: a currency or decimals right after it, a currency right before it
|
|
52
|
+
const CURRENCY = /^\s?(Kč|Kc|CZK|EUR|€|\$|USD|£|GBP|,-|[.,]\d)/i
|
|
53
|
+
const CURRENCY_BEFORE = /(Kč|CZK|EUR|€|\$|USD|£|GBP)\s?$/i
|
|
54
|
+
// a phone number written 603 123 456 counts only next to a word that says so (or another phone number)
|
|
55
|
+
const TEL = /(?:^|[^\p{L}])(?:tel|telefon|telephone|phone|mobil|mobile|mob|gsm|fax|volejte|zavolejte|call|kontakt|contact|sms|whatsapp|hotline|linka)\p{L}*\.?(?![\p{L}])|[☎📞]|\+\d{2,3}[ ]?\d{3}/iu
|
|
56
|
+
// a birth number without the slash, or in the old nine-digit form, only next to its label
|
|
57
|
+
const BIRTH = /(?:^|[^\p{L}])(?:rodn\p{L}*|r\.\s?č\.?|rč|birth\s?number|personal\s?(?:id|number|identification))(?![\p{L}])/iu
|
|
58
|
+
const before = (text, at, ctx, n = 60) => (String(ctx || '') + ' ' + text.slice(Math.max(0, at - n), at)).slice(-n)
|
|
46
59
|
const finders = {
|
|
47
60
|
email (text, out) {
|
|
48
|
-
|
|
61
|
+
if (!text.includes('@')) return
|
|
62
|
+
// the local part at most 64 characters and from its start (a lookbehind): no quadratic search in long words
|
|
63
|
+
for (const m of text.matchAll(/(?<![\p{L}\p{N}._%+-])[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){0,8}\.(\p{L}{2,24}))(?![\p{L}\p{N}])/gu)) {
|
|
49
64
|
if (RESERVED.test(m[1]) || FILE_TLD.test(m[2]) || m[0].startsWith('.')) continue
|
|
50
65
|
out.push({ kind: 'email', start: m.index, end: m.index + m[0].length })
|
|
51
66
|
}
|
|
52
67
|
},
|
|
53
|
-
phone (text, out) {
|
|
54
|
-
// international (+420 603 123 456, 00420
|
|
55
|
-
for (const m of text.matchAll(/(?<![\p{L}\p{N}
|
|
68
|
+
phone (text, out, ctx) {
|
|
69
|
+
// international (+420 603 123 456, 00420 603 123 456); a fake keeps 000 after the code
|
|
70
|
+
for (const m of text.matchAll(/(?<![\p{L}\p{N}+\-./_#])(?:\+|00)\d{1,3}[ .-]?(?:\(\d{1,4}\)[ .-]?)?\d{1,4}(?:[ .-]?\d{2,4}){1,4}(?![\p{N}])/gu)) {
|
|
56
71
|
const n = digits(m[0]).replace(/^00/, '')
|
|
57
|
-
if (n.length < 9 || n.length > 15 || /^\+?(00)?\d{1,3}[
|
|
58
|
-
|
|
72
|
+
if (n.length < 9 || n.length > 15 || /^\+?(00)?\d{1,3}[ .-]?0{3}/.test(m[0])) continue
|
|
73
|
+
// order numbers and EANs start with 00 too: 00 only with a space after the country code
|
|
74
|
+
if (m[0].startsWith('00') && !/^00\d{1,3}[ ]\d/.test(m[0])) continue
|
|
75
|
+
const end = m.index + m[0].length
|
|
76
|
+
if (CURRENCY.test(text.slice(end, end + 6)) || CURRENCY_BEFORE.test(text.slice(Math.max(0, m.index - 6), m.index))) continue
|
|
77
|
+
out.push({ kind: 'phone', start: m.index, end })
|
|
59
78
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
79
|
+
// Czech grouped 3-3-3 (603 123 456), with a label nearby; amounts (250 000 000 Kč, 349 990 000,00) are not
|
|
80
|
+
for (const m of text.matchAll(/(?<![\p{N}+][ ]?)(?<![\p{L}\p{N}.,/:#-])[2-7]\d{2}([ ])\d{3}\1\d{3}(?![\p{N}])/gu)) {
|
|
81
|
+
const end = m.index + m[0].length
|
|
82
|
+
if (CURRENCY.test(text.slice(end, end + 6)) || CURRENCY_BEFORE.test(text.slice(Math.max(0, m.index - 6), m.index))) continue
|
|
83
|
+
if (!TEL.test(before(text, m.index, ctx))) continue
|
|
84
|
+
out.push({ kind: 'phone', start: m.index, end })
|
|
63
85
|
}
|
|
64
86
|
},
|
|
65
|
-
birth_number (text, out) {
|
|
66
|
-
for (const m of text.matchAll(/(?<![\p{N}
|
|
67
|
-
|
|
87
|
+
birth_number (text, out, ctx) {
|
|
88
|
+
for (const m of text.matchAll(/(?<![\p{N}/.,-])(\d{2})(\d{2})(\d{2})( ?\/ ?)?(\d{3,4})(?![\p{N}/])/gu)) {
|
|
89
|
+
const slash = !!m[4]; const tail = m[5]
|
|
90
|
+
if (tail.length === 3 && !slash) continue
|
|
91
|
+
if ((tail.length === 3 || !slash) && !BIRTH.test(before(text, m.index, ctx, 40))) continue
|
|
92
|
+
if (birthNumberOk(m[1], m[2], m[3], tail)) out.push({ kind: 'birth_number', start: m.index, end: m.index + m[0].length })
|
|
68
93
|
}
|
|
69
94
|
},
|
|
70
95
|
account (text, out) {
|
|
@@ -73,13 +98,13 @@ export function piiKit () {
|
|
|
73
98
|
}
|
|
74
99
|
},
|
|
75
100
|
iban (text, out) {
|
|
76
|
-
for (const m of text.matchAll(/(?<![\p{L}\p{N}])[A-Z]{2}\d{2}[
|
|
101
|
+
for (const m of text.matchAll(/(?<![\p{L}\p{N}])[A-Z]{2}\d{2}[ ]?[A-Z0-9]{4}/g)) {
|
|
77
102
|
// the longest run of groups that is a valid IBAN (the next word in capitals is not part of it)
|
|
78
|
-
const tail = text.slice(m.index, m.index + 50).match(/^[A-Z0-9]+(?:[
|
|
103
|
+
const tail = text.slice(m.index, m.index + 50).match(/^[A-Z0-9]+(?:[ ][A-Z0-9]{1,4})*/)?.[0] || m[0]
|
|
79
104
|
for (let end = tail.length; end >= 15; end--) {
|
|
80
105
|
const cand = tail.slice(0, end)
|
|
81
|
-
if (/[
|
|
82
|
-
if (ibanOk(cand.replace(/[
|
|
106
|
+
if (/[ ]$/.test(cand) || /[A-Z0-9]/.test(tail[end] || '') && !/[ ]/.test(tail[end] || '')) continue
|
|
107
|
+
if (ibanOk(cand.replace(/[ ]/g, ''))) { out.push({ kind: 'iban', start: m.index, end: m.index + end }); break }
|
|
83
108
|
}
|
|
84
109
|
}
|
|
85
110
|
},
|
|
@@ -92,8 +117,8 @@ export function piiKit () {
|
|
|
92
117
|
address (text, out) {
|
|
93
118
|
// Czech: street and number, postcode and town (one line or two); English: number and street
|
|
94
119
|
for (const m of text.matchAll(/[\p{Lu}][\p{L}.'’ -]{1,40}? \d{1,5}(?:\/\d{1,5})?[a-zA-Z]?,?\s+\d{3} ?\d{2}\s+[\p{Lu}][\p{L}-]+(?: (?:[\p{Lu}][\p{L}-]+|\d{1,2}))*/gu)) out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
|
|
95
|
-
for (const m of text.matchAll(/(?<![\p{N}])[1-9]\d{2}[
|
|
96
|
-
if (/^\d{3}[
|
|
120
|
+
for (const m of text.matchAll(/(?<![\p{N}])[1-9]\d{2}[ ]\d{2}[ ]+[\p{Lu}][\p{Ll}]+(?:[ -](?:[\p{Lu}][\p{Ll}]+|\d{1,2}))*(?![\p{L}])/gu)) {
|
|
121
|
+
if (/^\d{3}[ ]\d{2}[ ]+(Kč|Kc|Eur|Ks|Kusů|Kus)/.test(m[0])) continue
|
|
97
122
|
out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
|
|
98
123
|
}
|
|
99
124
|
for (const m of text.matchAll(/(?<![\p{N}])\d{1,5} (?:[\p{Lu}][\p{Ll}]+ ){1,3}(?:Street|St\.?|Avenue|Ave\.?|Road|Rd\.?|Lane|Ln\.?|Boulevard|Blvd\.?|Drive|Dr\.?|Court|Ct\.?|Way|Place|Pl\.?|Square)(?![\p{L}])/gu)) out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
|
|
@@ -106,19 +131,54 @@ export function piiKit () {
|
|
|
106
131
|
}
|
|
107
132
|
}
|
|
108
133
|
}
|
|
109
|
-
/**
|
|
110
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Percent-encoded text decoded, with each character's place in the original ({ text, from, to }), or null. UTF-8
|
|
136
|
+
* sequences decode together; a broken one stays as it is.
|
|
137
|
+
*/
|
|
138
|
+
function decodePct (text) {
|
|
139
|
+
if (!/%[0-9A-Fa-f]{2}/.test(text)) return null
|
|
140
|
+
let out = ''; const from = []; const to = []
|
|
141
|
+
const keep = (s, at) => { for (let i = 0; i < s.length; i++) { out += s[i]; from.push(at + i); to.push(at + i + 1) } }
|
|
142
|
+
let at = 0
|
|
143
|
+
for (const m of text.matchAll(/(?:%[0-9A-Fa-f]{2})+/g)) {
|
|
144
|
+
keep(text.slice(at, m.index), at)
|
|
145
|
+
const bytes = m[0].length / 3
|
|
146
|
+
for (let b = 0; b < bytes;) {
|
|
147
|
+
const first = parseInt(m[0].substr(b * 3 + 1, 2), 16)
|
|
148
|
+
const n = first < 0x80 ? 1 : first >= 0xf0 ? 4 : first >= 0xe0 ? 3 : first >= 0xc0 ? 2 : 1
|
|
149
|
+
let ch = null
|
|
150
|
+
if (n <= bytes - b) { try { ch = decodeURIComponent(m[0].substr(b * 3, n * 3)) } catch {} }
|
|
151
|
+
if (ch == null) { keep(m[0].substr(b * 3, 3), m.index + b * 3); b++; continue }
|
|
152
|
+
const s = m.index + b * 3; const e = s + n * 3
|
|
153
|
+
for (let k = 0; k < ch.length; k++) { out += ch[k]; from.push(s); to.push(e) }
|
|
154
|
+
b += n
|
|
155
|
+
}
|
|
156
|
+
at = m.index + m[0].length
|
|
157
|
+
}
|
|
158
|
+
keep(text.slice(at), at)
|
|
159
|
+
return out === text ? null : { text: out, from, to }
|
|
160
|
+
}
|
|
161
|
+
const scan = (text, kinds, ctx) => {
|
|
111
162
|
const all = []
|
|
112
|
-
if (
|
|
113
|
-
for (const k of kinds) finders[k]?.(
|
|
163
|
+
if (!/[\p{N}@]/u.test(text) && !kinds.includes('name')) return all
|
|
164
|
+
for (const k of kinds) finders[k]?.(text, all, ctx)
|
|
165
|
+
return all
|
|
166
|
+
}
|
|
167
|
+
/** Personal data in a text: [{ kind, start, end, value, plain? }], sorted, the longer of two overlapping matches kept. */
|
|
168
|
+
function find (text, { kinds = KINDS, allow = [], context = '' } = {}) {
|
|
169
|
+
text = text == null ? '' : String(text)
|
|
170
|
+
if (!text) return []
|
|
171
|
+
const all = scan(text, kinds, context)
|
|
172
|
+
const dec = text.length < 500000 ? decodePct(text) : null
|
|
173
|
+
if (dec) for (const r of scan(dec.text, kinds, context)) all.push({ kind: r.kind, start: dec.from[r.start], end: dec.to[r.end - 1], plain: dec.text.slice(r.start, r.end) })
|
|
114
174
|
all.sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start))
|
|
115
175
|
const out = []
|
|
116
176
|
let end = -1
|
|
117
177
|
for (const r of all) {
|
|
118
178
|
if (r.start < end) continue
|
|
119
179
|
const value = text.slice(r.start, r.end)
|
|
120
|
-
if (allowed(value, allow)) continue
|
|
121
|
-
out.push({
|
|
180
|
+
if (allowed(value, allow) || (r.plain && allowed(r.plain, allow))) continue
|
|
181
|
+
out.push({ kind: r.kind, start: r.start, end: r.end, value, ...(r.plain && r.plain !== value ? { plain: r.plain } : {}) }); end = r.end
|
|
122
182
|
}
|
|
123
183
|
return out
|
|
124
184
|
}
|
|
@@ -149,7 +209,7 @@ export function piiKit () {
|
|
|
149
209
|
/** The kind of a whole value (an element the scenario masks, a form field). */
|
|
150
210
|
function kindOf (value) {
|
|
151
211
|
const v = String(value || '').trim()
|
|
152
|
-
const hit = find(v, { kinds: KINDS.filter((k) => k !== 'name') })
|
|
212
|
+
const hit = find(v, { kinds: KINDS.filter((k) => k !== 'name'), context: 'tel rč' })
|
|
153
213
|
if (hit.length === 1 && hit[0].start === 0 && hit[0].end === v.length) return hit[0].kind
|
|
154
214
|
return NAME.test(v) ? 'name' : 'text'
|
|
155
215
|
}
|
|
@@ -168,31 +228,32 @@ export function piiKit () {
|
|
|
168
228
|
if (/(^|\s)(name|given-name|family-name|additional-name)\b/.test(ac) || /^(name|full_?name|first_?name|last_?name|surname|given_?name|family_?name|jmeno|prijmeni)$/.test(key)) return 'name'
|
|
169
229
|
return null
|
|
170
230
|
}
|
|
171
|
-
// fake values: a hash of the
|
|
231
|
+
// fake values: a hash of the salt (scenario and project) and the value picks them, so one value gets one fake in
|
|
232
|
+
// every run of the scenario; the hash keeps a few bits of the value at most, so a fake does not lead back to it
|
|
172
233
|
const hash = (s) => { let x = 0x811c9dc5; for (const ch of String(s)) { x ^= ch.codePointAt(0); x = Math.imul(x, 0x01000193) >>> 0 } return x }
|
|
173
234
|
const rng = (seed) => { let s = seed || 1; return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296 } }
|
|
174
235
|
const pick = (r, list) => list[Math.floor(r() * list.length)]
|
|
175
236
|
const dd = (r, n) => Array.from({ length: n }, () => Math.floor(r() * 10)).join('')
|
|
176
237
|
const ascii = (s) => s.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase()
|
|
177
|
-
const WOMEN = [['Jana', '
|
|
178
|
-
const MEN = [['Petr', '
|
|
238
|
+
const WOMEN = [['Jana', 'Eva', 'Lucie', 'Tereza', 'Klára', 'Anna', 'Petra', 'Hana', 'Marie', 'Lenka'], ['Nováková', 'Svobodová', 'Dvořáková', 'Černá', 'Procházková', 'Kučerová', 'Veselá', 'Horáková', 'Němcová', 'Marková']]
|
|
239
|
+
const MEN = [['Petr', 'Tomáš', 'Martin', 'Jakub', 'David', 'Ondřej', 'Pavel', 'Karel', 'Lukáš', 'Filip'], ['Novák', 'Svoboda', 'Dvořák', 'Černý', 'Procházka', 'Kučera', 'Veselý', 'Horák', 'Němec', 'Marek']]
|
|
179
240
|
const STREETS = ['Lipová', 'Krátká', 'Polní', 'Školní', 'Zahradní', 'Luční']
|
|
180
241
|
const TOWNS = ['Vzorov', 'Ukázkov', 'Testov']
|
|
181
|
-
function pseudonym (kind, value, salt = '') {
|
|
242
|
+
function pseudonym (kind, value, salt = '', attempt = 0) {
|
|
182
243
|
const v = String(value || '').trim()
|
|
183
244
|
const norm = kind === 'email' ? v.toLowerCase() : kind === 'phone' || kind === 'birth_number' || kind === 'account' ? digits(v) : v.replace(/\s+/g, ' ')
|
|
184
|
-
const r = rng(hash(`${salt}|${kind === 'known' ? kindOf(v) : kind}|${norm}`))
|
|
185
|
-
const person = () => { const
|
|
245
|
+
const r = rng(hash(`${salt}|${kind === 'known' ? kindOf(v) : kind}|${norm}${attempt ? `|${attempt}` : ''}`))
|
|
246
|
+
const person = () => { const [first, last] = r() < 0.5 ? WOMEN : MEN; return [pick(r, first), pick(r, last)] }
|
|
186
247
|
const k = kind === 'known' || kind === 'text' ? kindOf(v) : kind
|
|
187
248
|
if (k === 'email') { const [f, l] = person(); return `${ascii(f)}.${ascii(l)}${10 + Math.floor(r() * 90)}@example.com` }
|
|
188
249
|
if (k === 'phone') {
|
|
189
|
-
const cc = v.match(/^(?:\+|00)(\d{1,3})(?=[
|
|
250
|
+
const cc = v.match(/^(?:\+|00)(\d{1,3})(?=[ .(-])/)?.[1] || (/^(?:\+|00)/.test(v) ? '420' : '')
|
|
190
251
|
return `${cc ? `+${cc} ` : ''}000 ${dd(r, 3)} ${dd(r, 3)}`
|
|
191
252
|
}
|
|
192
253
|
// month 00 does not exist: a fake birth number never passes as a real one
|
|
193
254
|
if (k === 'birth_number') return `${dd(r, 2)}00${String(1 + Math.floor(r() * 28)).padStart(2, '0')}/${dd(r, 4)}`
|
|
194
255
|
if (k === 'account') return `${dd(r, 10).replace(/^0/, '1')}/0000`
|
|
195
|
-
if (k === 'iban') { const n = v.replace(/[
|
|
256
|
+
if (k === 'iban') { const n = v.replace(/[ ]/g, '').length; return (v.slice(0, 2) + '00' + dd(r, n - 4)).replace(/(.{4})(?=.)/g, '$1 ') }
|
|
196
257
|
if (k === 'birth_date') {
|
|
197
258
|
const d = String(1 + Math.floor(r() * 28)); const m = String(1 + Math.floor(r() * 12)); const y = String(1950 + Math.floor(r() * 50))
|
|
198
259
|
if (/^\d{4}-/.test(v)) return `${y}-${m.padStart(2, '0')}-${d.padStart(2, '0')}`
|
|
@@ -211,14 +272,63 @@ export function piiKit () {
|
|
|
211
272
|
}
|
|
212
273
|
/** "[hidden]" padded with dots to the length of the value it stands for (shorter values: dots only). */
|
|
213
274
|
const placeholder = (value, label = '[hidden]') => { const n = [...String(value)].length; return n >= label.length ? label + '•'.repeat(n - label.length) : '•'.repeat(Math.max(1, n)) }
|
|
214
|
-
|
|
215
|
-
|
|
275
|
+
/**
|
|
276
|
+
* The stand-in for a value: a placeholder, or a fake that is not in `taken` (fakes and real values already used),
|
|
277
|
+
* so two people never share one fake and a fake never equals a real value.
|
|
278
|
+
*/
|
|
279
|
+
const replacement = (kind, value, ctx, taken = null) => {
|
|
280
|
+
if (ctx.mode !== 'pseudonymize') return placeholder(value, ctx.label)
|
|
281
|
+
let v = pseudonym(kind, value, ctx.salt)
|
|
282
|
+
for (let i = 1; taken && taken.has(v) && i < 40; i++) v = pseudonym(kind, value, ctx.salt, i)
|
|
283
|
+
const base = v
|
|
284
|
+
for (let n = 2; taken && taken.has(v); n++) v = base.includes('@') ? base.replace('@', `-${n}@`) : `${base} ${n}`
|
|
285
|
+
return v
|
|
286
|
+
}
|
|
287
|
+
// --- in the page only ---
|
|
288
|
+
/** This document and its same-origin frames: [{ doc, ox, oy }], ox/oy turn a frame's viewport point into a page point. */
|
|
289
|
+
function pageFrames () {
|
|
290
|
+
const out = []
|
|
291
|
+
const add = (doc, ox, oy, depth) => {
|
|
292
|
+
out.push({ doc, ox, oy })
|
|
293
|
+
if (depth >= 3) return
|
|
294
|
+
for (const f of doc.querySelectorAll('iframe, frame')) {
|
|
295
|
+
let d = null
|
|
296
|
+
try { d = f.contentDocument } catch {}
|
|
297
|
+
if (!d || !d.documentElement) continue
|
|
298
|
+
const r = f.getBoundingClientRect()
|
|
299
|
+
add(d, ox + r.left + f.clientLeft, oy + r.top + f.clientTop, depth + 1)
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
add(document, scrollX, scrollY, 0)
|
|
303
|
+
return out
|
|
304
|
+
}
|
|
305
|
+
/** A document and every open shadow root in it. */
|
|
306
|
+
function rootsOf (doc) {
|
|
307
|
+
const out = [doc]
|
|
308
|
+
for (let i = 0; i < out.length; i++) for (const el of out[i].querySelectorAll('*')) if (el.shadowRoot && el.tagName !== 'KALOKO-MASK') out.push(el.shadowRoot)
|
|
309
|
+
return out
|
|
310
|
+
}
|
|
311
|
+
/** The text nodes of a document or shadow root, in order. */
|
|
312
|
+
function textsOf (root) {
|
|
313
|
+
const start = root.nodeType === 9 ? root.documentElement : root
|
|
314
|
+
if (!start) return []
|
|
315
|
+
const w = (root.ownerDocument || root).createTreeWalker(start, 4)
|
|
316
|
+
const out = []
|
|
317
|
+
for (let n = w.nextNode(); n; n = w.nextNode()) out.push(n)
|
|
318
|
+
return out
|
|
319
|
+
}
|
|
320
|
+
/** The element above, also across a shadow boundary. */
|
|
321
|
+
const up = (node) => node.parentElement || (node.parentNode && node.parentNode.nodeType === 11 ? node.parentNode.host : null)
|
|
322
|
+
return { KINDS, find, findKnown, merge, kindOf, fieldKind, pseudonym, placeholder, replacement, decodePct, pageFrames, rootsOf, textsOf, up }
|
|
216
323
|
}
|
|
217
324
|
|
|
218
325
|
/**
|
|
219
|
-
* Runs in the page (self-contained, `kit` is piiKit()): replaces masked text, form values and attributes
|
|
220
|
-
*
|
|
221
|
-
*
|
|
326
|
+
* Runs in the page (self-contained, `kit` is piiKit()): replaces masked text, form values and the attributes a reader
|
|
327
|
+
* or the outline sees (title, alt, aria-label, placeholder, value) in the live document, in open shadow roots and in
|
|
328
|
+
* same-origin frames, and keeps window.__kalokoMaskUndo to put them back (in place from the first change, so a failure
|
|
329
|
+
* half-way leaves nothing masked). Attributes nobody sees (data-*, content, href, src) are left alone and returned as
|
|
330
|
+
* `attrs` ([original, masked]) for the stored HTML. Returns the masked places in page coordinates ({ x, y, w, h, kind,
|
|
331
|
+
* box }: box = drawn as a box on the screenshot) and the values masked ([original, replacement, kind]).
|
|
222
332
|
*/
|
|
223
333
|
export function maskInPage (args, kit) {
|
|
224
334
|
const { selectors = [], auto = false, mode = 'redact', salt = '', label = '[hidden]', allow = [], known = [] } = args
|
|
@@ -226,167 +336,207 @@ export function maskInPage (args, kit) {
|
|
|
226
336
|
if (window.__kalokoMaskUndo) window.__kalokoMaskUndo()
|
|
227
337
|
window.__kalokoMask?.pause()
|
|
228
338
|
const undo = []
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
if (
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
const rest = (s, e) => { const seg = core.slice(s, e); const t = seg.trim(); if (!/[\p{L}\p{N}]/u.test(t)) return; const off = s + seg.indexOf(t); out.push({ kind: kit.kindOf(t), start: lead + off, end: lead + off + t.length, value: t }) }
|
|
254
|
-
for (const r of found) { rest(at, r.start); out.push({ ...r, start: lead + r.start, end: lead + r.end }); at = r.end }
|
|
255
|
-
rest(at, core.length)
|
|
256
|
-
return out
|
|
257
|
-
}
|
|
258
|
-
const ranges = (text, isMasked, code = false) => isMasked ? whole(text) : kit.merge(auto ? kit.find(text, { allow, kinds: code ? CODE : undefined }) : [], kit.findKnown(text, known))
|
|
259
|
-
const rendered = (el) => el && !el.closest('head, script, style, noscript, template, title') && el.getClientRects().length > 0
|
|
260
|
-
const push = (r, kind, box) => { if (rects.length < 500 && r.width > 0 && r.height > 0) rects.push({ x: Math.round(r.left + scrollX), y: Math.round(r.top + scrollY), w: Math.ceil(r.width), h: Math.ceil(r.height), kind, box }) }
|
|
261
|
-
// whole elements the scenario masks: one box each (drawn in both modes when they hold no text, e.g. a photo)
|
|
262
|
-
for (const el of masked) { if (rendered(el) && !masked.has(el.parentElement)) push(el.getBoundingClientRect(), 'mask', mode === 'redact' || !el.textContent.trim()) }
|
|
263
|
-
// text
|
|
264
|
-
const walker = document.createTreeWalker(document.documentElement, NodeFilter.SHOW_TEXT)
|
|
265
|
-
const nodes = []
|
|
266
|
-
for (let n = walker.nextNode(); n; n = walker.nextNode()) nodes.push(n)
|
|
267
|
-
for (const node of nodes) {
|
|
268
|
-
const parent = node.parentElement
|
|
269
|
-
if (!parent || parent.closest('style, kaloko-mask')) continue
|
|
270
|
-
const text = node.nodeValue
|
|
271
|
-
if (!text || !text.trim()) continue
|
|
272
|
-
const isMasked = inMasked(node)
|
|
273
|
-
const list = ranges(text, isMasked, !!parent.closest('script'))
|
|
274
|
-
if (!list.length) continue
|
|
275
|
-
let out = ''; let at = 0; const spans = []
|
|
276
|
-
for (const r of list) { const v = repl(r); out += text.slice(at, r.start); spans.push([out.length, out.length + v.length, r.kind === 'known' ? 'known' : r.kind]); out += v; at = r.end }
|
|
277
|
-
out += text.slice(at)
|
|
278
|
-
node.nodeValue = out
|
|
279
|
-
undo.push(() => { node.nodeValue = text })
|
|
280
|
-
if (!isMasked && rendered(parent)) {
|
|
281
|
-
for (const [s, e, kind] of spans) { const range = document.createRange(); range.setStart(node, s); range.setEnd(node, e); for (const r of range.getClientRects()) push(r, kind, mode === 'redact') }
|
|
339
|
+
window.__kalokoMaskUndo = () => { for (const f of undo.splice(0).reverse()) { try { f() } catch {} } delete window.__kalokoMaskUndo; window.__kalokoMask?.resume() }
|
|
340
|
+
try {
|
|
341
|
+
const pairs = new Map(known.map(([v, r]) => [v, r]))
|
|
342
|
+
const taken = new Set(known.flatMap(([v, r]) => [v, r]))
|
|
343
|
+
const fresh = new Map()
|
|
344
|
+
const rects = []
|
|
345
|
+
const counts = {}
|
|
346
|
+
const attrs = new Map()
|
|
347
|
+
const roots = kit.pageFrames().flatMap((f) => kit.rootsOf(f.doc).map((root) => ({ root, f })))
|
|
348
|
+
const masked = new Set()
|
|
349
|
+
for (const { root } of roots) for (const sel of selectors) { try { root.querySelectorAll(sel).forEach((el) => masked.add(el)) } catch {} }
|
|
350
|
+
const inMasked = (node) => { for (let el = node.nodeType === 1 ? node : kit.up(node); el; el = kit.up(el)) if (masked.has(el)) return true; return false }
|
|
351
|
+
const CODE = ['email', 'birth_number', 'iban']
|
|
352
|
+
const repl = (r) => {
|
|
353
|
+
if (r.repl != null) { counts.known = (counts.known || 0) + 1; return r.repl }
|
|
354
|
+
const real = r.plain ?? r.value
|
|
355
|
+
let v = pairs.get(real) ?? pairs.get(r.value)
|
|
356
|
+
if (v == null) { taken.add(real); v = kit.replacement(r.kind, real, ctx, taken); taken.add(v) }
|
|
357
|
+
pairs.set(r.value, v); pairs.set(real, v)
|
|
358
|
+
// a part of a masked element that is no kind of its own is plain text (remembered only when it is long)
|
|
359
|
+
const kind = r.guess ? 'text' : r.kind
|
|
360
|
+
fresh.set(r.value, [v, kind]); if (real !== r.value) fresh.set(real, [v, kind])
|
|
361
|
+
counts[r.kind] = (counts[r.kind] || 0) + 1
|
|
362
|
+
return v
|
|
282
363
|
}
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
const
|
|
298
|
-
|
|
299
|
-
if (
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
364
|
+
// every word-ish part of a masked value is masked: found data with its own kind, the rest as a whole
|
|
365
|
+
const whole = (text) => {
|
|
366
|
+
const lead = text.length - text.trimStart().length; const core = text.trim()
|
|
367
|
+
if (!core) return []
|
|
368
|
+
const found = kit.merge(kit.find(core, { allow: [] }), kit.findKnown(core, known))
|
|
369
|
+
const out = []; let at = 0
|
|
370
|
+
const rest = (s, e) => { const seg = core.slice(s, e); const t = seg.trim(); if (!/[\p{L}\p{N}]/u.test(t)) return; const off = s + seg.indexOf(t); out.push({ kind: kit.kindOf(t), guess: true, start: lead + off, end: lead + off + t.length, value: t }) }
|
|
371
|
+
for (const r of found) { rest(at, r.start); out.push({ ...r, start: lead + r.start, end: lead + r.end }); at = r.end }
|
|
372
|
+
rest(at, core.length)
|
|
373
|
+
return out
|
|
374
|
+
}
|
|
375
|
+
const ranges = (text, isMasked, kinds, context = '') => isMasked ? whole(text) : kit.merge(auto ? kit.find(text, { allow, kinds, context }) : [], kit.findKnown(text, known))
|
|
376
|
+
const rendered = (el) => el && !el.closest('head, script, style, noscript, template, title') && el.getClientRects().length > 0
|
|
377
|
+
const push = (r, kind, box, f) => { if (rects.length < 500 && r.width > 0 && r.height > 0) rects.push({ x: Math.round(r.left + f.ox), y: Math.round(r.top + f.oy), w: Math.ceil(r.width), h: Math.ceil(r.height), kind, box }) }
|
|
378
|
+
const spliceIn = (text, list) => { let out = ''; let at = 0; const spans = []; for (const r of list) { const v = repl(r); out += text.slice(at, r.start); spans.push([out.length, out.length + v.length, r.kind === 'known' ? 'known' : r.kind]); out += v; at = r.end } return { out: out + text.slice(at), spans } }
|
|
379
|
+
// whole elements the scenario masks: one box each (drawn in both modes when they hold no text, e.g. a photo)
|
|
380
|
+
for (const { root, f } of roots) for (const el of masked) if (el.getRootNode() === root && rendered(el) && !masked.has(kit.up(el))) push(el.getBoundingClientRect(), 'mask', mode === 'redact' || !el.textContent.trim(), f)
|
|
381
|
+
// text; the text before a node is its context (a "Telefon" label in the cell before the number)
|
|
382
|
+
let prev = ''
|
|
383
|
+
for (const { root, f } of roots) {
|
|
384
|
+
for (const node of kit.textsOf(root)) {
|
|
385
|
+
const parent = kit.up(node)
|
|
386
|
+
if (!parent || parent.closest('style, kaloko-mask')) continue
|
|
387
|
+
const text = node.nodeValue
|
|
388
|
+
if (!text || !text.trim()) continue
|
|
389
|
+
const script = parent.closest('script')
|
|
390
|
+
// inline JSON (JSON-LD, state) gets every kind; script code only the kinds a checksum or an @ confirms
|
|
391
|
+
const kinds = script && !/json/i.test(script.type || '') ? CODE : undefined
|
|
392
|
+
const isMasked = inMasked(node)
|
|
393
|
+
const list = ranges(text, isMasked, kinds, prev)
|
|
394
|
+
if (!script) prev = (prev + ' ' + text).slice(-80)
|
|
395
|
+
if (!list.length) continue
|
|
396
|
+
const { out, spans } = spliceIn(text, list)
|
|
397
|
+
node.nodeValue = out
|
|
398
|
+
undo.push(() => { node.nodeValue = text })
|
|
399
|
+
if (!isMasked && rendered(parent)) {
|
|
400
|
+
for (const [s, e, kind] of spans) { const range = node.ownerDocument.createRange(); range.setStart(node, s); range.setEnd(node, e); for (const r of range.getClientRects()) push(r, kind, mode === 'redact', f) }
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
// form fields as they are filled in (the value property, not only the attribute), hidden ones included
|
|
405
|
+
const SKIP = /^(password|submit|button|checkbox|radio|file|image|reset|range|color)$/
|
|
406
|
+
const handled = new Set()
|
|
407
|
+
for (const { root, f } of roots) {
|
|
408
|
+
for (const el of root.querySelectorAll('input, textarea')) {
|
|
409
|
+
if (SKIP.test(el.type || '')) continue
|
|
410
|
+
const value = el.value
|
|
411
|
+
if (!value || !value.trim()) continue
|
|
412
|
+
const hidden = el.type === 'hidden'
|
|
413
|
+
const isMasked = inMasked(el)
|
|
414
|
+
let kind = !isMasked && auto ? kit.fieldKind(el) : null
|
|
415
|
+
if (hidden && kind === 'name') kind = null // name="name" on a hidden field is rarely a person
|
|
416
|
+
const list = kind ? [{ kind, start: value.length - value.trimStart().length, end: value.trimEnd().length, value: value.trim() }] : ranges(value, isMasked)
|
|
417
|
+
if (!list.length) continue
|
|
418
|
+
const { out } = spliceIn(value, list)
|
|
419
|
+
const attr = el.getAttribute('value')
|
|
420
|
+
el.value = out
|
|
421
|
+
if (attr != null) el.setAttribute('value', out)
|
|
422
|
+
handled.add(el)
|
|
423
|
+
undo.push(() => { el.value = value; if (attr != null) el.setAttribute('value', attr) })
|
|
424
|
+
if (!hidden && !isMasked && rendered(el)) push(el.getBoundingClientRect(), kind || list[0].kind, mode === 'redact', f)
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
// attributes: what a reader or the outline sees is masked in the page; the rest only in the stored HTML (no
|
|
428
|
+
// observer of the page sees a data-* attribute change)
|
|
429
|
+
const LIVE = /^(title|alt|aria-label|aria-description|aria-valuetext|placeholder|value|label)$/i
|
|
430
|
+
const STORED = /^(data-[\w-]+|content|href|src|action|formaction|poster|srcset)$/i
|
|
431
|
+
const LINK = /^(href|src|action|formaction|poster|srcset)$/i
|
|
432
|
+
for (const { root, f } of roots) {
|
|
433
|
+
const top = f.doc === document && root === document
|
|
434
|
+
for (const el of root.querySelectorAll('*')) {
|
|
435
|
+
if (el.tagName === 'KALOKO-MASK') continue
|
|
436
|
+
const isMasked = inMasked(el)
|
|
437
|
+
for (const a of [...el.attributes]) {
|
|
438
|
+
const live = LIVE.test(a.name)
|
|
439
|
+
if ((!live && !(top && STORED.test(a.name))) || !a.value.trim()) continue
|
|
440
|
+
if (a.name === 'value' && handled.has(el)) continue // the fields above
|
|
441
|
+
const link = LINK.test(a.name)
|
|
442
|
+
const scheme = link && /^\s*(mailto|tel|sms):/i.test(a.value)
|
|
443
|
+
const at = a.value.indexOf(':') + 1
|
|
444
|
+
const hits = isMasked && scheme ? [{ kind: 'text', guess: true, start: at, end: a.value.length, value: a.value.slice(at) }] : isMasked && !link ? whole(a.value) : ranges(a.value, false)
|
|
445
|
+
if (!hits.length) continue
|
|
446
|
+
const { out } = spliceIn(a.value, hits)
|
|
447
|
+
if (live) { const name = a.name; const before = a.value; el.setAttribute(name, out); undo.push(() => el.setAttribute(name, before)) } else attrs.set(a.value, out)
|
|
448
|
+
}
|
|
449
|
+
}
|
|
321
450
|
}
|
|
451
|
+
return { rects, pairs: [...fresh].map(([v, [r, k]]) => [v, r, k]), counts, attrs: [...attrs], changed: undo.length + attrs.size }
|
|
452
|
+
} catch (error) {
|
|
453
|
+
window.__kalokoMaskUndo?.()
|
|
454
|
+
throw error
|
|
322
455
|
}
|
|
323
|
-
window.__kalokoMaskUndo = () => { for (const f of undo.reverse()) { try { f() } catch {} } delete window.__kalokoMaskUndo; window.__kalokoMask?.resume() }
|
|
324
|
-
return { rects, pairs: [...fresh].filter(([v]) => v.length >= 3), counts }
|
|
325
456
|
}
|
|
326
457
|
|
|
327
458
|
/**
|
|
328
459
|
* Runs in the page (self-contained, `kit` is piiKit()): an overlay that keeps every masked place covered while the page
|
|
329
|
-
* changes, for recordings
|
|
460
|
+
* changes, for recordings. It reads open shadow roots and same-origin frames, and paints inside an open modal dialog
|
|
461
|
+
* or popover too (the top layer is above anything else in the page). window.__kalokoMask: update(args), pause(),
|
|
462
|
+
* resume().
|
|
330
463
|
*/
|
|
331
464
|
export function maskPainter (args, kit) {
|
|
332
465
|
if (window.__kalokoMask) { window.__kalokoMask.update(args); return }
|
|
333
466
|
let cfg = { selectors: [], known: [], ...args }
|
|
334
|
-
let paused = false; let pending = false
|
|
335
|
-
const
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
host.
|
|
339
|
-
|
|
340
|
-
|
|
467
|
+
let paused = false; let pending = false
|
|
468
|
+
const hosts = new Map()
|
|
469
|
+
const hostIn = (container, fixed) => {
|
|
470
|
+
const had = hosts.get(container)
|
|
471
|
+
if (had?.host.isConnected) return had
|
|
472
|
+
const host = document.createElement('kaloko-mask')
|
|
473
|
+
host.style.cssText = `position:${fixed ? 'fixed' : 'absolute'};left:0;top:0;width:0;height:0;overflow:visible;z-index:2147483647;pointer-events:none;display:block;margin:0;padding:0;border:0`
|
|
474
|
+
const root = host.attachShadow({ mode: 'closed' })
|
|
475
|
+
container.appendChild(host)
|
|
476
|
+
const h = { host, root, fixed }
|
|
477
|
+
hosts.set(container, h)
|
|
478
|
+
return h
|
|
341
479
|
}
|
|
480
|
+
const topLayer = () => { for (const q of ['dialog:modal, :popover-open', 'dialog[open]']) { try { return [...document.querySelectorAll(q)] } catch {} } return [] }
|
|
342
481
|
const collect = () => {
|
|
343
482
|
const out = []
|
|
344
|
-
const add = (r) => { if (out.length < 500 && r.width > 0 && r.height > 0) out.push([r.left +
|
|
345
|
-
const
|
|
346
|
-
|
|
347
|
-
for (const
|
|
348
|
-
const inMasked = (el) =>
|
|
349
|
-
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
483
|
+
const add = (r, f) => { if (out.length < 500 && r.width > 0 && r.height > 0) out.push([r.left + f.ox, r.top + f.oy, r.width, r.height]) }
|
|
484
|
+
const roots = kit.pageFrames().flatMap((f) => kit.rootsOf(f.doc).map((root) => ({ root, f })))
|
|
485
|
+
const masked = new Set()
|
|
486
|
+
for (const { root, f } of roots) for (const sel of cfg.selectors) { try { root.querySelectorAll(sel).forEach((el) => { masked.add(el); add(el.getBoundingClientRect(), f) }) } catch {} }
|
|
487
|
+
const inMasked = (el) => { for (let e = el; e; e = kit.up(e)) if (masked.has(e)) return true; return false }
|
|
488
|
+
let prev = ''
|
|
489
|
+
for (const { root, f } of roots) {
|
|
490
|
+
for (const n of kit.textsOf(root)) {
|
|
491
|
+
const p = kit.up(n)
|
|
492
|
+
if (!p || p.closest('head, title, script, style, noscript, template, kaloko-mask') || !n.nodeValue.trim()) continue
|
|
493
|
+
const list = kit.merge(cfg.auto ? kit.find(n.nodeValue, { allow: cfg.allow, context: prev }) : [], kit.findKnown(n.nodeValue, cfg.known))
|
|
494
|
+
prev = (prev + ' ' + n.nodeValue).slice(-80)
|
|
495
|
+
if (!list.length || inMasked(p)) continue
|
|
496
|
+
for (const r of list) { const range = n.ownerDocument.createRange(); range.setStart(n, r.start); range.setEnd(n, r.end); for (const c of range.getClientRects()) add(c, f) }
|
|
497
|
+
}
|
|
498
|
+
for (const el of root.querySelectorAll('input, textarea')) {
|
|
499
|
+
if (/^(password|hidden|submit|button|checkbox|radio|file|image|reset|range|color)$/.test(el.type || '') || !el.value.trim() || inMasked(el)) continue
|
|
500
|
+
if ((cfg.auto && (kit.fieldKind(el) || kit.find(el.value, { allow: cfg.allow }).length)) || kit.findKnown(el.value, cfg.known).length) add(el.getBoundingClientRect(), f)
|
|
501
|
+
}
|
|
361
502
|
}
|
|
362
503
|
return out
|
|
363
504
|
}
|
|
364
505
|
const paint = () => {
|
|
365
506
|
pending = false
|
|
366
|
-
if (paused) return
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
507
|
+
if (paused || !document.documentElement) return
|
|
508
|
+
const boxes = collect()
|
|
509
|
+
const draw = (h) => {
|
|
510
|
+
// a host inside the top layer is fixed: viewport coordinates, from wherever it actually sits
|
|
511
|
+
const o = h.fixed ? h.host.getBoundingClientRect() : null
|
|
512
|
+
const dx = o ? -scrollX - o.left : 0; const dy = o ? -scrollY - o.top : 0
|
|
513
|
+
h.root.replaceChildren(...boxes.map(([x, y, w, hh]) => { const d = document.createElement('div'); d.style.cssText = `position:absolute;left:${x + dx}px;top:${y + dy}px;width:${w}px;height:${hh}px;background:repeating-linear-gradient(135deg,#1f1f1d 0 6px,#34342f 6px 12px)`; return d }))
|
|
514
|
+
}
|
|
515
|
+
draw(hostIn(document.documentElement, false))
|
|
516
|
+
const tops = new Set(topLayer())
|
|
517
|
+
for (const [c, h] of hosts) if (c !== document.documentElement && !tops.has(c)) { h.host.remove(); hosts.delete(c) }
|
|
518
|
+
for (const el of tops) draw(hostIn(el, true))
|
|
371
519
|
}
|
|
372
520
|
const schedule = () => { if (!pending && !paused) { pending = true; requestAnimationFrame(paint) } }
|
|
373
521
|
const start = () => {
|
|
374
|
-
new MutationObserver(schedule).observe(document.documentElement, { subtree: true, childList: true, characterData: true })
|
|
375
|
-
for (const ev of ['scroll', 'input', 'resize', 'load']) addEventListener(ev, schedule, true)
|
|
522
|
+
new MutationObserver(schedule).observe(document.documentElement, { subtree: true, childList: true, characterData: true, attributes: true, attributeFilter: ['open', 'popover', 'value', 'style', 'class', 'hidden'] })
|
|
523
|
+
for (const ev of ['scroll', 'input', 'resize', 'load', 'toggle']) addEventListener(ev, schedule, true)
|
|
376
524
|
setInterval(schedule, 400)
|
|
377
525
|
schedule()
|
|
378
526
|
}
|
|
379
527
|
window.__kalokoMask = {
|
|
380
528
|
update (a) { cfg = { ...cfg, ...a, selectors: [...new Set([...cfg.selectors, ...(a.selectors || [])])], known: [...new Map([...cfg.known, ...(a.known || [])]).entries()] }; schedule() },
|
|
381
|
-
pause () { paused = true; host
|
|
529
|
+
pause () { paused = true; for (const h of hosts.values()) h.host.remove(); hosts.clear() },
|
|
382
530
|
resume () { paused = false; schedule() }
|
|
383
531
|
}
|
|
384
|
-
|
|
532
|
+
// an init script runs before the document has an element: start with the first one, not at DOMContentLoaded
|
|
533
|
+
if (document.documentElement) start()
|
|
534
|
+
else new MutationObserver((_, o) => { if (document.documentElement) { o.disconnect(); start() } }).observe(document, { childList: true })
|
|
385
535
|
}
|
|
386
536
|
|
|
387
537
|
const kit = piiKit()
|
|
388
538
|
export const findPii = (text, opts) => kit.find(text, opts)
|
|
389
|
-
export const pseudonym = (kind, value, salt) => kit.pseudonym(kind, value, salt)
|
|
539
|
+
export const pseudonym = (kind, value, salt, attempt) => kit.pseudonym(kind, value, salt, attempt)
|
|
390
540
|
export const placeholder = (value, label) => kit.placeholder(value, label)
|
|
391
541
|
|
|
392
542
|
/** The text of the masked placeholder in a locale. */
|
|
@@ -414,6 +564,11 @@ export function piiSettings ({ config = {}, env = {}, scenario = {}, args = {} }
|
|
|
414
564
|
return { mode, allow: [...(config.pii?.allow || [])].map(String) }
|
|
415
565
|
}
|
|
416
566
|
|
|
567
|
+
/** Whether a run asked for hiding personal data (its own setting, the environment, the scenario or a mask: auto). */
|
|
568
|
+
export function piiRequested ({ run = null, env = {}, scenario = {} }) {
|
|
569
|
+
return !!(run?.meta?.pii || ['required', ...PII_MODES].includes(env?.pii) || PII_MODES.includes(scenario?.pii) || scenarioMasks(scenario).auto)
|
|
570
|
+
}
|
|
571
|
+
|
|
417
572
|
/** capture.mask as given (a list of selectors, `auto`, or a list with `auto` in it): { selectors, auto }. */
|
|
418
573
|
export function maskList (mask) {
|
|
419
574
|
const list = mask == null ? [] : Array.isArray(mask) ? mask : [mask]
|
|
@@ -421,9 +576,19 @@ export function maskList (mask) {
|
|
|
421
576
|
return { selectors: items.filter((s) => s !== 'auto'), auto: items.includes('auto') }
|
|
422
577
|
}
|
|
423
578
|
|
|
579
|
+
/**
|
|
580
|
+
* The salt of fake values: the scenario and the project, so one value gets the same fake in every run of the scenario
|
|
581
|
+
* (pixel diffs and approvals carry over). KALOKO_PII_SALT adds a secret of your own.
|
|
582
|
+
*/
|
|
583
|
+
export function piiSalt (meta = {}) {
|
|
584
|
+
const p = meta?.project
|
|
585
|
+
const project = typeof p === 'string' ? p : p?.id || p?.slug || p?.name || ''
|
|
586
|
+
return [process.env.KALOKO_PII_SALT || '', project, meta?.scenario?.id || meta?.id || ''].join('|')
|
|
587
|
+
}
|
|
588
|
+
|
|
424
589
|
/**
|
|
425
590
|
* What one capture masks: the selectors of the scenario and the step, automatic detection (the run's pii setting or
|
|
426
|
-
* `mask: auto`), the mode and the
|
|
591
|
+
* `mask: auto`), the mode and the salt for fake values. `active` is false when there is nothing to mask.
|
|
427
592
|
*/
|
|
428
593
|
export function maskSpec ({ run = null, selectors = [], auto = false, locale = null }) {
|
|
429
594
|
const pii = run?.meta?.pii || null
|
|
@@ -431,7 +596,7 @@ export function maskSpec ({ run = null, selectors = [], auto = false, locale = n
|
|
|
431
596
|
selectors: [...new Set(selectors)],
|
|
432
597
|
auto: !!(auto || pii),
|
|
433
598
|
mode: pii?.mode || 'redact',
|
|
434
|
-
salt: run?.meta
|
|
599
|
+
salt: piiSalt(run?.meta),
|
|
435
600
|
label: maskLabel(locale),
|
|
436
601
|
allow: pii?.allow || []
|
|
437
602
|
}
|
|
@@ -445,23 +610,55 @@ export function scenarioMasks (scenario) {
|
|
|
445
610
|
return { selectors: [...new Set(all.flatMap((m) => m.selectors))], auto: all.some((m) => m.auto) }
|
|
446
611
|
}
|
|
447
612
|
|
|
448
|
-
// --- the values masked in this
|
|
613
|
+
// --- the values masked in this run ---------------------------------------------------------------------------------
|
|
449
614
|
|
|
450
615
|
const remembered = new Map()
|
|
616
|
+
const kindsOf = new Map()
|
|
617
|
+
const loaded = new Set()
|
|
451
618
|
let pattern = null
|
|
452
619
|
|
|
453
|
-
|
|
620
|
+
const digitsOf = (s) => String(s).replace(/\D/g, '')
|
|
621
|
+
const NUMERIC = /^[\d\s.,:/+%-]+$/
|
|
622
|
+
/**
|
|
623
|
+
* Whether a masked value is replaced everywhere else: values of a detected kind, other text from 6 characters on,
|
|
624
|
+
* numbers only from 6 digits on (2024, 100 or Praha are not worth replacing across pages and markup).
|
|
625
|
+
*/
|
|
626
|
+
function worthRemembering (v, kind) {
|
|
627
|
+
if (v.length < 3) return false
|
|
628
|
+
if (NUMERIC.test(v) && digitsOf(v).length < 6) return false
|
|
629
|
+
if (!kind || kind === 'text' || kind === 'known' || kind === 'mask') return v.length >= 6
|
|
630
|
+
return true
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
/** Remembers masked values ([original, replacement, kind?]) so every later output of the run replaces them too. */
|
|
454
634
|
export function rememberPii (pairs) {
|
|
455
|
-
for (const [value, repl] of pairs || []) {
|
|
635
|
+
for (const [value, repl, kind] of pairs || []) {
|
|
456
636
|
const v = String(value || '').trim()
|
|
457
|
-
if (v
|
|
637
|
+
if (worthRemembering(v, kind) && !remembered.has(v)) { remembered.set(v, String(repl)); kindsOf.set(v, kind || 'text'); pattern = null }
|
|
458
638
|
}
|
|
459
639
|
}
|
|
460
640
|
|
|
461
641
|
/** The values remembered so far, as [[original, replacement]] (for the page and the overlay). */
|
|
462
642
|
export const knownPii = () => [...remembered]
|
|
463
643
|
|
|
464
|
-
export const _forgetPii = () => { remembered.clear(); pattern = null } // tests
|
|
644
|
+
export const _forgetPii = () => { remembered.clear(); kindsOf.clear(); loaded.clear(); pattern = null } // tests
|
|
645
|
+
|
|
646
|
+
// The values travel to the next `kaloko` process of the run in the run folder. The file stays on this computer:
|
|
647
|
+
// share uploads only the files the manifest names.
|
|
648
|
+
export const KNOWN_FILE = '.pii-known.json'
|
|
649
|
+
|
|
650
|
+
/** Reads the values an earlier process of this run masked (once per run and process). */
|
|
651
|
+
export async function loadKnownPii (dir) {
|
|
652
|
+
if (!dir || loaded.has(dir)) return
|
|
653
|
+
loaded.add(dir)
|
|
654
|
+
try { rememberPii(JSON.parse(await readFile(join(dir, KNOWN_FILE), 'utf8'))) } catch {}
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
/** Keeps the values masked so far for the next process of this run. */
|
|
658
|
+
export async function saveKnownPii (dir) {
|
|
659
|
+
if (!dir || !remembered.size) return
|
|
660
|
+
await writeFile(join(dir, KNOWN_FILE), JSON.stringify([...remembered].map(([v, r]) => [v, r, kindsOf.get(v)])), { mode: 0o600 }).catch(() => {})
|
|
661
|
+
}
|
|
465
662
|
|
|
466
663
|
/** Replaces every remembered value (and its URL-encoded form) where it stands as a whole word. */
|
|
467
664
|
export function redactPii (text) {
|
|
@@ -472,39 +669,68 @@ export function redactPii (text) {
|
|
|
472
669
|
const keys = [...forms.keys()].sort((a, b) => b.length - a.length).map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
|
|
473
670
|
pattern = { re: new RegExp(`(?<![\\p{L}\\p{N}])(?:${keys.join('|')})(?![\\p{L}\\p{N}])`, 'gu'), forms }
|
|
474
671
|
}
|
|
475
|
-
return text.replace(pattern.re, (m
|
|
672
|
+
return text.replace(pattern.re, (m, at, all) => {
|
|
673
|
+
// a number in markup stays: a colour (#123456), a percentage
|
|
674
|
+
if (NUMERIC.test(m) && (all[at - 1] === '#' || all[at + m.length] === '%')) return m
|
|
675
|
+
return pattern.forms.get(m) ?? m
|
|
676
|
+
})
|
|
476
677
|
}
|
|
477
678
|
|
|
679
|
+
const takenNow = () => new Set([...remembered.keys(), ...remembered.values()])
|
|
680
|
+
|
|
478
681
|
/** A text from outside the page (trace, console, mail headers): remembered values, then detection when it is on. */
|
|
479
682
|
export function scrubText (text, spec) {
|
|
480
683
|
let out = redactPii(text)
|
|
481
684
|
if (typeof out !== 'string' || !spec?.auto) return out
|
|
482
685
|
const hits = kit.find(out, { allow: spec.allow })
|
|
686
|
+
const taken = takenNow()
|
|
483
687
|
for (const r of hits.reverse()) {
|
|
484
|
-
const
|
|
485
|
-
|
|
688
|
+
const real = r.plain ?? r.value
|
|
689
|
+
let repl = remembered.get(real)
|
|
690
|
+
if (repl == null) { taken.add(real); repl = kit.replacement(r.kind, real, spec, taken); taken.add(repl) }
|
|
691
|
+
rememberPii([[real, repl, r.kind], ...(r.plain ? [[r.value, repl, r.kind]] : [])])
|
|
486
692
|
out = out.slice(0, r.start) + repl + out.slice(r.end)
|
|
487
693
|
}
|
|
488
694
|
return out
|
|
489
695
|
}
|
|
490
696
|
|
|
491
|
-
/** Deep copy with every string scrubbed (scrubText). */
|
|
492
|
-
export function scrubDeep (value, spec) {
|
|
697
|
+
/** Deep copy with every string scrubbed (scrubText); keys in `skip` are copied as they are. */
|
|
698
|
+
export function scrubDeep (value, spec, skip = null) {
|
|
493
699
|
if (typeof value === 'string') return scrubText(value, spec)
|
|
494
|
-
if (Array.isArray(value)) return value.map((v) => scrubDeep(v, spec))
|
|
495
|
-
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, scrubDeep(v, spec)]))
|
|
700
|
+
if (Array.isArray(value)) return value.map((v) => scrubDeep(v, spec, skip))
|
|
701
|
+
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, skip?.has(k) ? v : scrubDeep(v, spec, skip)]))
|
|
496
702
|
return value
|
|
497
703
|
}
|
|
498
704
|
|
|
705
|
+
/** Keys of a capture record that hold no page text (paths, hashes, coordinates). */
|
|
706
|
+
export const RECORD_PLAIN_KEYS = new Set(['schema', 'files', 'criteria_hash', 'captured_at', 'boxes', 'rects', 'ignore', 'unstable', 'capture', 'pii', 'node', 'locale', 'viewport', 'view'])
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* The stored HTML of a masked page: attributes masked in the page only as values (maskInPage's `attrs`: data-*,
|
|
710
|
+
* content, links) replaced, as the browser serialises them.
|
|
711
|
+
*/
|
|
712
|
+
export function maskHtml (html, attrs = []) {
|
|
713
|
+
if (typeof html !== 'string' || !attrs?.length) return html
|
|
714
|
+
const esc = (s, lt) => { let v = s.replace(/&/g, '&').replace(/"/g, '"').replace(/ /g, ' '); if (lt) v = v.replace(/</g, '<').replace(/>/g, '>'); return v }
|
|
715
|
+
let out = html
|
|
716
|
+
for (const [from, to] of [...attrs].sort((a, b) => b[0].length - a[0].length)) {
|
|
717
|
+
for (const lt of [false, true]) out = out.split(`="${esc(from, lt)}"`).join(`="${esc(to, lt)}"`)
|
|
718
|
+
}
|
|
719
|
+
return out
|
|
720
|
+
}
|
|
721
|
+
|
|
499
722
|
/** The e-mail headers of a captured message with recipients replaced (To and Cc are always personal data). */
|
|
500
723
|
export function maskMailHeaders (mail, spec) {
|
|
501
724
|
if (!mail || !spec?.active) return mail
|
|
725
|
+
const taken = takenNow()
|
|
726
|
+
const stand = (kind, v) => { const had = remembered.get(v); if (had != null) return had; taken.add(v); const r = kit.replacement(kind, v, spec, taken); taken.add(r); return r }
|
|
502
727
|
const one = (addr) => {
|
|
503
728
|
const a = String(addr || '').trim()
|
|
504
729
|
if (!a) return a
|
|
505
730
|
const email = a.match(/<([^>]+)>/)?.[1] || a
|
|
506
|
-
const repl =
|
|
507
|
-
|
|
731
|
+
const repl = stand('email', email)
|
|
732
|
+
const name = a !== email ? a.replace(/<[^>]+>/, '').trim().replace(/^"|"$/g, '') : ''
|
|
733
|
+
rememberPii([[email, repl, 'email'], ...(name ? [[name, stand('name', name), 'name']] : [])])
|
|
508
734
|
return repl
|
|
509
735
|
}
|
|
510
736
|
const list = (v) => String(v || '').split(/,(?=(?:[^"]*"[^"]*")*[^"]*$)/).map(one).filter(Boolean).join(', ')
|
|
@@ -514,17 +740,29 @@ export function maskMailHeaders (mail, spec) {
|
|
|
514
740
|
/** Page argument for maskInPage / maskPainter: the kit travels as source with the call. */
|
|
515
741
|
export const inPage = (fn, args) => `(${fn})(${JSON.stringify(args)}, (${piiKit})())`
|
|
516
742
|
|
|
517
|
-
/**
|
|
743
|
+
/**
|
|
744
|
+
* Installs (or extends) the overlay on a page and on its next documents; resolves false where it cannot. An init
|
|
745
|
+
* script cannot be changed, so a new one is added whenever the values to cover have grown: a later document starts
|
|
746
|
+
* with everything masked so far.
|
|
747
|
+
*/
|
|
518
748
|
export async function paintMasks (page, spec) {
|
|
519
749
|
if (!spec?.active) return false
|
|
520
750
|
const args = { selectors: spec.selectors, auto: spec.auto, allow: spec.allow, known: knownPii() }
|
|
521
751
|
try {
|
|
522
|
-
|
|
523
|
-
|
|
752
|
+
const script = inPage(maskPainter, args)
|
|
753
|
+
SPECS.set(page, spec)
|
|
754
|
+
if (SCRIPTS.get(page) !== script) { await page.addInitScript(script); SCRIPTS.set(page, script) }
|
|
755
|
+
await page.evaluate(script)
|
|
524
756
|
return true
|
|
525
757
|
} catch { return false }
|
|
526
758
|
}
|
|
527
|
-
const
|
|
759
|
+
const SCRIPTS = new WeakMap()
|
|
760
|
+
const SPECS = new WeakMap()
|
|
761
|
+
|
|
762
|
+
/** After a capture: an overlay drawn on the page learns the values masked since (nothing when none is drawn). */
|
|
763
|
+
export async function updateMasks (page) {
|
|
764
|
+
if (SPECS.has(page)) await paintMasks(page, SPECS.get(page))
|
|
765
|
+
}
|
|
528
766
|
|
|
529
767
|
// --- the guard before upload ---------------------------------------------------------------------------------------
|
|
530
768
|
|
|
@@ -551,43 +789,122 @@ function elementAt (html, at) {
|
|
|
551
789
|
return `<${m[1].toLowerCase()}${id ? `#${id}` : ''}${cls ? `.${cls}` : ''}>`
|
|
552
790
|
}
|
|
553
791
|
|
|
554
|
-
|
|
792
|
+
// sample addresses of forms and docs (you@company.com): never anyone's
|
|
793
|
+
const SAMPLE = /^(you|your[._-]?(name|email|mail)?|name|user(name)?|e-?mail|jmeno|jmeno\.prijmeni|vas[._-]?email|vase[._-]?jmeno|john[._-]?doe|jane[._-]?doe)@|@(your)?(company|domain)\.(com|cz)$|@(vase|moje)firma\.cz$/i
|
|
794
|
+
// text in a placeholder (an attribute in evidence, a key in JSON) is a hint, not data
|
|
795
|
+
const HINT_BEFORE = /placeholder\s*[=:]\s*["'“\\]*[^"'\n]{0,80}$/i
|
|
796
|
+
// HTML values that never hold personal data: SVG geometry, styles, placeholders, responsive image lists
|
|
797
|
+
const GEOMETRY = /\s(d|points|viewBox|transform|x|y|x1|x2|y1|y2|cx|cy|r|rx|ry|dx|dy|width|height|offset|stroke-dasharray|stroke-dashoffset|stroke-width|style|placeholder|srcset|sizes|integrity|nonce)\s*=\s*("[^"]*"|'[^']*')/gi
|
|
798
|
+
/** HTML for the guard: values that are never personal data blanked (offsets stay, so elements are still named). */
|
|
799
|
+
function blankHtml (html) {
|
|
800
|
+
return html
|
|
801
|
+
.replace(GEOMETRY, (m, name, val) => m.slice(0, m.length - val.length) + val[0] + ' '.repeat(val.length - 2) + val.at(-1))
|
|
802
|
+
.replace(/(<style\b[^>]*>)([\s\S]*?)(<\/style>)/gi, (m, a, b, c) => a + ' '.repeat(b.length) + c)
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
/** Personal data in one text: [{ kind, value, plain?, where }]. */
|
|
555
806
|
export function scanText (text, { allow = [], where = null } = {}) {
|
|
556
|
-
|
|
807
|
+
const s = String(text || '')
|
|
808
|
+
return kit.find(s, { kinds: GUARD_KINDS, allow })
|
|
809
|
+
.filter((r) => !HINT_BEFORE.test(s.slice(Math.max(0, r.start - 100), r.start)))
|
|
810
|
+
.map((r) => ({ kind: r.kind, value: r.value, ...(r.plain ? { plain: r.plain } : {}), where: typeof where === 'function' ? where(r.start) : where }))
|
|
557
811
|
}
|
|
558
812
|
|
|
813
|
+
const PLAIN_KEYS = new Set(['files', 'criteria_hash', 'boxes', 'rects', 'ignore', 'unstable'])
|
|
559
814
|
/** Personal data in a JSON value: hits with the field path. */
|
|
560
815
|
export function scanJson (value, { allow = [], path = '' } = {}) {
|
|
561
816
|
if (typeof value === 'string') return scanText(value, { allow, where: path || 'value' })
|
|
562
817
|
if (Array.isArray(value)) return value.flatMap((v, i) => scanJson(v, { allow, path: `${path}[${i}]` }))
|
|
563
|
-
if (value && typeof value === 'object') return Object.entries(value).flatMap(([k, v]) => (k
|
|
818
|
+
if (value && typeof value === 'object') return Object.entries(value).flatMap(([k, v]) => (PLAIN_KEYS.has(k) ? [] : scanJson(v, { allow, path: path ? `${path}.${k}` : k })))
|
|
564
819
|
return []
|
|
565
820
|
}
|
|
566
821
|
|
|
822
|
+
/** A contact as compared: an e-mail in lower case, a phone number as its digits without 00. */
|
|
823
|
+
const contactKey = (v) => { const s = String(v || '').trim().replace(/^(mailto|tel):/i, ''); return s.includes('@') ? s.toLowerCase() : digitsOf(s).replace(/^00/, '') }
|
|
824
|
+
const ORG = /Organi[sz]ation|Corporation|Business|Store|ContactPoint|WebSite|WebPage|Brand|Place|Restaurant|Hotel|NGO|School|College|University|Library|Museum|Shop/i
|
|
825
|
+
const PEOPLE = /^(employees?|founders?|members?|authors?|customers?|persons?|people|contactPersons?|creators?|editors?|reviews?)$/i
|
|
826
|
+
/** The e-mails and phone numbers an organisation publishes about itself in JSON-LD (Organization, ContactPoint). */
|
|
827
|
+
function ldContacts (value, out, org = false) {
|
|
828
|
+
if (Array.isArray(value)) { for (const v of value) ldContacts(v, out, org); return }
|
|
829
|
+
if (!value || typeof value !== 'object') return
|
|
830
|
+
const types = [].concat(value['@type'] || []).map(String)
|
|
831
|
+
const isOrg = types.length ? types.some((t) => ORG.test(t)) && !types.some((t) => /Person|Patient/i.test(t)) : org
|
|
832
|
+
if (isOrg) for (const k of ['email', 'telephone', 'faxNumber']) for (const x of [].concat(value[k] || [])) if (typeof x === 'string') out.add(contactKey(x))
|
|
833
|
+
for (const [k, v] of Object.entries(value)) if (v && typeof v === 'object') ldContacts(v, out, isOrg && !PEOPLE.test(k))
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/** [start, end] ranges of the page footer (<footer>, role=contentinfo) in an HTML text. */
|
|
837
|
+
function footerRanges (html) {
|
|
838
|
+
const out = []
|
|
839
|
+
for (const m of html.matchAll(/<(footer)\b[^>]*>|<([a-z][\w-]*)\b[^>]*\brole=["']?contentinfo\b[^>]*>/gi)) {
|
|
840
|
+
const tag = (m[1] || m[2]).toLowerCase()
|
|
841
|
+
const re = new RegExp(`<${tag}\\b|</${tag}\\s*>`, 'gi')
|
|
842
|
+
re.lastIndex = m.index + m[0].length
|
|
843
|
+
let depth = 1; let end = html.length
|
|
844
|
+
for (let t = re.exec(html); t; t = re.exec(html)) { if (t[0][1] === '/') { if (!--depth) { end = t.index; break } } else depth++ }
|
|
845
|
+
out.push([m.index, end])
|
|
846
|
+
}
|
|
847
|
+
return out
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/** The registrable part of a host name (shop.acme.cz → acme.cz, www.acme.co.uk → acme.co.uk). */
|
|
851
|
+
function baseDomain (host) {
|
|
852
|
+
const parts = String(host || '').toLowerCase().replace(/\.$/, '').split('.')
|
|
853
|
+
const n = parts.length >= 3 && /^(co|com|org|net|gov|ac|edu)$/.test(parts.at(-2)) ? 3 : 2
|
|
854
|
+
return parts.slice(-n).join('.')
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
const TEXT_FILES = /\.(json|html?|xml|svg|txt|eml|csv|md|log)$/i
|
|
858
|
+
|
|
567
859
|
/**
|
|
568
860
|
* What `kaloko share` would upload, scanned for unmasked e-mails, phone numbers, birth numbers and IBANs:
|
|
569
|
-
* the manifest's steps (captures, mail headers, console lines, evidence) and the files that travel with it:
|
|
861
|
+
* the manifest's steps (captures, mail headers, console lines, evidence) and the text files that travel with it:
|
|
570
862
|
* [{ kind, value, node, locale, viewport, file, where }]. Images and recordings are not read (masks cover them).
|
|
863
|
+
* The site's own contacts are not personal data (site.contacts, on unless pii.site_contacts: false): e-mails on the
|
|
864
|
+
* domain of the environment, e-mails and phone numbers in the page footer, and those of an Organization or
|
|
865
|
+
* ContactPoint in JSON-LD. Sample addresses (you@company.com) and placeholder text never count.
|
|
571
866
|
*/
|
|
572
|
-
export async function piiFindings ({ dir, files, allow = [], manifest = null }) {
|
|
573
|
-
const
|
|
867
|
+
export async function piiFindings ({ dir, files, allow = [], manifest = null, site = null }) {
|
|
868
|
+
const hits = []
|
|
869
|
+
const contacts = new Set()
|
|
870
|
+
const siteContacts = site?.contacts !== false
|
|
871
|
+
let own = null
|
|
872
|
+
try { own = site?.url ? baseDomain(new URL(site.url).hostname) : null } catch {}
|
|
574
873
|
for (const n of manifest?.nodes || []) {
|
|
575
874
|
const { captures, ...rest } = n
|
|
576
|
-
for (const h of scanJson(rest, { allow }))
|
|
875
|
+
for (const h of scanJson(rest, { allow })) hits.push({ ...h, node: n.id, file: 'run.json' })
|
|
577
876
|
for (const [locale, byVp] of Object.entries(captures || {})) {
|
|
578
|
-
for (const [viewport, c] of Object.entries(byVp || {})) for (const h of scanJson(c, { allow }))
|
|
877
|
+
for (const [viewport, c] of Object.entries(byVp || {})) for (const h of scanJson(c, { allow })) hits.push({ ...h, node: n.id, locale, viewport, file: 'run.json' })
|
|
579
878
|
}
|
|
580
879
|
}
|
|
880
|
+
if (manifest) ldContacts(manifest, contacts)
|
|
581
881
|
for (const file of files) {
|
|
582
|
-
if (
|
|
882
|
+
if (!TEXT_FILES.test(file)) continue
|
|
583
883
|
let text
|
|
584
884
|
try { text = await readFile(join(dir, file), 'utf8') } catch { continue }
|
|
585
885
|
const at = stepOfPath(file)
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
886
|
+
if (/\.json$/i.test(file)) {
|
|
887
|
+
let data
|
|
888
|
+
try { data = JSON.parse(text) } catch { data = text }
|
|
889
|
+
ldContacts(data, contacts)
|
|
890
|
+
for (const h of scanJson(data, { allow })) hits.push({ ...h, ...at, file })
|
|
891
|
+
} else if (/\.(html?|xml|svg)$/i.test(file)) {
|
|
892
|
+
for (const m of text.matchAll(/<script\b[^>]*type=["']?application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) { try { ldContacts(JSON.parse(m[1]), contacts) } catch {} }
|
|
893
|
+
const footers = footerRanges(text)
|
|
894
|
+
for (const h of scanText(blankHtml(text), { allow, where: (i) => ({ i, el: elementAt(text, i) }) })) {
|
|
895
|
+
if (siteContacts && footers.some(([s, e]) => h.where.i >= s && h.where.i < e)) { contacts.add(contactKey(h.plain ?? h.value)); continue }
|
|
896
|
+
hits.push({ ...h, where: h.where.el, ...at, file })
|
|
897
|
+
}
|
|
898
|
+
} else for (const h of scanText(text, { allow, where: 'text' })) hits.push({ ...h, ...at, file })
|
|
589
899
|
}
|
|
590
|
-
return
|
|
900
|
+
return hits.filter((h) => {
|
|
901
|
+
const v = String(h.plain ?? h.value).trim()
|
|
902
|
+
if (SAMPLE.test(v)) return false
|
|
903
|
+
if (!siteContacts) return true
|
|
904
|
+
if (contacts.has(contactKey(v))) return false
|
|
905
|
+
if (own && h.kind === 'email' && baseDomain(v.split('@')[1]) === own) return false
|
|
906
|
+
return true
|
|
907
|
+
})
|
|
591
908
|
}
|
|
592
909
|
|
|
593
910
|
/** The values a project says are not personal data, plus the run's own test mailboxes ({run} = any run). */
|
|
@@ -596,11 +913,11 @@ export function guardAllow ({ config = {}, env = {}, scenario = {}, run = null }
|
|
|
596
913
|
return [...(config.pii?.allow || []), ...(run?.meta?.pii?.allow || []), ...boxes.filter(Boolean).map((b) => String(b).replace('{run}', '*'))].map(String)
|
|
597
914
|
}
|
|
598
915
|
|
|
599
|
-
/** The refusal (or the warning
|
|
916
|
+
/** The refusal (or the warning) for findings: one line per hit, step and element first. */
|
|
600
917
|
export function formatFindings (findings, max = 12) {
|
|
601
918
|
const lines = findings.slice(0, max).map((f) => {
|
|
602
919
|
const where = f.node ? `${f.node}${f.locale ? ` (${f.locale}, ${f.viewport})` : ''}` : f.file
|
|
603
|
-
return ` ${where} ${f.where || ''} ${f.kind.replace('_', ' ')} ${hint(f.value)}`.replace(/ {3,}/g, ' ')
|
|
920
|
+
return ` ${where} ${f.where || ''} ${f.kind.replace('_', ' ')} ${hint(f.plain ?? f.value)}`.replace(/ {3,}/g, ' ')
|
|
604
921
|
})
|
|
605
922
|
if (findings.length > max) lines.push(` … ${findings.length - max} more`)
|
|
606
923
|
return lines.join('\n')
|