kaloko 0.12.0 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/lib/pii.mjs CHANGED
@@ -1,19 +1,23 @@
1
1
  // Personal data in captures (#64): one mask for every output of a step. The mask is applied to the page itself for the
2
- // moment of the capture (maskInPage): the screenshot, the stored HTML, the page outline for the evaluators, the
3
- // keyboard shots and the HTML view all see placeholders ("[hidden]") or consistent fake values (pseudonymize), and the
4
- // page gets its own text back right after. Recordings get an overlay that covers the same places while the walk runs
5
- // (maskPainter). Every value masked is remembered for the run (rememberPii), so traces, console lines, capture records
6
- // and later steps replace it too. `kaloko share` scans what it is about to upload (piiFindings) and refuses unmasked
7
- // e-mails, phone numbers, birth numbers and IBANs unless told --allow-pii.
2
+ // moment of the capture (maskInPage): the screenshot, the stored HTML, the page outline for the evaluators and the HTML
3
+ // view all see placeholders ("[hidden]") or consistent fake values (pseudonymize), and the page gets its own text back
4
+ // right after. Attributes nobody sees (data-*, content, links) are not touched in the live page: their masked values
5
+ // are applied to the stored HTML (maskHtml). Recordings get an overlay that covers the same places while the walk runs
6
+ // (maskPainter). Every value masked is remembered for the run (rememberPii, kept in the run folder for the next
7
+ // process), so traces, console lines, capture records and later steps replace it too. `kaloko share` scans what it is
8
+ // about to upload (piiFindings): a run that asked for hiding personal data is refused with unmasked e-mails, phone
9
+ // numbers, birth numbers or IBANs in it (unless --allow-pii); any other run gets a warning.
8
10
  //
9
11
  // piiKit, maskInPage and maskPainter run inside the browser: keep them self-contained (no outer references).
10
- import { readFile } from 'node:fs/promises'
12
+ import { readFile, writeFile } from 'node:fs/promises'
11
13
  import { join } from 'node:path'
12
14
  import { UserError } from './util.mjs'
13
15
 
14
16
  /**
15
17
  * Detection, placeholders and fake values. Self-contained: Node calls it directly, the page gets it serialised.
16
- * find(text, { kinds, allow }) → [{ kind, start, end, value }] in text order, without overlaps.
18
+ * find(text, { kinds, allow, context }) → [{ kind, start, end, value, plain? }] in text order, without overlaps.
19
+ * `context` is the text just before (a label in the previous element); `plain` is the decoded value of a hit in
20
+ * percent-encoded text (?email=jan%40firma.cz).
17
21
  */
18
22
  export function piiKit () {
19
23
  const KINDS = ['email', 'phone', 'birth_number', 'account', 'iban', 'birth_date', 'address', 'name']
@@ -22,10 +26,12 @@ export function piiKit () {
22
26
  const digits = (s) => String(s).replace(/\D/g, '')
23
27
  const wild = (p) => new RegExp('^' + String(p).trim().split('*').map((x) => x.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$', 'i')
24
28
  const allowed = (value, allow) => (allow || []).some((p) => wild(p).test(value.trim()) || (digits(p).length >= 6 && !String(p).includes('*') && digits(p) === digits(value)))
29
+ const DAYS = [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
25
30
  const birthNumberOk = (yy, mm, dd, tail) => {
26
31
  let m = Number(mm); const d = Number(dd)
27
- if (m > 70) m -= 70; else if (m > 50) m -= 50; else if (m > 20 && tail.length === 4) m -= 20
28
- if (m < 1 || m > 12 || d < 1 || d > 31) return false
32
+ // women +50; since 2004 also +20 (men) and +70 (women), only in the ten-digit form
33
+ if (m > 70 && tail.length === 4) m -= 70; else if (m > 50) m -= 50; else if (m > 20 && tail.length === 4) m -= 20
34
+ if (m < 1 || m > 12 || d < 1 || d > DAYS[m - 1]) return false
29
35
  if (tail.length === 3) return Number(yy) < 54
30
36
  const n = Number(`${yy}${mm}${dd}${tail}`)
31
37
  return n % 11 === 0 || (Math.floor(n / 10) % 11 === 10 && n % 10 === 0)
@@ -42,29 +48,48 @@ export function piiKit () {
42
48
  for (const c of r) m = (m * 10 + Number(c)) % 97
43
49
  return m === 1
44
50
  }
45
- const CURRENCY = /^\s?(Kč|Kc|CZK|EUR|€|\$|USD|,-)/
51
+ // an amount, not a phone number: a currency or decimals right after it, a currency right before it
52
+ const CURRENCY = /^\s?(Kč|Kc|CZK|EUR|€|\$|USD|£|GBP|,-|[.,]\d)/i
53
+ const CURRENCY_BEFORE = /(Kč|CZK|EUR|€|\$|USD|£|GBP)\s?$/i
54
+ // a phone number written 603 123 456 counts only next to a word that says so (or another phone number)
55
+ const TEL = /(?:^|[^\p{L}])(?:tel|telefon|telephone|phone|mobil|mobile|mob|gsm|fax|volejte|zavolejte|call|kontakt|contact|sms|whatsapp|hotline|linka)\p{L}*\.?(?![\p{L}])|[☎📞]|\+\d{2,3}[  ]?\d{3}/iu
56
+ // a birth number without the slash, or in the old nine-digit form, only next to its label
57
+ const BIRTH = /(?:^|[^\p{L}])(?:rodn\p{L}*|r\.\s?č\.?|rč|birth\s?number|personal\s?(?:id|number|identification))(?![\p{L}])/iu
58
+ const before = (text, at, ctx, n = 60) => (String(ctx || '') + ' ' + text.slice(Math.max(0, at - n), at)).slice(-n)
46
59
  const finders = {
47
60
  email (text, out) {
48
- for (const m of text.matchAll(/[\p{L}\p{N}._%+-]+@([\p{L}\p{N}-]+(?:\.[\p{L}\p{N}-]+)*\.(\p{L}{2,24}))(?![\p{L}\p{N}])/gu)) {
61
+ if (!text.includes('@')) return
62
+ // the local part at most 64 characters and from its start (a lookbehind): no quadratic search in long words
63
+ for (const m of text.matchAll(/(?<![\p{L}\p{N}._%+-])[\p{L}\p{N}._%+-]{1,64}@([\p{L}\p{N}-]{1,63}(?:\.[\p{L}\p{N}-]{1,63}){0,8}\.(\p{L}{2,24}))(?![\p{L}\p{N}])/gu)) {
49
64
  if (RESERVED.test(m[1]) || FILE_TLD.test(m[2]) || m[0].startsWith('.')) continue
50
65
  out.push({ kind: 'email', start: m.index, end: m.index + m[0].length })
51
66
  }
52
67
  },
53
- phone (text, out) {
54
- // international (+420 603 123 456, 00420…) and Czech grouped 3-3-3 (603 123 456); a fake keeps 000 after the code
55
- for (const m of text.matchAll(/(?<![\p{L}\p{N}+])(?:\+|00)\d{1,3}[ \u00a0.-]?(?:\(\d{1,4}\)[ \u00a0.-]?)?\d{1,4}(?:[ \u00a0.-]?\d{2,4}){1,4}(?![\p{N}])/gu)) {
68
+ phone (text, out, ctx) {
69
+ // international (+420 603 123 456, 00420 603 123 456); a fake keeps 000 after the code
70
+ for (const m of text.matchAll(/(?<![\p{L}\p{N}+\-./_#])(?:\+|00)\d{1,3}[  .-]?(?:\(\d{1,4}\)[  .-]?)?\d{1,4}(?:[  .-]?\d{2,4}){1,4}(?![\p{N}])/gu)) {
56
71
  const n = digits(m[0]).replace(/^00/, '')
57
- if (n.length < 9 || n.length > 15 || /^\+?(00)?\d{1,3}[ \u00a0.-]?0{3}/.test(m[0])) continue
58
- out.push({ kind: 'phone', start: m.index, end: m.index + m[0].length })
72
+ if (n.length < 9 || n.length > 15 || /^\+?(00)?\d{1,3}[  .-]?0{3}/.test(m[0])) continue
73
+ // order numbers and EANs start with 00 too: 00 only with a space after the country code
74
+ if (m[0].startsWith('00') && !/^00\d{1,3}[  ]\d/.test(m[0])) continue
75
+ const end = m.index + m[0].length
76
+ if (CURRENCY.test(text.slice(end, end + 6)) || CURRENCY_BEFORE.test(text.slice(Math.max(0, m.index - 6), m.index))) continue
77
+ out.push({ kind: 'phone', start: m.index, end })
59
78
  }
60
- for (const m of text.matchAll(/(?<![\p{N}+][ \u00a0]?)(?<![.,/-])[2-7]\d{2}[ \u00a0]\d{3}[ \u00a0]\d{3}(?![\p{N}])/gu)) {
61
- if (CURRENCY.test(text.slice(m.index + m[0].length, m.index + m[0].length + 5))) continue
62
- out.push({ kind: 'phone', start: m.index, end: m.index + m[0].length })
79
+ // Czech grouped 3-3-3 (603 123 456), with a label nearby; amounts (250 000 000 Kč, 349 990 000,00) are not
80
+ for (const m of text.matchAll(/(?<![\p{N}+][  ]?)(?<![\p{L}\p{N}.,/:#-])[2-7]\d{2}([  ])\d{3}\1\d{3}(?![\p{N}])/gu)) {
81
+ const end = m.index + m[0].length
82
+ if (CURRENCY.test(text.slice(end, end + 6)) || CURRENCY_BEFORE.test(text.slice(Math.max(0, m.index - 6), m.index))) continue
83
+ if (!TEL.test(before(text, m.index, ctx))) continue
84
+ out.push({ kind: 'phone', start: m.index, end })
63
85
  }
64
86
  },
65
- birth_number (text, out) {
66
- for (const m of text.matchAll(/(?<![\p{N}/])(\d{2})(\d{2})(\d{2}) ?\/ ?(\d{3,4})(?![\p{N}/])/gu)) {
67
- if (birthNumberOk(m[1], m[2], m[3], m[4])) out.push({ kind: 'birth_number', start: m.index, end: m.index + m[0].length })
87
+ birth_number (text, out, ctx) {
88
+ for (const m of text.matchAll(/(?<![\p{N}/.,-])(\d{2})(\d{2})(\d{2})( ?\/ ?)?(\d{3,4})(?![\p{N}/])/gu)) {
89
+ const slash = !!m[4]; const tail = m[5]
90
+ if (tail.length === 3 && !slash) continue
91
+ if ((tail.length === 3 || !slash) && !BIRTH.test(before(text, m.index, ctx, 40))) continue
92
+ if (birthNumberOk(m[1], m[2], m[3], tail)) out.push({ kind: 'birth_number', start: m.index, end: m.index + m[0].length })
68
93
  }
69
94
  },
70
95
  account (text, out) {
@@ -73,13 +98,13 @@ export function piiKit () {
73
98
  }
74
99
  },
75
100
  iban (text, out) {
76
- for (const m of text.matchAll(/(?<![\p{L}\p{N}])[A-Z]{2}\d{2}[ \u00a0]?[A-Z0-9]{4}/g)) {
101
+ for (const m of text.matchAll(/(?<![\p{L}\p{N}])[A-Z]{2}\d{2}[  ]?[A-Z0-9]{4}/g)) {
77
102
  // the longest run of groups that is a valid IBAN (the next word in capitals is not part of it)
78
- const tail = text.slice(m.index, m.index + 50).match(/^[A-Z0-9]+(?:[ \u00a0][A-Z0-9]{1,4})*/)?.[0] || m[0]
103
+ const tail = text.slice(m.index, m.index + 50).match(/^[A-Z0-9]+(?:[  ][A-Z0-9]{1,4})*/)?.[0] || m[0]
79
104
  for (let end = tail.length; end >= 15; end--) {
80
105
  const cand = tail.slice(0, end)
81
- if (/[ \u00a0]$/.test(cand) || /[A-Z0-9]/.test(tail[end] || '') && !/[ \u00a0]/.test(tail[end] || '')) continue
82
- if (ibanOk(cand.replace(/[ \u00a0]/g, ''))) { out.push({ kind: 'iban', start: m.index, end: m.index + end }); break }
106
+ if (/[  ]$/.test(cand) || /[A-Z0-9]/.test(tail[end] || '') && !/[  ]/.test(tail[end] || '')) continue
107
+ if (ibanOk(cand.replace(/[  ]/g, ''))) { out.push({ kind: 'iban', start: m.index, end: m.index + end }); break }
83
108
  }
84
109
  }
85
110
  },
@@ -92,8 +117,8 @@ export function piiKit () {
92
117
  address (text, out) {
93
118
  // Czech: street and number, postcode and town (one line or two); English: number and street
94
119
  for (const m of text.matchAll(/[\p{Lu}][\p{L}.'’ -]{1,40}? \d{1,5}(?:\/\d{1,5})?[a-zA-Z]?,?\s+\d{3} ?\d{2}\s+[\p{Lu}][\p{L}-]+(?: (?:[\p{Lu}][\p{L}-]+|\d{1,2}))*/gu)) out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
95
- for (const m of text.matchAll(/(?<![\p{N}])[1-9]\d{2}[ \u00a0]\d{2}[ \u00a0]+[\p{Lu}][\p{Ll}]+(?:[ -](?:[\p{Lu}][\p{Ll}]+|\d{1,2}))*(?![\p{L}])/gu)) {
96
- if (/^\d{3}[ \u00a0]\d{2}[ \u00a0]+(Kč|Kc|Eur|Ks|Kusů|Kus)/.test(m[0])) continue
120
+ for (const m of text.matchAll(/(?<![\p{N}])[1-9]\d{2}[  ]\d{2}[  ]+[\p{Lu}][\p{Ll}]+(?:[ -](?:[\p{Lu}][\p{Ll}]+|\d{1,2}))*(?![\p{L}])/gu)) {
121
+ if (/^\d{3}[  ]\d{2}[  ]+(Kč|Kc|Eur|Ks|Kusů|Kus)/.test(m[0])) continue
97
122
  out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
98
123
  }
99
124
  for (const m of text.matchAll(/(?<![\p{N}])\d{1,5} (?:[\p{Lu}][\p{Ll}]+ ){1,3}(?:Street|St\.?|Avenue|Ave\.?|Road|Rd\.?|Lane|Ln\.?|Boulevard|Blvd\.?|Drive|Dr\.?|Court|Ct\.?|Way|Place|Pl\.?|Square)(?![\p{L}])/gu)) out.push({ kind: 'address', start: m.index, end: m.index + m[0].length })
@@ -106,19 +131,54 @@ export function piiKit () {
106
131
  }
107
132
  }
108
133
  }
109
- /** Personal data in a text: [{ kind, start, end, value }], sorted, the longer of two overlapping matches kept. */
110
- function find (text, { kinds = KINDS, allow = [] } = {}) {
134
+ /**
135
+ * Percent-encoded text decoded, with each character's place in the original ({ text, from, to }), or null. UTF-8
136
+ * sequences decode together; a broken one stays as it is.
137
+ */
138
+ function decodePct (text) {
139
+ if (!/%[0-9A-Fa-f]{2}/.test(text)) return null
140
+ let out = ''; const from = []; const to = []
141
+ const keep = (s, at) => { for (let i = 0; i < s.length; i++) { out += s[i]; from.push(at + i); to.push(at + i + 1) } }
142
+ let at = 0
143
+ for (const m of text.matchAll(/(?:%[0-9A-Fa-f]{2})+/g)) {
144
+ keep(text.slice(at, m.index), at)
145
+ const bytes = m[0].length / 3
146
+ for (let b = 0; b < bytes;) {
147
+ const first = parseInt(m[0].substr(b * 3 + 1, 2), 16)
148
+ const n = first < 0x80 ? 1 : first >= 0xf0 ? 4 : first >= 0xe0 ? 3 : first >= 0xc0 ? 2 : 1
149
+ let ch = null
150
+ if (n <= bytes - b) { try { ch = decodeURIComponent(m[0].substr(b * 3, n * 3)) } catch {} }
151
+ if (ch == null) { keep(m[0].substr(b * 3, 3), m.index + b * 3); b++; continue }
152
+ const s = m.index + b * 3; const e = s + n * 3
153
+ for (let k = 0; k < ch.length; k++) { out += ch[k]; from.push(s); to.push(e) }
154
+ b += n
155
+ }
156
+ at = m.index + m[0].length
157
+ }
158
+ keep(text.slice(at), at)
159
+ return out === text ? null : { text: out, from, to }
160
+ }
161
+ const scan = (text, kinds, ctx) => {
111
162
  const all = []
112
- if (!text || !/[\p{N}@]/u.test(text) && !kinds.includes('name')) return all
113
- for (const k of kinds) finders[k]?.(String(text), all)
163
+ if (!/[\p{N}@]/u.test(text) && !kinds.includes('name')) return all
164
+ for (const k of kinds) finders[k]?.(text, all, ctx)
165
+ return all
166
+ }
167
+ /** Personal data in a text: [{ kind, start, end, value, plain? }], sorted, the longer of two overlapping matches kept. */
168
+ function find (text, { kinds = KINDS, allow = [], context = '' } = {}) {
169
+ text = text == null ? '' : String(text)
170
+ if (!text) return []
171
+ const all = scan(text, kinds, context)
172
+ const dec = text.length < 500000 ? decodePct(text) : null
173
+ if (dec) for (const r of scan(dec.text, kinds, context)) all.push({ kind: r.kind, start: dec.from[r.start], end: dec.to[r.end - 1], plain: dec.text.slice(r.start, r.end) })
114
174
  all.sort((a, b) => a.start - b.start || (b.end - b.start) - (a.end - a.start))
115
175
  const out = []
116
176
  let end = -1
117
177
  for (const r of all) {
118
178
  if (r.start < end) continue
119
179
  const value = text.slice(r.start, r.end)
120
- if (allowed(value, allow)) continue
121
- out.push({ ...r, value }); end = r.end
180
+ if (allowed(value, allow) || (r.plain && allowed(r.plain, allow))) continue
181
+ out.push({ kind: r.kind, start: r.start, end: r.end, value, ...(r.plain && r.plain !== value ? { plain: r.plain } : {}) }); end = r.end
122
182
  }
123
183
  return out
124
184
  }
@@ -149,7 +209,7 @@ export function piiKit () {
149
209
  /** The kind of a whole value (an element the scenario masks, a form field). */
150
210
  function kindOf (value) {
151
211
  const v = String(value || '').trim()
152
- const hit = find(v, { kinds: KINDS.filter((k) => k !== 'name') })
212
+ const hit = find(v, { kinds: KINDS.filter((k) => k !== 'name'), context: 'tel rč' })
153
213
  if (hit.length === 1 && hit[0].start === 0 && hit[0].end === v.length) return hit[0].kind
154
214
  return NAME.test(v) ? 'name' : 'text'
155
215
  }
@@ -168,31 +228,32 @@ export function piiKit () {
168
228
  if (/(^|\s)(name|given-name|family-name|additional-name)\b/.test(ac) || /^(name|full_?name|first_?name|last_?name|surname|given_?name|family_?name|jmeno|prijmeni)$/.test(key)) return 'name'
169
229
  return null
170
230
  }
171
- // fake values: a hash of the run's salt and the value picks them, so one value gets one fake in the whole run
231
+ // fake values: a hash of the salt (scenario and project) and the value picks them, so one value gets one fake in
232
+ // every run of the scenario; the hash keeps a few bits of the value at most, so a fake does not lead back to it
172
233
  const hash = (s) => { let x = 0x811c9dc5; for (const ch of String(s)) { x ^= ch.codePointAt(0); x = Math.imul(x, 0x01000193) >>> 0 } return x }
173
234
  const rng = (seed) => { let s = seed || 1; return () => { s ^= s << 13; s >>>= 0; s ^= s >>> 17; s ^= s << 5; s >>>= 0; return s / 4294967296 } }
174
235
  const pick = (r, list) => list[Math.floor(r() * list.length)]
175
236
  const dd = (r, n) => Array.from({ length: n }, () => Math.floor(r() * 10)).join('')
176
237
  const ascii = (s) => s.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase()
177
- const WOMEN = [['Jana', 'Nováková'], ['Eva', 'Svobodová'], ['Lucie', 'Dvořáková'], ['Tereza', 'Černá'], ['Klára', 'Procházková'], ['Anna', 'Kučerová']]
178
- const MEN = [['Petr', 'Novák'], ['Tomáš', 'Svoboda'], ['Martin', 'Dvořák'], ['Jakub', 'Černý'], ['David', 'Procházka'], ['Ondřej', 'Kučera']]
238
+ const WOMEN = [['Jana', 'Eva', 'Lucie', 'Tereza', 'Klára', 'Anna', 'Petra', 'Hana', 'Marie', 'Lenka'], ['Nováková', 'Svobodová', 'Dvořáková', 'Černá', 'Procházková', 'Kučerová', 'Veselá', 'Horáková', 'Němcová', 'Marková']]
239
+ const MEN = [['Petr', 'Tomáš', 'Martin', 'Jakub', 'David', 'Ondřej', 'Pavel', 'Karel', 'Lukáš', 'Filip'], ['Novák', 'Svoboda', 'Dvořák', 'Černý', 'Procházka', 'Kučera', 'Veselý', 'Horák', 'Němec', 'Marek']]
179
240
  const STREETS = ['Lipová', 'Krátká', 'Polní', 'Školní', 'Zahradní', 'Luční']
180
241
  const TOWNS = ['Vzorov', 'Ukázkov', 'Testov']
181
- function pseudonym (kind, value, salt = '') {
242
+ function pseudonym (kind, value, salt = '', attempt = 0) {
182
243
  const v = String(value || '').trim()
183
244
  const norm = kind === 'email' ? v.toLowerCase() : kind === 'phone' || kind === 'birth_number' || kind === 'account' ? digits(v) : v.replace(/\s+/g, ' ')
184
- const r = rng(hash(`${salt}|${kind === 'known' ? kindOf(v) : kind}|${norm}`))
185
- const person = () => { const p = pick(r, r() < 0.5 ? WOMEN : MEN); return p }
245
+ const r = rng(hash(`${salt}|${kind === 'known' ? kindOf(v) : kind}|${norm}${attempt ? `|${attempt}` : ''}`))
246
+ const person = () => { const [first, last] = r() < 0.5 ? WOMEN : MEN; return [pick(r, first), pick(r, last)] }
186
247
  const k = kind === 'known' || kind === 'text' ? kindOf(v) : kind
187
248
  if (k === 'email') { const [f, l] = person(); return `${ascii(f)}.${ascii(l)}${10 + Math.floor(r() * 90)}@example.com` }
188
249
  if (k === 'phone') {
189
- const cc = v.match(/^(?:\+|00)(\d{1,3})(?=[ \u00a0.(-])/)?.[1] || (/^(?:\+|00)/.test(v) ? '420' : '')
250
+ const cc = v.match(/^(?:\+|00)(\d{1,3})(?=[  .(-])/)?.[1] || (/^(?:\+|00)/.test(v) ? '420' : '')
190
251
  return `${cc ? `+${cc} ` : ''}000 ${dd(r, 3)} ${dd(r, 3)}`
191
252
  }
192
253
  // month 00 does not exist: a fake birth number never passes as a real one
193
254
  if (k === 'birth_number') return `${dd(r, 2)}00${String(1 + Math.floor(r() * 28)).padStart(2, '0')}/${dd(r, 4)}`
194
255
  if (k === 'account') return `${dd(r, 10).replace(/^0/, '1')}/0000`
195
- if (k === 'iban') { const n = v.replace(/[ \u00a0]/g, '').length; return (v.slice(0, 2) + '00' + dd(r, n - 4)).replace(/(.{4})(?=.)/g, '$1 ') }
256
+ if (k === 'iban') { const n = v.replace(/[  ]/g, '').length; return (v.slice(0, 2) + '00' + dd(r, n - 4)).replace(/(.{4})(?=.)/g, '$1 ') }
196
257
  if (k === 'birth_date') {
197
258
  const d = String(1 + Math.floor(r() * 28)); const m = String(1 + Math.floor(r() * 12)); const y = String(1950 + Math.floor(r() * 50))
198
259
  if (/^\d{4}-/.test(v)) return `${y}-${m.padStart(2, '0')}-${d.padStart(2, '0')}`
@@ -211,14 +272,63 @@ export function piiKit () {
211
272
  }
212
273
  /** "[hidden]" padded with dots to the length of the value it stands for (shorter values: dots only). */
213
274
  const placeholder = (value, label = '[hidden]') => { const n = [...String(value)].length; return n >= label.length ? label + '•'.repeat(n - label.length) : '•'.repeat(Math.max(1, n)) }
214
- const replacement = (kind, value, ctx) => ctx.mode === 'pseudonymize' ? pseudonym(kind, value, ctx.salt) : placeholder(value, ctx.label)
215
- return { KINDS, find, findKnown, merge, kindOf, fieldKind, pseudonym, placeholder, replacement }
275
+ /**
276
+ * The stand-in for a value: a placeholder, or a fake that is not in `taken` (fakes and real values already used),
277
+ * so two people never share one fake and a fake never equals a real value.
278
+ */
279
+ const replacement = (kind, value, ctx, taken = null) => {
280
+ if (ctx.mode !== 'pseudonymize') return placeholder(value, ctx.label)
281
+ let v = pseudonym(kind, value, ctx.salt)
282
+ for (let i = 1; taken && taken.has(v) && i < 40; i++) v = pseudonym(kind, value, ctx.salt, i)
283
+ const base = v
284
+ for (let n = 2; taken && taken.has(v); n++) v = base.includes('@') ? base.replace('@', `-${n}@`) : `${base} ${n}`
285
+ return v
286
+ }
287
+ // --- in the page only ---
288
+ /** This document and its same-origin frames: [{ doc, ox, oy }], ox/oy turn a frame's viewport point into a page point. */
289
+ function pageFrames () {
290
+ const out = []
291
+ const add = (doc, ox, oy, depth) => {
292
+ out.push({ doc, ox, oy })
293
+ if (depth >= 3) return
294
+ for (const f of doc.querySelectorAll('iframe, frame')) {
295
+ let d = null
296
+ try { d = f.contentDocument } catch {}
297
+ if (!d || !d.documentElement) continue
298
+ const r = f.getBoundingClientRect()
299
+ add(d, ox + r.left + f.clientLeft, oy + r.top + f.clientTop, depth + 1)
300
+ }
301
+ }
302
+ add(document, scrollX, scrollY, 0)
303
+ return out
304
+ }
305
+ /** A document and every open shadow root in it. */
306
+ function rootsOf (doc) {
307
+ const out = [doc]
308
+ for (let i = 0; i < out.length; i++) for (const el of out[i].querySelectorAll('*')) if (el.shadowRoot && el.tagName !== 'KALOKO-MASK') out.push(el.shadowRoot)
309
+ return out
310
+ }
311
+ /** The text nodes of a document or shadow root, in order. */
312
+ function textsOf (root) {
313
+ const start = root.nodeType === 9 ? root.documentElement : root
314
+ if (!start) return []
315
+ const w = (root.ownerDocument || root).createTreeWalker(start, 4)
316
+ const out = []
317
+ for (let n = w.nextNode(); n; n = w.nextNode()) out.push(n)
318
+ return out
319
+ }
320
+ /** The element above, also across a shadow boundary. */
321
+ const up = (node) => node.parentElement || (node.parentNode && node.parentNode.nodeType === 11 ? node.parentNode.host : null)
322
+ return { KINDS, find, findKnown, merge, kindOf, fieldKind, pseudonym, placeholder, replacement, decodePct, pageFrames, rootsOf, textsOf, up }
216
323
  }
217
324
 
218
325
  /**
219
- * Runs in the page (self-contained, `kit` is piiKit()): replaces masked text, form values and attributes in the live
220
- * document and keeps window.__kalokoMaskUndo to put them back. Returns the masked places in page coordinates
221
- * ({ x, y, w, h, kind, box }: box = drawn as a box on the screenshot) and the values masked ([original, replacement]).
326
+ * Runs in the page (self-contained, `kit` is piiKit()): replaces masked text, form values and the attributes a reader
327
+ * or the outline sees (title, alt, aria-label, placeholder, value) in the live document, in open shadow roots and in
328
+ * same-origin frames, and keeps window.__kalokoMaskUndo to put them back (in place from the first change, so a failure
329
+ * half-way leaves nothing masked). Attributes nobody sees (data-*, content, href, src) are left alone and returned as
330
+ * `attrs` ([original, masked]) for the stored HTML. Returns the masked places in page coordinates ({ x, y, w, h, kind,
331
+ * box }: box = drawn as a box on the screenshot) and the values masked ([original, replacement, kind]).
222
332
  */
223
333
  export function maskInPage (args, kit) {
224
334
  const { selectors = [], auto = false, mode = 'redact', salt = '', label = '[hidden]', allow = [], known = [] } = args
@@ -226,167 +336,207 @@ export function maskInPage (args, kit) {
226
336
  if (window.__kalokoMaskUndo) window.__kalokoMaskUndo()
227
337
  window.__kalokoMask?.pause()
228
338
  const undo = []
229
- const pairs = new Map(known.map(([v, r]) => [v, r]))
230
- const fresh = new Map()
231
- const rects = []
232
- const counts = {}
233
- const masked = new Set()
234
- for (const sel of selectors) { try { document.querySelectorAll(sel).forEach((el) => masked.add(el)) } catch {} }
235
- const inMasked = (node) => { for (let el = node.nodeType === 1 ? node : node.parentElement; el; el = el.parentElement) if (masked.has(el)) return true; return false }
236
- const CODE = ['email', 'birth_number', 'iban']
237
- const made = new Set(known.map(([, r]) => r))
238
- const repl = (r) => {
239
- if (r.repl != null) { counts.known = (counts.known || 0) + 1; return r.repl }
240
- if (made.has(r.value)) return r.value
241
- let v = pairs.get(r.value)
242
- if (v == null) { v = kit.replacement(r.kind, r.value, ctx); pairs.set(r.value, v) }
243
- fresh.set(r.value, v); made.add(v)
244
- counts[r.kind] = (counts[r.kind] || 0) + 1
245
- return v
246
- }
247
- // every word-ish part of a masked value is masked: found data with its own kind, the rest as a whole
248
- const whole = (text) => {
249
- const lead = text.length - text.trimStart().length; const core = text.trim()
250
- if (!core) return []
251
- const found = kit.merge(kit.find(core, { allow: [] }), kit.findKnown(core, known))
252
- const out = []; let at = 0
253
- const rest = (s, e) => { const seg = core.slice(s, e); const t = seg.trim(); if (!/[\p{L}\p{N}]/u.test(t)) return; const off = s + seg.indexOf(t); out.push({ kind: kit.kindOf(t), start: lead + off, end: lead + off + t.length, value: t }) }
254
- for (const r of found) { rest(at, r.start); out.push({ ...r, start: lead + r.start, end: lead + r.end }); at = r.end }
255
- rest(at, core.length)
256
- return out
257
- }
258
- const ranges = (text, isMasked, code = false) => isMasked ? whole(text) : kit.merge(auto ? kit.find(text, { allow, kinds: code ? CODE : undefined }) : [], kit.findKnown(text, known))
259
- const rendered = (el) => el && !el.closest('head, script, style, noscript, template, title') && el.getClientRects().length > 0
260
- const push = (r, kind, box) => { if (rects.length < 500 && r.width > 0 && r.height > 0) rects.push({ x: Math.round(r.left + scrollX), y: Math.round(r.top + scrollY), w: Math.ceil(r.width), h: Math.ceil(r.height), kind, box }) }
261
- // whole elements the scenario masks: one box each (drawn in both modes when they hold no text, e.g. a photo)
262
- for (const el of masked) { if (rendered(el) && !masked.has(el.parentElement)) push(el.getBoundingClientRect(), 'mask', mode === 'redact' || !el.textContent.trim()) }
263
- // text
264
- const walker = document.createTreeWalker(document.documentElement, NodeFilter.SHOW_TEXT)
265
- const nodes = []
266
- for (let n = walker.nextNode(); n; n = walker.nextNode()) nodes.push(n)
267
- for (const node of nodes) {
268
- const parent = node.parentElement
269
- if (!parent || parent.closest('style, kaloko-mask')) continue
270
- const text = node.nodeValue
271
- if (!text || !text.trim()) continue
272
- const isMasked = inMasked(node)
273
- const list = ranges(text, isMasked, !!parent.closest('script'))
274
- if (!list.length) continue
275
- let out = ''; let at = 0; const spans = []
276
- for (const r of list) { const v = repl(r); out += text.slice(at, r.start); spans.push([out.length, out.length + v.length, r.kind === 'known' ? 'known' : r.kind]); out += v; at = r.end }
277
- out += text.slice(at)
278
- node.nodeValue = out
279
- undo.push(() => { node.nodeValue = text })
280
- if (!isMasked && rendered(parent)) {
281
- for (const [s, e, kind] of spans) { const range = document.createRange(); range.setStart(node, s); range.setEnd(node, e); for (const r of range.getClientRects()) push(r, kind, mode === 'redact') }
339
+ window.__kalokoMaskUndo = () => { for (const f of undo.splice(0).reverse()) { try { f() } catch {} } delete window.__kalokoMaskUndo; window.__kalokoMask?.resume() }
340
+ try {
341
+ const pairs = new Map(known.map(([v, r]) => [v, r]))
342
+ const taken = new Set(known.flatMap(([v, r]) => [v, r]))
343
+ const fresh = new Map()
344
+ const rects = []
345
+ const counts = {}
346
+ const attrs = new Map()
347
+ const roots = kit.pageFrames().flatMap((f) => kit.rootsOf(f.doc).map((root) => ({ root, f })))
348
+ const masked = new Set()
349
+ for (const { root } of roots) for (const sel of selectors) { try { root.querySelectorAll(sel).forEach((el) => masked.add(el)) } catch {} }
350
+ const inMasked = (node) => { for (let el = node.nodeType === 1 ? node : kit.up(node); el; el = kit.up(el)) if (masked.has(el)) return true; return false }
351
+ const CODE = ['email', 'birth_number', 'iban']
352
+ const repl = (r) => {
353
+ if (r.repl != null) { counts.known = (counts.known || 0) + 1; return r.repl }
354
+ const real = r.plain ?? r.value
355
+ let v = pairs.get(real) ?? pairs.get(r.value)
356
+ if (v == null) { taken.add(real); v = kit.replacement(r.kind, real, ctx, taken); taken.add(v) }
357
+ pairs.set(r.value, v); pairs.set(real, v)
358
+ // a part of a masked element that is no kind of its own is plain text (remembered only when it is long)
359
+ const kind = r.guess ? 'text' : r.kind
360
+ fresh.set(r.value, [v, kind]); if (real !== r.value) fresh.set(real, [v, kind])
361
+ counts[r.kind] = (counts[r.kind] || 0) + 1
362
+ return v
282
363
  }
283
- }
284
- // form fields as they are filled in (the value property, not only the attribute)
285
- const SKIP = /^(password|hidden|submit|button|checkbox|radio|file|image|reset|range|color)$/
286
- for (const el of document.querySelectorAll('input, textarea')) {
287
- if (SKIP.test(el.type || '')) continue
288
- const value = el.value
289
- if (!value || !value.trim()) continue
290
- const isMasked = inMasked(el)
291
- const kind = !isMasked && auto ? kit.fieldKind(el) : null
292
- const list = kind ? [{ kind, start: value.length - value.trimStart().length, end: value.trimEnd().length, value: value.trim() }] : ranges(value, isMasked)
293
- if (!list.length) continue
294
- let out = ''; let at = 0
295
- for (const r of list) { out += value.slice(at, r.start) + repl(r); at = r.end }
296
- out += value.slice(at)
297
- const attr = el.getAttribute('value')
298
- el.value = out
299
- if (attr != null) el.setAttribute('value', out)
300
- undo.push(() => { el.value = value; if (attr != null) el.setAttribute('value', attr) })
301
- if (!isMasked && rendered(el)) push(el.getBoundingClientRect(), kind || list[0].kind, mode === 'redact')
302
- }
303
- // attributes that show or carry text: title, alt, aria-label, placeholder, value, content, mailto:/tel: links, data-*
304
- const ATTR = /^(title|alt|aria-label|aria-description|placeholder|value|content|href|data-[\w-]+)$/i
305
- for (const el of document.querySelectorAll('*')) {
306
- if (el.closest('kaloko-mask')) continue
307
- const isMasked = inMasked(el)
308
- for (const a of [...el.attributes]) {
309
- if (!ATTR.test(a.name) || !a.value.trim()) continue
310
- if (a.name === 'href' && !/^\s*(mailto|tel):/i.test(a.value)) continue
311
- if (a.name === 'value' && /^(INPUT|TEXTAREA)$/.test(el.tagName)) continue // the fields above
312
- const list = isMasked && a.name !== 'href' ? whole(a.value) : ranges(a.value, false)
313
- const hits = isMasked && a.name === 'href' ? [{ kind: 'text', start: a.value.indexOf(':') + 1, end: a.value.length, value: a.value.slice(a.value.indexOf(':') + 1) }] : list
314
- if (!hits.length) continue
315
- let out = ''; let at = 0
316
- for (const r of hits) { out += a.value.slice(at, r.start) + repl(r); at = r.end }
317
- out += a.value.slice(at)
318
- const name = a.name; const before = a.value
319
- el.setAttribute(name, out)
320
- undo.push(() => el.setAttribute(name, before))
364
+ // every word-ish part of a masked value is masked: found data with its own kind, the rest as a whole
365
+ const whole = (text) => {
366
+ const lead = text.length - text.trimStart().length; const core = text.trim()
367
+ if (!core) return []
368
+ const found = kit.merge(kit.find(core, { allow: [] }), kit.findKnown(core, known))
369
+ const out = []; let at = 0
370
+ const rest = (s, e) => { const seg = core.slice(s, e); const t = seg.trim(); if (!/[\p{L}\p{N}]/u.test(t)) return; const off = s + seg.indexOf(t); out.push({ kind: kit.kindOf(t), guess: true, start: lead + off, end: lead + off + t.length, value: t }) }
371
+ for (const r of found) { rest(at, r.start); out.push({ ...r, start: lead + r.start, end: lead + r.end }); at = r.end }
372
+ rest(at, core.length)
373
+ return out
374
+ }
375
+ const ranges = (text, isMasked, kinds, context = '') => isMasked ? whole(text) : kit.merge(auto ? kit.find(text, { allow, kinds, context }) : [], kit.findKnown(text, known))
376
+ const rendered = (el) => el && !el.closest('head, script, style, noscript, template, title') && el.getClientRects().length > 0
377
+ const push = (r, kind, box, f) => { if (rects.length < 500 && r.width > 0 && r.height > 0) rects.push({ x: Math.round(r.left + f.ox), y: Math.round(r.top + f.oy), w: Math.ceil(r.width), h: Math.ceil(r.height), kind, box }) }
378
+ const spliceIn = (text, list) => { let out = ''; let at = 0; const spans = []; for (const r of list) { const v = repl(r); out += text.slice(at, r.start); spans.push([out.length, out.length + v.length, r.kind === 'known' ? 'known' : r.kind]); out += v; at = r.end } return { out: out + text.slice(at), spans } }
379
+ // whole elements the scenario masks: one box each (drawn in both modes when they hold no text, e.g. a photo)
380
+ for (const { root, f } of roots) for (const el of masked) if (el.getRootNode() === root && rendered(el) && !masked.has(kit.up(el))) push(el.getBoundingClientRect(), 'mask', mode === 'redact' || !el.textContent.trim(), f)
381
+ // text; the text before a node is its context (a "Telefon" label in the cell before the number)
382
+ let prev = ''
383
+ for (const { root, f } of roots) {
384
+ for (const node of kit.textsOf(root)) {
385
+ const parent = kit.up(node)
386
+ if (!parent || parent.closest('style, kaloko-mask')) continue
387
+ const text = node.nodeValue
388
+ if (!text || !text.trim()) continue
389
+ const script = parent.closest('script')
390
+ // inline JSON (JSON-LD, state) gets every kind; script code only the kinds a checksum or an @ confirms
391
+ const kinds = script && !/json/i.test(script.type || '') ? CODE : undefined
392
+ const isMasked = inMasked(node)
393
+ const list = ranges(text, isMasked, kinds, prev)
394
+ if (!script) prev = (prev + ' ' + text).slice(-80)
395
+ if (!list.length) continue
396
+ const { out, spans } = spliceIn(text, list)
397
+ node.nodeValue = out
398
+ undo.push(() => { node.nodeValue = text })
399
+ if (!isMasked && rendered(parent)) {
400
+ for (const [s, e, kind] of spans) { const range = node.ownerDocument.createRange(); range.setStart(node, s); range.setEnd(node, e); for (const r of range.getClientRects()) push(r, kind, mode === 'redact', f) }
401
+ }
402
+ }
403
+ }
404
+ // form fields as they are filled in (the value property, not only the attribute), hidden ones included
405
+ const SKIP = /^(password|submit|button|checkbox|radio|file|image|reset|range|color)$/
406
+ const handled = new Set()
407
+ for (const { root, f } of roots) {
408
+ for (const el of root.querySelectorAll('input, textarea')) {
409
+ if (SKIP.test(el.type || '')) continue
410
+ const value = el.value
411
+ if (!value || !value.trim()) continue
412
+ const hidden = el.type === 'hidden'
413
+ const isMasked = inMasked(el)
414
+ let kind = !isMasked && auto ? kit.fieldKind(el) : null
415
+ if (hidden && kind === 'name') kind = null // name="name" on a hidden field is rarely a person
416
+ const list = kind ? [{ kind, start: value.length - value.trimStart().length, end: value.trimEnd().length, value: value.trim() }] : ranges(value, isMasked)
417
+ if (!list.length) continue
418
+ const { out } = spliceIn(value, list)
419
+ const attr = el.getAttribute('value')
420
+ el.value = out
421
+ if (attr != null) el.setAttribute('value', out)
422
+ handled.add(el)
423
+ undo.push(() => { el.value = value; if (attr != null) el.setAttribute('value', attr) })
424
+ if (!hidden && !isMasked && rendered(el)) push(el.getBoundingClientRect(), kind || list[0].kind, mode === 'redact', f)
425
+ }
426
+ }
427
+ // attributes: what a reader or the outline sees is masked in the page; the rest only in the stored HTML (no
428
+ // observer of the page sees a data-* attribute change)
429
+ const LIVE = /^(title|alt|aria-label|aria-description|aria-valuetext|placeholder|value|label)$/i
430
+ const STORED = /^(data-[\w-]+|content|href|src|action|formaction|poster|srcset)$/i
431
+ const LINK = /^(href|src|action|formaction|poster|srcset)$/i
432
+ for (const { root, f } of roots) {
433
+ const top = f.doc === document && root === document
434
+ for (const el of root.querySelectorAll('*')) {
435
+ if (el.tagName === 'KALOKO-MASK') continue
436
+ const isMasked = inMasked(el)
437
+ for (const a of [...el.attributes]) {
438
+ const live = LIVE.test(a.name)
439
+ if ((!live && !(top && STORED.test(a.name))) || !a.value.trim()) continue
440
+ if (a.name === 'value' && handled.has(el)) continue // the fields above
441
+ const link = LINK.test(a.name)
442
+ const scheme = link && /^\s*(mailto|tel|sms):/i.test(a.value)
443
+ const at = a.value.indexOf(':') + 1
444
+ const hits = isMasked && scheme ? [{ kind: 'text', guess: true, start: at, end: a.value.length, value: a.value.slice(at) }] : isMasked && !link ? whole(a.value) : ranges(a.value, false)
445
+ if (!hits.length) continue
446
+ const { out } = spliceIn(a.value, hits)
447
+ if (live) { const name = a.name; const before = a.value; el.setAttribute(name, out); undo.push(() => el.setAttribute(name, before)) } else attrs.set(a.value, out)
448
+ }
449
+ }
321
450
  }
451
+ return { rects, pairs: [...fresh].map(([v, [r, k]]) => [v, r, k]), counts, attrs: [...attrs], changed: undo.length + attrs.size }
452
+ } catch (error) {
453
+ window.__kalokoMaskUndo?.()
454
+ throw error
322
455
  }
323
- window.__kalokoMaskUndo = () => { for (const f of undo.reverse()) { try { f() } catch {} } delete window.__kalokoMaskUndo; window.__kalokoMask?.resume() }
324
- return { rects, pairs: [...fresh].filter(([v]) => v.length >= 3), counts }
325
456
  }
326
457
 
327
458
  /**
328
459
  * Runs in the page (self-contained, `kit` is piiKit()): an overlay that keeps every masked place covered while the page
329
- * changes, for recordings, focus shots and crops. window.__kalokoMask: update(args), pause(), resume().
460
+ * changes, for recordings. It reads open shadow roots and same-origin frames, and paints inside an open modal dialog
461
+ * or popover too (the top layer is above anything else in the page). window.__kalokoMask: update(args), pause(),
462
+ * resume().
330
463
  */
331
464
  export function maskPainter (args, kit) {
332
465
  if (window.__kalokoMask) { window.__kalokoMask.update(args); return }
333
466
  let cfg = { selectors: [], known: [], ...args }
334
- let paused = false; let pending = false; let host = null; let root = null
335
- const ensure = () => {
336
- if (host?.isConnected || !document.documentElement) return
337
- host = document.createElement('kaloko-mask')
338
- host.style.cssText = 'position:absolute;left:0;top:0;width:0;height:0;overflow:visible;z-index:2147483647;pointer-events:none;display:block;margin:0;padding:0;border:0'
339
- root = host.attachShadow({ mode: 'closed' })
340
- document.documentElement.appendChild(host)
467
+ let paused = false; let pending = false
468
+ const hosts = new Map()
469
+ const hostIn = (container, fixed) => {
470
+ const had = hosts.get(container)
471
+ if (had?.host.isConnected) return had
472
+ const host = document.createElement('kaloko-mask')
473
+ host.style.cssText = `position:${fixed ? 'fixed' : 'absolute'};left:0;top:0;width:0;height:0;overflow:visible;z-index:2147483647;pointer-events:none;display:block;margin:0;padding:0;border:0`
474
+ const root = host.attachShadow({ mode: 'closed' })
475
+ container.appendChild(host)
476
+ const h = { host, root, fixed }
477
+ hosts.set(container, h)
478
+ return h
341
479
  }
480
+ const topLayer = () => { for (const q of ['dialog:modal, :popover-open', 'dialog[open]']) { try { return [...document.querySelectorAll(q)] } catch {} } return [] }
342
481
  const collect = () => {
343
482
  const out = []
344
- const add = (r) => { if (out.length < 500 && r.width > 0 && r.height > 0) out.push([r.left + scrollX, r.top + scrollY, r.width, r.height]) }
345
- const masked = []
346
- for (const sel of cfg.selectors) { try { document.querySelectorAll(sel).forEach((el) => masked.push(el)) } catch {} }
347
- for (const el of masked) add(el.getBoundingClientRect())
348
- const inMasked = (el) => masked.some((m) => m.contains(el))
349
- if (!document.body) return out
350
- const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT)
351
- for (let n = walker.nextNode(); n; n = walker.nextNode()) {
352
- const p = n.parentElement
353
- if (!p || p.closest('script, style, noscript, template, kaloko-mask') || !n.nodeValue.trim()) continue
354
- const list = kit.merge(cfg.auto ? kit.find(n.nodeValue, { allow: cfg.allow }) : [], kit.findKnown(n.nodeValue, cfg.known))
355
- if (!list.length || inMasked(p)) continue
356
- for (const r of list) { const range = document.createRange(); range.setStart(n, r.start); range.setEnd(n, r.end); for (const c of range.getClientRects()) add(c) }
357
- }
358
- for (const el of document.querySelectorAll('input, textarea')) {
359
- if (/^(password|hidden|submit|button|checkbox|radio|file|image|reset|range|color)$/.test(el.type || '') || !el.value.trim() || inMasked(el)) continue
360
- if ((cfg.auto && (kit.fieldKind(el) || kit.find(el.value, { allow: cfg.allow }).length)) || kit.findKnown(el.value, cfg.known).length) add(el.getBoundingClientRect())
483
+ const add = (r, f) => { if (out.length < 500 && r.width > 0 && r.height > 0) out.push([r.left + f.ox, r.top + f.oy, r.width, r.height]) }
484
+ const roots = kit.pageFrames().flatMap((f) => kit.rootsOf(f.doc).map((root) => ({ root, f })))
485
+ const masked = new Set()
486
+ for (const { root, f } of roots) for (const sel of cfg.selectors) { try { root.querySelectorAll(sel).forEach((el) => { masked.add(el); add(el.getBoundingClientRect(), f) }) } catch {} }
487
+ const inMasked = (el) => { for (let e = el; e; e = kit.up(e)) if (masked.has(e)) return true; return false }
488
+ let prev = ''
489
+ for (const { root, f } of roots) {
490
+ for (const n of kit.textsOf(root)) {
491
+ const p = kit.up(n)
492
+ if (!p || p.closest('head, title, script, style, noscript, template, kaloko-mask') || !n.nodeValue.trim()) continue
493
+ const list = kit.merge(cfg.auto ? kit.find(n.nodeValue, { allow: cfg.allow, context: prev }) : [], kit.findKnown(n.nodeValue, cfg.known))
494
+ prev = (prev + ' ' + n.nodeValue).slice(-80)
495
+ if (!list.length || inMasked(p)) continue
496
+ for (const r of list) { const range = n.ownerDocument.createRange(); range.setStart(n, r.start); range.setEnd(n, r.end); for (const c of range.getClientRects()) add(c, f) }
497
+ }
498
+ for (const el of root.querySelectorAll('input, textarea')) {
499
+ if (/^(password|hidden|submit|button|checkbox|radio|file|image|reset|range|color)$/.test(el.type || '') || !el.value.trim() || inMasked(el)) continue
500
+ if ((cfg.auto && (kit.fieldKind(el) || kit.find(el.value, { allow: cfg.allow }).length)) || kit.findKnown(el.value, cfg.known).length) add(el.getBoundingClientRect(), f)
501
+ }
361
502
  }
362
503
  return out
363
504
  }
364
505
  const paint = () => {
365
506
  pending = false
366
- if (paused) return
367
- ensure()
368
- if (!root) return
369
- const boxes = collect().map(([x, y, w, h]) => { const d = document.createElement('div'); d.style.cssText = `position:absolute;left:${x}px;top:${y}px;width:${w}px;height:${h}px;background:repeating-linear-gradient(135deg,#1f1f1d 0 6px,#34342f 6px 12px)`; return d })
370
- root.replaceChildren(...boxes)
507
+ if (paused || !document.documentElement) return
508
+ const boxes = collect()
509
+ const draw = (h) => {
510
+ // a host inside the top layer is fixed: viewport coordinates, from wherever it actually sits
511
+ const o = h.fixed ? h.host.getBoundingClientRect() : null
512
+ const dx = o ? -scrollX - o.left : 0; const dy = o ? -scrollY - o.top : 0
513
+ h.root.replaceChildren(...boxes.map(([x, y, w, hh]) => { const d = document.createElement('div'); d.style.cssText = `position:absolute;left:${x + dx}px;top:${y + dy}px;width:${w}px;height:${hh}px;background:repeating-linear-gradient(135deg,#1f1f1d 0 6px,#34342f 6px 12px)`; return d }))
514
+ }
515
+ draw(hostIn(document.documentElement, false))
516
+ const tops = new Set(topLayer())
517
+ for (const [c, h] of hosts) if (c !== document.documentElement && !tops.has(c)) { h.host.remove(); hosts.delete(c) }
518
+ for (const el of tops) draw(hostIn(el, true))
371
519
  }
372
520
  const schedule = () => { if (!pending && !paused) { pending = true; requestAnimationFrame(paint) } }
373
521
  const start = () => {
374
- new MutationObserver(schedule).observe(document.documentElement, { subtree: true, childList: true, characterData: true })
375
- for (const ev of ['scroll', 'input', 'resize', 'load']) addEventListener(ev, schedule, true)
522
+ new MutationObserver(schedule).observe(document.documentElement, { subtree: true, childList: true, characterData: true, attributes: true, attributeFilter: ['open', 'popover', 'value', 'style', 'class', 'hidden'] })
523
+ for (const ev of ['scroll', 'input', 'resize', 'load', 'toggle']) addEventListener(ev, schedule, true)
376
524
  setInterval(schedule, 400)
377
525
  schedule()
378
526
  }
379
527
  window.__kalokoMask = {
380
528
  update (a) { cfg = { ...cfg, ...a, selectors: [...new Set([...cfg.selectors, ...(a.selectors || [])])], known: [...new Map([...cfg.known, ...(a.known || [])]).entries()] }; schedule() },
381
- pause () { paused = true; host?.remove() },
529
+ pause () { paused = true; for (const h of hosts.values()) h.host.remove(); hosts.clear() },
382
530
  resume () { paused = false; schedule() }
383
531
  }
384
- if (document.documentElement) start(); else document.addEventListener('DOMContentLoaded', start, { once: true })
532
+ // an init script runs before the document has an element: start with the first one, not at DOMContentLoaded
533
+ if (document.documentElement) start()
534
+ else new MutationObserver((_, o) => { if (document.documentElement) { o.disconnect(); start() } }).observe(document, { childList: true })
385
535
  }
386
536
 
387
537
  const kit = piiKit()
388
538
  export const findPii = (text, opts) => kit.find(text, opts)
389
- export const pseudonym = (kind, value, salt) => kit.pseudonym(kind, value, salt)
539
+ export const pseudonym = (kind, value, salt, attempt) => kit.pseudonym(kind, value, salt, attempt)
390
540
  export const placeholder = (value, label) => kit.placeholder(value, label)
391
541
 
392
542
  /** The text of the masked placeholder in a locale. */
@@ -414,6 +564,11 @@ export function piiSettings ({ config = {}, env = {}, scenario = {}, args = {} }
414
564
  return { mode, allow: [...(config.pii?.allow || [])].map(String) }
415
565
  }
416
566
 
567
+ /** Whether a run asked for hiding personal data (its own setting, the environment, the scenario or a mask: auto). */
568
+ export function piiRequested ({ run = null, env = {}, scenario = {} }) {
569
+ return !!(run?.meta?.pii || ['required', ...PII_MODES].includes(env?.pii) || PII_MODES.includes(scenario?.pii) || scenarioMasks(scenario).auto)
570
+ }
571
+
417
572
  /** capture.mask as given (a list of selectors, `auto`, or a list with `auto` in it): { selectors, auto }. */
418
573
  export function maskList (mask) {
419
574
  const list = mask == null ? [] : Array.isArray(mask) ? mask : [mask]
@@ -421,9 +576,19 @@ export function maskList (mask) {
421
576
  return { selectors: items.filter((s) => s !== 'auto'), auto: items.includes('auto') }
422
577
  }
423
578
 
579
+ /**
580
+ * The salt of fake values: the scenario and the project, so one value gets the same fake in every run of the scenario
581
+ * (pixel diffs and approvals carry over). KALOKO_PII_SALT adds a secret of your own.
582
+ */
583
+ export function piiSalt (meta = {}) {
584
+ const p = meta?.project
585
+ const project = typeof p === 'string' ? p : p?.id || p?.slug || p?.name || ''
586
+ return [process.env.KALOKO_PII_SALT || '', project, meta?.scenario?.id || meta?.id || ''].join('|')
587
+ }
588
+
424
589
  /**
425
590
  * What one capture masks: the selectors of the scenario and the step, automatic detection (the run's pii setting or
426
- * `mask: auto`), the mode and the run's salt for fake values. `active` is false when there is nothing to mask.
591
+ * `mask: auto`), the mode and the salt for fake values. `active` is false when there is nothing to mask.
427
592
  */
428
593
  export function maskSpec ({ run = null, selectors = [], auto = false, locale = null }) {
429
594
  const pii = run?.meta?.pii || null
@@ -431,7 +596,7 @@ export function maskSpec ({ run = null, selectors = [], auto = false, locale = n
431
596
  selectors: [...new Set(selectors)],
432
597
  auto: !!(auto || pii),
433
598
  mode: pii?.mode || 'redact',
434
- salt: run?.meta?.id || '',
599
+ salt: piiSalt(run?.meta),
435
600
  label: maskLabel(locale),
436
601
  allow: pii?.allow || []
437
602
  }
@@ -445,23 +610,55 @@ export function scenarioMasks (scenario) {
445
610
  return { selectors: [...new Set(all.flatMap((m) => m.selectors))], auto: all.some((m) => m.auto) }
446
611
  }
447
612
 
448
- // --- the values masked in this process ---------------------------------------------------------------------------
613
+ // --- the values masked in this run ---------------------------------------------------------------------------------
449
614
 
450
615
  const remembered = new Map()
616
+ const kindsOf = new Map()
617
+ const loaded = new Set()
451
618
  let pattern = null
452
619
 
453
- /** Remembers masked values ([original, replacement]) so every later output of the run replaces them too. */
620
+ const digitsOf = (s) => String(s).replace(/\D/g, '')
621
+ const NUMERIC = /^[\d\s.,:/+%-]+$/
622
+ /**
623
+ * Whether a masked value is replaced everywhere else: values of a detected kind, other text from 6 characters on,
624
+ * numbers only from 6 digits on (2024, 100 or Praha are not worth replacing across pages and markup).
625
+ */
626
+ function worthRemembering (v, kind) {
627
+ if (v.length < 3) return false
628
+ if (NUMERIC.test(v) && digitsOf(v).length < 6) return false
629
+ if (!kind || kind === 'text' || kind === 'known' || kind === 'mask') return v.length >= 6
630
+ return true
631
+ }
632
+
633
+ /** Remembers masked values ([original, replacement, kind?]) so every later output of the run replaces them too. */
454
634
  export function rememberPii (pairs) {
455
- for (const [value, repl] of pairs || []) {
635
+ for (const [value, repl, kind] of pairs || []) {
456
636
  const v = String(value || '').trim()
457
- if (v.length >= 3 && !remembered.has(v)) { remembered.set(v, String(repl)); pattern = null }
637
+ if (worthRemembering(v, kind) && !remembered.has(v)) { remembered.set(v, String(repl)); kindsOf.set(v, kind || 'text'); pattern = null }
458
638
  }
459
639
  }
460
640
 
461
641
  /** The values remembered so far, as [[original, replacement]] (for the page and the overlay). */
462
642
  export const knownPii = () => [...remembered]
463
643
 
464
- export const _forgetPii = () => { remembered.clear(); pattern = null } // tests
644
+ export const _forgetPii = () => { remembered.clear(); kindsOf.clear(); loaded.clear(); pattern = null } // tests
645
+
646
+ // The values travel to the next `kaloko` process of the run in the run folder. The file stays on this computer:
647
+ // share uploads only the files the manifest names.
648
+ export const KNOWN_FILE = '.pii-known.json'
649
+
650
+ /** Reads the values an earlier process of this run masked (once per run and process). */
651
+ export async function loadKnownPii (dir) {
652
+ if (!dir || loaded.has(dir)) return
653
+ loaded.add(dir)
654
+ try { rememberPii(JSON.parse(await readFile(join(dir, KNOWN_FILE), 'utf8'))) } catch {}
655
+ }
656
+
657
+ /** Keeps the values masked so far for the next process of this run. */
658
+ export async function saveKnownPii (dir) {
659
+ if (!dir || !remembered.size) return
660
+ await writeFile(join(dir, KNOWN_FILE), JSON.stringify([...remembered].map(([v, r]) => [v, r, kindsOf.get(v)])), { mode: 0o600 }).catch(() => {})
661
+ }
465
662
 
466
663
  /** Replaces every remembered value (and its URL-encoded form) where it stands as a whole word. */
467
664
  export function redactPii (text) {
@@ -472,39 +669,68 @@ export function redactPii (text) {
472
669
  const keys = [...forms.keys()].sort((a, b) => b.length - a.length).map((k) => k.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))
473
670
  pattern = { re: new RegExp(`(?<![\\p{L}\\p{N}])(?:${keys.join('|')})(?![\\p{L}\\p{N}])`, 'gu'), forms }
474
671
  }
475
- return text.replace(pattern.re, (m) => pattern.forms.get(m) ?? m)
672
+ return text.replace(pattern.re, (m, at, all) => {
673
+ // a number in markup stays: a colour (#123456), a percentage
674
+ if (NUMERIC.test(m) && (all[at - 1] === '#' || all[at + m.length] === '%')) return m
675
+ return pattern.forms.get(m) ?? m
676
+ })
476
677
  }
477
678
 
679
+ const takenNow = () => new Set([...remembered.keys(), ...remembered.values()])
680
+
478
681
  /** A text from outside the page (trace, console, mail headers): remembered values, then detection when it is on. */
479
682
  export function scrubText (text, spec) {
480
683
  let out = redactPii(text)
481
684
  if (typeof out !== 'string' || !spec?.auto) return out
482
685
  const hits = kit.find(out, { allow: spec.allow })
686
+ const taken = takenNow()
483
687
  for (const r of hits.reverse()) {
484
- const repl = kit.replacement(r.kind, r.value, spec)
485
- rememberPii([[r.value, repl]])
688
+ const real = r.plain ?? r.value
689
+ let repl = remembered.get(real)
690
+ if (repl == null) { taken.add(real); repl = kit.replacement(r.kind, real, spec, taken); taken.add(repl) }
691
+ rememberPii([[real, repl, r.kind], ...(r.plain ? [[r.value, repl, r.kind]] : [])])
486
692
  out = out.slice(0, r.start) + repl + out.slice(r.end)
487
693
  }
488
694
  return out
489
695
  }
490
696
 
491
- /** Deep copy with every string scrubbed (scrubText). */
492
- export function scrubDeep (value, spec) {
697
+ /** Deep copy with every string scrubbed (scrubText); keys in `skip` are copied as they are. */
698
+ export function scrubDeep (value, spec, skip = null) {
493
699
  if (typeof value === 'string') return scrubText(value, spec)
494
- if (Array.isArray(value)) return value.map((v) => scrubDeep(v, spec))
495
- if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, scrubDeep(v, spec)]))
700
+ if (Array.isArray(value)) return value.map((v) => scrubDeep(v, spec, skip))
701
+ if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, skip?.has(k) ? v : scrubDeep(v, spec, skip)]))
496
702
  return value
497
703
  }
498
704
 
705
+ /** Keys of a capture record that hold no page text (paths, hashes, coordinates). */
706
+ export const RECORD_PLAIN_KEYS = new Set(['schema', 'files', 'criteria_hash', 'captured_at', 'boxes', 'rects', 'ignore', 'unstable', 'capture', 'pii', 'node', 'locale', 'viewport', 'view'])
707
+
708
+ /**
709
+ * The stored HTML of a masked page: attributes masked in the page only as values (maskInPage's `attrs`: data-*,
710
+ * content, links) replaced, as the browser serialises them.
711
+ */
712
+ export function maskHtml (html, attrs = []) {
713
+ if (typeof html !== 'string' || !attrs?.length) return html
714
+ const esc = (s, lt) => { let v = s.replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/ /g, '&nbsp;'); if (lt) v = v.replace(/</g, '&lt;').replace(/>/g, '&gt;'); return v }
715
+ let out = html
716
+ for (const [from, to] of [...attrs].sort((a, b) => b[0].length - a[0].length)) {
717
+ for (const lt of [false, true]) out = out.split(`="${esc(from, lt)}"`).join(`="${esc(to, lt)}"`)
718
+ }
719
+ return out
720
+ }
721
+
499
722
  /** The e-mail headers of a captured message with recipients replaced (To and Cc are always personal data). */
500
723
  export function maskMailHeaders (mail, spec) {
501
724
  if (!mail || !spec?.active) return mail
725
+ const taken = takenNow()
726
+ const stand = (kind, v) => { const had = remembered.get(v); if (had != null) return had; taken.add(v); const r = kit.replacement(kind, v, spec, taken); taken.add(r); return r }
502
727
  const one = (addr) => {
503
728
  const a = String(addr || '').trim()
504
729
  if (!a) return a
505
730
  const email = a.match(/<([^>]+)>/)?.[1] || a
506
- const repl = kit.replacement('email', email, spec)
507
- rememberPii([[email, repl], ...(a !== email ? [[a.replace(/<[^>]+>/, '').trim().replace(/^"|"$/g, ''), kit.replacement('name', a.replace(/<[^>]+>/, '').trim(), spec)]] : [])])
731
+ const repl = stand('email', email)
732
+ const name = a !== email ? a.replace(/<[^>]+>/, '').trim().replace(/^"|"$/g, '') : ''
733
+ rememberPii([[email, repl, 'email'], ...(name ? [[name, stand('name', name), 'name']] : [])])
508
734
  return repl
509
735
  }
510
736
  const list = (v) => String(v || '').split(/,(?=(?:[^"]*"[^"]*")*[^"]*$)/).map(one).filter(Boolean).join(', ')
@@ -514,17 +740,29 @@ export function maskMailHeaders (mail, spec) {
514
740
  /** Page argument for maskInPage / maskPainter: the kit travels as source with the call. */
515
741
  export const inPage = (fn, args) => `(${fn})(${JSON.stringify(args)}, (${piiKit})())`
516
742
 
517
- /** Installs (or extends) the overlay on a page and on its next documents; resolves false where it cannot. */
743
+ /**
744
+ * Installs (or extends) the overlay on a page and on its next documents; resolves false where it cannot. An init
745
+ * script cannot be changed, so a new one is added whenever the values to cover have grown: a later document starts
746
+ * with everything masked so far.
747
+ */
518
748
  export async function paintMasks (page, spec) {
519
749
  if (!spec?.active) return false
520
750
  const args = { selectors: spec.selectors, auto: spec.auto, allow: spec.allow, known: knownPii() }
521
751
  try {
522
- if (!PAINTED.has(page)) { PAINTED.add(page); await page.addInitScript(inPage(maskPainter, args)) }
523
- await page.evaluate(inPage(maskPainter, args))
752
+ const script = inPage(maskPainter, args)
753
+ SPECS.set(page, spec)
754
+ if (SCRIPTS.get(page) !== script) { await page.addInitScript(script); SCRIPTS.set(page, script) }
755
+ await page.evaluate(script)
524
756
  return true
525
757
  } catch { return false }
526
758
  }
527
- const PAINTED = new WeakSet()
759
+ const SCRIPTS = new WeakMap()
760
+ const SPECS = new WeakMap()
761
+
762
+ /** After a capture: an overlay drawn on the page learns the values masked since (nothing when none is drawn). */
763
+ export async function updateMasks (page) {
764
+ if (SPECS.has(page)) await paintMasks(page, SPECS.get(page))
765
+ }
528
766
 
529
767
  // --- the guard before upload ---------------------------------------------------------------------------------------
530
768
 
@@ -551,43 +789,122 @@ function elementAt (html, at) {
551
789
  return `<${m[1].toLowerCase()}${id ? `#${id}` : ''}${cls ? `.${cls}` : ''}>`
552
790
  }
553
791
 
554
- /** Personal data in one text: [{ kind, value, where }]. */
792
+ // sample addresses of forms and docs (you@company.com): never anyone's
793
+ const SAMPLE = /^(you|your[._-]?(name|email|mail)?|name|user(name)?|e-?mail|jmeno|jmeno\.prijmeni|vas[._-]?email|vase[._-]?jmeno|john[._-]?doe|jane[._-]?doe)@|@(your)?(company|domain)\.(com|cz)$|@(vase|moje)firma\.cz$/i
794
+ // text in a placeholder (an attribute in evidence, a key in JSON) is a hint, not data
795
+ const HINT_BEFORE = /placeholder\s*[=:]\s*["'“\\]*[^"'\n]{0,80}$/i
796
+ // HTML values that never hold personal data: SVG geometry, styles, placeholders, responsive image lists
797
+ const GEOMETRY = /\s(d|points|viewBox|transform|x|y|x1|x2|y1|y2|cx|cy|r|rx|ry|dx|dy|width|height|offset|stroke-dasharray|stroke-dashoffset|stroke-width|style|placeholder|srcset|sizes|integrity|nonce)\s*=\s*("[^"]*"|'[^']*')/gi
798
+ /** HTML for the guard: values that are never personal data blanked (offsets stay, so elements are still named). */
799
+ function blankHtml (html) {
800
+ return html
801
+ .replace(GEOMETRY, (m, name, val) => m.slice(0, m.length - val.length) + val[0] + ' '.repeat(val.length - 2) + val.at(-1))
802
+ .replace(/(<style\b[^>]*>)([\s\S]*?)(<\/style>)/gi, (m, a, b, c) => a + ' '.repeat(b.length) + c)
803
+ }
804
+
805
+ /** Personal data in one text: [{ kind, value, plain?, where }]. */
555
806
  export function scanText (text, { allow = [], where = null } = {}) {
556
- return kit.find(String(text || ''), { kinds: GUARD_KINDS, allow }).map((r) => ({ kind: r.kind, value: r.value, where: typeof where === 'function' ? where(r.start) : where }))
807
+ const s = String(text || '')
808
+ return kit.find(s, { kinds: GUARD_KINDS, allow })
809
+ .filter((r) => !HINT_BEFORE.test(s.slice(Math.max(0, r.start - 100), r.start)))
810
+ .map((r) => ({ kind: r.kind, value: r.value, ...(r.plain ? { plain: r.plain } : {}), where: typeof where === 'function' ? where(r.start) : where }))
557
811
  }
558
812
 
813
+ const PLAIN_KEYS = new Set(['files', 'criteria_hash', 'boxes', 'rects', 'ignore', 'unstable'])
559
814
  /** Personal data in a JSON value: hits with the field path. */
560
815
  export function scanJson (value, { allow = [], path = '' } = {}) {
561
816
  if (typeof value === 'string') return scanText(value, { allow, where: path || 'value' })
562
817
  if (Array.isArray(value)) return value.flatMap((v, i) => scanJson(v, { allow, path: `${path}[${i}]` }))
563
- if (value && typeof value === 'object') return Object.entries(value).flatMap(([k, v]) => (k === 'files' || k === 'criteria_hash' ? [] : scanJson(v, { allow, path: path ? `${path}.${k}` : k })))
818
+ if (value && typeof value === 'object') return Object.entries(value).flatMap(([k, v]) => (PLAIN_KEYS.has(k) ? [] : scanJson(v, { allow, path: path ? `${path}.${k}` : k })))
564
819
  return []
565
820
  }
566
821
 
822
+ /** A contact as compared: an e-mail in lower case, a phone number as its digits without 00. */
823
+ const contactKey = (v) => { const s = String(v || '').trim().replace(/^(mailto|tel):/i, ''); return s.includes('@') ? s.toLowerCase() : digitsOf(s).replace(/^00/, '') }
824
+ const ORG = /Organi[sz]ation|Corporation|Business|Store|ContactPoint|WebSite|WebPage|Brand|Place|Restaurant|Hotel|NGO|School|College|University|Library|Museum|Shop/i
825
+ const PEOPLE = /^(employees?|founders?|members?|authors?|customers?|persons?|people|contactPersons?|creators?|editors?|reviews?)$/i
826
+ /** The e-mails and phone numbers an organisation publishes about itself in JSON-LD (Organization, ContactPoint). */
827
+ function ldContacts (value, out, org = false) {
828
+ if (Array.isArray(value)) { for (const v of value) ldContacts(v, out, org); return }
829
+ if (!value || typeof value !== 'object') return
830
+ const types = [].concat(value['@type'] || []).map(String)
831
+ const isOrg = types.length ? types.some((t) => ORG.test(t)) && !types.some((t) => /Person|Patient/i.test(t)) : org
832
+ if (isOrg) for (const k of ['email', 'telephone', 'faxNumber']) for (const x of [].concat(value[k] || [])) if (typeof x === 'string') out.add(contactKey(x))
833
+ for (const [k, v] of Object.entries(value)) if (v && typeof v === 'object') ldContacts(v, out, isOrg && !PEOPLE.test(k))
834
+ }
835
+
836
+ /** [start, end] ranges of the page footer (<footer>, role=contentinfo) in an HTML text. */
837
+ function footerRanges (html) {
838
+ const out = []
839
+ for (const m of html.matchAll(/<(footer)\b[^>]*>|<([a-z][\w-]*)\b[^>]*\brole=["']?contentinfo\b[^>]*>/gi)) {
840
+ const tag = (m[1] || m[2]).toLowerCase()
841
+ const re = new RegExp(`<${tag}\\b|</${tag}\\s*>`, 'gi')
842
+ re.lastIndex = m.index + m[0].length
843
+ let depth = 1; let end = html.length
844
+ for (let t = re.exec(html); t; t = re.exec(html)) { if (t[0][1] === '/') { if (!--depth) { end = t.index; break } } else depth++ }
845
+ out.push([m.index, end])
846
+ }
847
+ return out
848
+ }
849
+
850
+ /** The registrable part of a host name (shop.acme.cz → acme.cz, www.acme.co.uk → acme.co.uk). */
851
+ function baseDomain (host) {
852
+ const parts = String(host || '').toLowerCase().replace(/\.$/, '').split('.')
853
+ const n = parts.length >= 3 && /^(co|com|org|net|gov|ac|edu)$/.test(parts.at(-2)) ? 3 : 2
854
+ return parts.slice(-n).join('.')
855
+ }
856
+
857
+ const TEXT_FILES = /\.(json|html?|xml|svg|txt|eml|csv|md|log)$/i
858
+
567
859
  /**
568
860
  * What `kaloko share` would upload, scanned for unmasked e-mails, phone numbers, birth numbers and IBANs:
569
- * the manifest's steps (captures, mail headers, console lines, evidence) and the files that travel with it:
861
+ * the manifest's steps (captures, mail headers, console lines, evidence) and the text files that travel with it:
570
862
  * [{ kind, value, node, locale, viewport, file, where }]. Images and recordings are not read (masks cover them).
863
+ * The site's own contacts are not personal data (site.contacts, on unless pii.site_contacts: false): e-mails on the
864
+ * domain of the environment, e-mails and phone numbers in the page footer, and those of an Organization or
865
+ * ContactPoint in JSON-LD. Sample addresses (you@company.com) and placeholder text never count.
571
866
  */
572
- export async function piiFindings ({ dir, files, allow = [], manifest = null }) {
573
- const out = []
867
+ export async function piiFindings ({ dir, files, allow = [], manifest = null, site = null }) {
868
+ const hits = []
869
+ const contacts = new Set()
870
+ const siteContacts = site?.contacts !== false
871
+ let own = null
872
+ try { own = site?.url ? baseDomain(new URL(site.url).hostname) : null } catch {}
574
873
  for (const n of manifest?.nodes || []) {
575
874
  const { captures, ...rest } = n
576
- for (const h of scanJson(rest, { allow })) out.push({ ...h, node: n.id, file: 'run.json' })
875
+ for (const h of scanJson(rest, { allow })) hits.push({ ...h, node: n.id, file: 'run.json' })
577
876
  for (const [locale, byVp] of Object.entries(captures || {})) {
578
- for (const [viewport, c] of Object.entries(byVp || {})) for (const h of scanJson(c, { allow })) out.push({ ...h, node: n.id, locale, viewport, file: 'run.json' })
877
+ for (const [viewport, c] of Object.entries(byVp || {})) for (const h of scanJson(c, { allow })) hits.push({ ...h, node: n.id, locale, viewport, file: 'run.json' })
579
878
  }
580
879
  }
880
+ if (manifest) ldContacts(manifest, contacts)
581
881
  for (const file of files) {
582
- if (!/\.(json|html?)$/i.test(file)) continue
882
+ if (!TEXT_FILES.test(file)) continue
583
883
  let text
584
884
  try { text = await readFile(join(dir, file), 'utf8') } catch { continue }
585
885
  const at = stepOfPath(file)
586
- let hits
587
- if (/\.json$/i.test(file)) { let data; try { data = JSON.parse(text) } catch { data = text } hits = scanJson(data, { allow }) } else hits = scanText(text, { allow, where: (i) => elementAt(text, i) })
588
- for (const h of hits) out.push({ ...h, ...at, file })
886
+ if (/\.json$/i.test(file)) {
887
+ let data
888
+ try { data = JSON.parse(text) } catch { data = text }
889
+ ldContacts(data, contacts)
890
+ for (const h of scanJson(data, { allow })) hits.push({ ...h, ...at, file })
891
+ } else if (/\.(html?|xml|svg)$/i.test(file)) {
892
+ for (const m of text.matchAll(/<script\b[^>]*type=["']?application\/ld\+json[^>]*>([\s\S]*?)<\/script>/gi)) { try { ldContacts(JSON.parse(m[1]), contacts) } catch {} }
893
+ const footers = footerRanges(text)
894
+ for (const h of scanText(blankHtml(text), { allow, where: (i) => ({ i, el: elementAt(text, i) }) })) {
895
+ if (siteContacts && footers.some(([s, e]) => h.where.i >= s && h.where.i < e)) { contacts.add(contactKey(h.plain ?? h.value)); continue }
896
+ hits.push({ ...h, where: h.where.el, ...at, file })
897
+ }
898
+ } else for (const h of scanText(text, { allow, where: 'text' })) hits.push({ ...h, ...at, file })
589
899
  }
590
- return out
900
+ return hits.filter((h) => {
901
+ const v = String(h.plain ?? h.value).trim()
902
+ if (SAMPLE.test(v)) return false
903
+ if (!siteContacts) return true
904
+ if (contacts.has(contactKey(v))) return false
905
+ if (own && h.kind === 'email' && baseDomain(v.split('@')[1]) === own) return false
906
+ return true
907
+ })
591
908
  }
592
909
 
593
910
  /** The values a project says are not personal data, plus the run's own test mailboxes ({run} = any run). */
@@ -596,11 +913,11 @@ export function guardAllow ({ config = {}, env = {}, scenario = {}, run = null }
596
913
  return [...(config.pii?.allow || []), ...(run?.meta?.pii?.allow || []), ...boxes.filter(Boolean).map((b) => String(b).replace('{run}', '*'))].map(String)
597
914
  }
598
915
 
599
- /** The refusal (or the warning with --allow-pii) for findings: one line per hit, step and element first. */
916
+ /** The refusal (or the warning) for findings: one line per hit, step and element first. */
600
917
  export function formatFindings (findings, max = 12) {
601
918
  const lines = findings.slice(0, max).map((f) => {
602
919
  const where = f.node ? `${f.node}${f.locale ? ` (${f.locale}, ${f.viewport})` : ''}` : f.file
603
- return ` ${where} ${f.where || ''} ${f.kind.replace('_', ' ')} ${hint(f.value)}`.replace(/ {3,}/g, ' ')
920
+ return ` ${where} ${f.where || ''} ${f.kind.replace('_', ' ')} ${hint(f.plain ?? f.value)}`.replace(/ {3,}/g, ' ')
604
921
  })
605
922
  if (findings.length > max) lines.push(` … ${findings.length - max} more`)
606
923
  return lines.join('\n')