@sema-agent/client-core 0.87.0 → 0.89.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +177 -0
  2. package/README.md +20 -15
  3. package/dist/adapter/activeRunSelfHeal.js +4 -2
  4. package/dist/adapter/downstream/eventToSdkMessage.js +17 -2
  5. package/dist/adapter/downstream/wiringManifestView.d.ts +11 -0
  6. package/dist/adapter/downstream/wiringManifestView.js +28 -0
  7. package/dist/controlRouter.d.ts +7 -1
  8. package/dist/controlRouter.js +43 -1
  9. package/dist/disableSkillsWire.d.ts +17 -0
  10. package/dist/disableSkillsWire.js +52 -0
  11. package/dist/displayUntrusted.d.ts +4 -0
  12. package/dist/displayUntrusted.js +1742 -591
  13. package/dist/engineErrorCodes.d.ts +2 -0
  14. package/dist/engineErrorCodes.js +1 -0
  15. package/dist/engineNoticeCodes.d.ts +44 -1
  16. package/dist/engineNoticeCodes.js +211 -39
  17. package/dist/engineWireFor.d.ts +1 -0
  18. package/dist/engineWireFor.js +1 -1
  19. package/dist/gateOutcome.d.ts +4 -0
  20. package/dist/gateOutcome.js +5 -0
  21. package/dist/generated/engineFactTables.d.ts +1 -0
  22. package/dist/generated/engineFactTables.js +3 -0
  23. package/dist/generated/engineNoticeTables.d.ts +3 -0
  24. package/dist/generated/engineNoticeTables.js +110 -0
  25. package/dist/hitl/crashConverged.d.ts +1 -1
  26. package/dist/hitl/crashConverged.js +6 -1
  27. package/dist/hitl/planReviewWire.js +30 -10
  28. package/dist/hitl/toolApprovalWire.d.ts +6 -2
  29. package/dist/hitl/toolApprovalWire.js +30 -0
  30. package/dist/hooksWireCaps.js +80 -12
  31. package/dist/index.d.ts +12 -3
  32. package/dist/index.js +8 -3
  33. package/dist/pluginHooksWire.d.ts +46 -4
  34. package/dist/pluginHooksWire.js +277 -24
  35. package/dist/promptDelivery.d.ts +20 -0
  36. package/dist/promptDelivery.js +153 -0
  37. package/dist/providerRejection.d.ts +6 -0
  38. package/dist/providerRejection.js +52 -0
  39. package/dist/registryConflict.d.ts +3 -0
  40. package/dist/registryConflict.js +44 -0
  41. package/dist/request/taskRequest.d.ts +3 -0
  42. package/dist/request/taskRequest.js +64 -8
  43. package/dist/resumeRefusalCopy.d.ts +7 -0
  44. package/dist/resumeRefusalCopy.js +45 -1
  45. package/dist/runTerminal.d.ts +6 -0
  46. package/dist/runTerminal.js +23 -0
  47. package/dist/seam.d.ts +4 -0
  48. package/dist/seatContract.d.ts +9 -1
  49. package/dist/seatContract.js +10 -0
  50. package/dist/sessionMemoryErase.d.ts +76 -0
  51. package/dist/sessionMemoryErase.js +432 -0
  52. package/dist/subagent/engineSubagentResume.js +2 -1
  53. package/dist/subagent/engineSubagentSteer.d.ts +8 -1
  54. package/dist/subagent/engineSubagentSteer.js +26 -3
  55. package/docs/INTEGRATION-CLIENTS.md +767 -14
  56. package/package.json +3 -3
@@ -26,135 +26,6 @@ function pushEdit(edits, src, start, end, text) {
26
26
  const UNIT_NONE = 0;
27
27
  const UNIT_SEQUENCE = 1;
28
28
  const UNIT_SINGLE = 2;
29
- const NO_UNITS = () => UNIT_NONE;
30
- function applyEdits(text, edits) {
31
- if (edits.length === 0)
32
- return text;
33
- const parts = [];
34
- let cursor = 0;
35
- for (const e of edits) {
36
- parts.push(text.slice(cursor, e.start), e.text);
37
- cursor = e.end;
38
- }
39
- parts.push(text.slice(cursor));
40
- return parts.join('');
41
- }
42
- class PlainTarget {
43
- text;
44
- origin;
45
- trace;
46
- constructor(text, trace = false) {
47
- this.text = text;
48
- this.trace = trace;
49
- }
50
- apply(edits) {
51
- if (this.trace && edits.length > 0)
52
- this.origin = editedOrigin(this.origin, this.text.length, edits);
53
- this.text = applyEdits(this.text, edits);
54
- }
55
- unitAt() {
56
- return UNIT_NONE;
57
- }
58
- }
59
- const traced = (text, origin) => (origin === undefined ? { text } : { text, origin });
60
- function editedOrigin(prev, length, edits) {
61
- let n = length;
62
- for (const e of edits)
63
- n += e.text.length - (e.end - e.start);
64
- const out = new Int32Array(n);
65
- let w = 0;
66
- let cursor = 0;
67
- const copy = (a, b) => {
68
- for (let k = a; k < b; k++)
69
- out[w++] = prev === undefined ? k : (prev[k] ?? -1);
70
- };
71
- for (const e of edits) {
72
- copy(cursor, e.start);
73
- out.fill(-1, w, w + e.text.length);
74
- w += e.text.length;
75
- cursor = e.end;
76
- }
77
- copy(cursor, length);
78
- return out;
79
- }
80
- function composeOrigin(outer, inner) {
81
- if (inner === undefined)
82
- return outer;
83
- if (outer === undefined)
84
- return inner;
85
- const out = new Int32Array(inner.length);
86
- for (let k = 0; k < inner.length; k++) {
87
- const s = inner[k] ?? -1;
88
- out[k] = s < 0 ? -1 : (outer[s] ?? -1);
89
- }
90
- return out;
91
- }
92
- class TrackedTarget {
93
- text;
94
- src;
95
- spans = [];
96
- viewLength;
97
- units;
98
- constructor(text, viewLength, units) {
99
- this.text = text;
100
- this.viewLength = viewLength;
101
- this.units = units;
102
- this.src = new Int32Array(text.length);
103
- for (let k = 0; k < text.length; k++)
104
- this.src[k] = k;
105
- }
106
- unitAt(k) {
107
- const s = this.src[k] ?? -1;
108
- return s >= 0 && this.units !== undefined ? (this.units[s] ?? UNIT_NONE) : UNIT_NONE;
109
- }
110
- rangeStart(k) {
111
- const s = this.src[k] ?? 0;
112
- return s >= 0 ? s : (this.spans[2 * (-1 - s)] ?? 0);
113
- }
114
- rangeEnd(k) {
115
- const s = this.src[k] ?? 0;
116
- return s >= 0 ? s + 1 : (this.spans[2 * (-1 - s) + 1] ?? 0);
117
- }
118
- apply(edits) {
119
- if (edits.length === 0)
120
- return;
121
- let length = this.text.length;
122
- for (const e of edits)
123
- length += e.text.length - (e.end - e.start);
124
- const src = new Int32Array(length);
125
- const parts = [];
126
- let cursor = 0;
127
- let w = 0;
128
- for (const e of edits) {
129
- src.set(this.src.subarray(cursor, e.start), w);
130
- w += e.start - cursor;
131
- parts.push(this.text.slice(cursor, e.start), e.text);
132
- let a;
133
- let b;
134
- if (e.start === e.end) {
135
- a = e.start < this.text.length ? this.rangeStart(e.start) : this.viewLength;
136
- b = a;
137
- }
138
- else {
139
- a = this.rangeStart(e.start);
140
- b = this.rangeEnd(e.start);
141
- for (let k = e.start + 1; k < e.end; k++) {
142
- a = Math.min(a, this.rangeStart(k));
143
- b = Math.max(b, this.rangeEnd(k));
144
- }
145
- }
146
- const id = this.spans.length / 2;
147
- this.spans.push(a, b);
148
- src.fill(-1 - id, w, w + e.text.length);
149
- w += e.text.length;
150
- cursor = e.end;
151
- }
152
- src.set(this.src.subarray(cursor), w);
153
- parts.push(this.text.slice(cursor));
154
- this.text = parts.join('');
155
- this.src = src;
156
- }
157
- }
158
29
  const SECRET_LITERAL_RE = /^(sk-|gh[opus]_|github_pat_|xox[baprse]-|xapp-|bearer\s|basic\s|eyJ[A-Za-z0-9_-]{10,}\.|AKIA[0-9A-Z]{12,}|AIza[0-9A-Za-z_-]{20,}|-----BEGIN[A-Z0-9 ]*PRIVATE KEY)/i;
159
30
  const SCHEME_TOKEN_RE = /(?:^|[^A-Za-z0-9_-])(?:bearer|basic)[ \t]+\S{8,4096}/i;
160
31
  const JWT_RE = /(?:^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{4,2048}/i;
@@ -181,6 +52,147 @@ export function looksLikeKnownSecret(s) {
181
52
  }
182
53
  return false;
183
54
  }
55
+ const KIND_BY_TEXT = new Map([
56
+ [REDACTED_USERINFO, 'userinfo'],
57
+ [REDACTED_QUERY, 'query'],
58
+ [REDACTED_FRAGMENT, 'fragment'],
59
+ [REDACTED_SECRET, 'secret'],
60
+ ]);
61
+ const TEXT_BY_KIND = {
62
+ userinfo: REDACTED_USERINFO,
63
+ query: REDACTED_QUERY,
64
+ fragment: REDACTED_FRAGMENT,
65
+ secret: REDACTED_SECRET,
66
+ };
67
+ const isStructuralKind = (kind) => kind !== 'secret';
68
+ const MARKER_WITH_DOTS = /^[. ]*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ]*)+$/;
69
+ function valueIsWholeMarker(S, start, end, tokenInner = true) {
70
+ const text = S.text;
71
+ let k = start;
72
+ while (k < end && (S.dropped(k) || text[k] === '.' || text[k] === ' '))
73
+ k++;
74
+ let m;
75
+ const retired = text.startsWith(RETIRED_MARKER, k);
76
+ if (retired)
77
+ m = k + RETIRED_MARKER.length;
78
+ else {
79
+ if (!text.startsWith(REDACTION_MARKER_PREFIX, k))
80
+ return false;
81
+ const re = new RegExp(MARKER_ANYWHERE.source, 'y');
82
+ re.lastIndex = k;
83
+ if (!re.test(text))
84
+ return false;
85
+ m = re.lastIndex;
86
+ }
87
+ let j = m;
88
+ while (j < end && S.dropped(j))
89
+ j++;
90
+ if (text[j] === '@' && text.startsWith(REDACTED_USERINFO, k))
91
+ return false;
92
+ if (retired) {
93
+ while (j < end && (S.dropped(j) || S.painted(j) || text[j] === '.' || text[j] === ' '))
94
+ j++;
95
+ return j >= end || credentialWordStartsExactlyAt(text, j, tokenInner) || credentialPairAt(S, j);
96
+ }
97
+ while (j < end && (S.dropped(j) || S.painted(j)))
98
+ j++;
99
+ return j >= end || !isKeyWordChar(text.charCodeAt(j)) || credentialWordStartsExactlyAt(text, j, tokenInner) || credentialPairAt(S, j);
100
+ }
101
+ const CREDENTIAL_NAME = /(?:(?:access|refresh|id|client|api|x[-_]api|session|auth)[-_]?)?(?:token|key|secret|password|authorization|credential)s?|bearer|basic/iy;
102
+ const LABEL_SEPARATOR = /(?:\\*["'])?[\s\u115F\u1160\u3164\uFFA0\u2800]*[:=\uFF1A\uFF1D]/y;
103
+ const SCHEME_SEPARATOR = /[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]/y;
104
+ function credentialPairAt(S, at) {
105
+ const text = S.text;
106
+ const re = new RegExp(CREDENTIAL_NAME.source, 'iy');
107
+ re.lastIndex = at;
108
+ const m = re.exec(text);
109
+ if (m === null)
110
+ return false;
111
+ let k = at + m[0].length;
112
+ if (isKeyWordChar(text.charCodeAt(k)))
113
+ return false;
114
+ const skippable = (p) => S.dropped(p) || S.painted(p) || text[p] === '.' || text[p] === ' ';
115
+ const skipped = k < text.length && skippable(k);
116
+ while (k < text.length && skippable(k))
117
+ k++;
118
+ if (k >= text.length)
119
+ return true;
120
+ const scheme = /^(?:bearer|basic)$/i.test(m[0]);
121
+ if (!(scheme && skipped)) {
122
+ const sep = new RegExp((scheme ? SCHEME_SEPARATOR : LABEL_SEPARATOR).source, 'y');
123
+ sep.lastIndex = k;
124
+ if (!sep.test(text))
125
+ return false;
126
+ k = sep.lastIndex;
127
+ }
128
+ while (k < text.length && (skippable(k) || isRegexSpace(text.charCodeAt(k)) || text[k] === '"' || text[k] === "'" || text[k] === '\\'))
129
+ k++;
130
+ if (k >= text.length || isBareValueStop(text.charAt(k)))
131
+ return true;
132
+ if (text.startsWith(RETIRED_MARKER, k))
133
+ return true;
134
+ const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
135
+ mk.lastIndex = k;
136
+ return mk.test(text);
137
+ }
138
+ const MARKER_LEAD = /^[. ,;:!?']*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ,;:!?']*)+/;
139
+ function nextPairOrNameAt(v, at) {
140
+ const name = new RegExp(CREDENTIAL_NAME.source, 'iy');
141
+ name.lastIndex = at;
142
+ if (name.test(v)) {
143
+ let k = name.lastIndex;
144
+ while (k < v.length && TRAILING_PUNCTUATION.has(v.charAt(k)))
145
+ k++;
146
+ if (k >= v.length)
147
+ return true;
148
+ }
149
+ return credentialWordStartsExactlyAt(v, at);
150
+ }
151
+ function userinfoIsWholeMarker(part) {
152
+ const v = withoutSeparators(part);
153
+ if (v.replace(/^[. ]+/, '').startsWith(`${REDACTED_USERINFO}@`))
154
+ return true;
155
+ const lead = MARKER_LEAD.exec(v);
156
+ if (lead === null)
157
+ return false;
158
+ return lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length);
159
+ }
160
+ function slotIsWholeMarker(part, soft) {
161
+ if (soft !== undefined && markerPunctSoftGlue(part, soft))
162
+ return false;
163
+ const v = withoutSeparators(part);
164
+ let k = 0;
165
+ while (k < v.length && (v[k] === '.' || v[k] === ' '))
166
+ k++;
167
+ if (v.startsWith(RETIRED_MARKER, k)) {
168
+ const lead = MARKER_LEAD.exec(v);
169
+ return lead !== null && (lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length));
170
+ }
171
+ const re = new RegExp(MARKER_ANYWHERE.source, 'y');
172
+ re.lastIndex = k;
173
+ if (!re.test(v))
174
+ return false;
175
+ let j = re.lastIndex;
176
+ while (j < v.length && (v[j] === '.' || v[j] === ' '))
177
+ j++;
178
+ return j >= v.length || !isKeyWordChar(v.charCodeAt(j)) || nextPairOrNameAt(v, j);
179
+ }
180
+ function markerPunctSoftGlue(part, soft) {
181
+ let k = 0;
182
+ while (k < part.length && (part[k] === VIEW_SEPARATOR || part[k] === '.' || part[k] === ' '))
183
+ k++;
184
+ const re = new RegExp(MARKER_ANYWHERE.source, 'y');
185
+ re.lastIndex = k;
186
+ if (!re.test(part))
187
+ return false;
188
+ let j = re.lastIndex;
189
+ while (j < part.length && part[j] !== VIEW_SEPARATOR && !soft(j) && TRAILING_PUNCTUATION.has(part.charAt(j)))
190
+ j++;
191
+ const s0 = j;
192
+ while (j < part.length && (part[j] === VIEW_SEPARATOR || soft(j)))
193
+ j++;
194
+ return j > s0 && j < part.length && isKeyWordChar(part.charCodeAt(j)) && !nextPairOrNameAt(part, j);
195
+ }
184
196
  const TOKEN_STOP = new Set(['"', '<', '>', '`', '|', '\\', '^', '{', '}']);
185
197
  const TRAILING_PUNCTUATION = new Set(['.', ',', ';', ':', '!', '?', "'"]);
186
198
  const VALUE_BOUNDARY_TAIL = /[?#&=;]$/;
@@ -188,6 +200,10 @@ const isWhitespaceOrControl = (ch) => {
188
200
  const c = ch.charCodeAt(0);
189
201
  return c <= 0x20 || c === 0x7f;
190
202
  };
203
+ const endsUrlFragmentAt = (s, at) => {
204
+ const ch = s.charAt(at);
205
+ return ch !== '' && (isWhitespaceOrControl(ch) || TOKEN_STOP.has(ch));
206
+ };
191
207
  const isSchemeChar = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch === '+' || ch === '.' || ch === '-';
192
208
  const isAlpha = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
193
209
  const isHexDigit = (ch) => (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
@@ -226,7 +242,8 @@ function schemelessHostEnd(text, from) {
226
242
  if (d > k + 1)
227
243
  k = d;
228
244
  }
229
- return endsSchemelessHost(text.charCodeAt(k)) ? k : -1;
245
+ const c = text.charCodeAt(k);
246
+ return endsSchemelessHost(c) || c === 0x27 ? k : -1;
230
247
  }
231
248
  function valueCoreForJudging(value) {
232
249
  if (value.length === 0 || isMarker(value))
@@ -240,38 +257,49 @@ function isKnownSecretSegment(value) {
240
257
  const core = valueCoreForJudging(value);
241
258
  return core.length > 0 && looksLikeKnownSecret(core);
242
259
  }
243
- function queryEdits(src, base, query, edits) {
260
+ function queryCandidates(src, base, query, edits, soft) {
244
261
  let pos = base;
245
262
  for (const part of query.split('&')) {
246
263
  if (part.length > 0) {
247
264
  const eq = part.indexOf('=');
248
265
  if (eq === -1) {
249
- if (!isMarker(part))
266
+ const at = pos;
267
+ if (!slotIsWholeMarker(part, soft && ((i) => soft(at + i))))
250
268
  pushEdit(edits, src, pos, pos + part.length, REDACTED_QUERY);
251
269
  }
252
270
  else {
253
271
  const value = part.slice(eq + 1);
254
- if (value.length > 0 && !isMarker(value))
272
+ const at = pos + eq + 1;
273
+ if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
255
274
  pushEdit(edits, src, pos + eq + 1, pos + part.length, REDACTED_QUERY);
256
275
  }
257
276
  }
258
277
  pos += part.length + 1;
259
278
  }
260
279
  }
261
- function pathEdits(src, base, path, opaqueSegments, edits) {
280
+ function pathCandidates(src, base, path, opaqueSegments, edits, soft) {
262
281
  let pos = base;
263
282
  for (const segment of path.split('/')) {
264
283
  const semi = segment.indexOf(';');
265
284
  const head = semi === -1 ? segment : segment.slice(0, semi);
266
- if (opaqueSegments && isKnownSecretSegment(head))
267
- pushEdit(edits, src, pos, pos + head.length, REDACTED_SECRET);
285
+ if (opaqueSegments && isKnownSecretSegment(head)) {
286
+ let a = 0;
287
+ let b = head.length;
288
+ while (a < b && !isKeyWordChar(head.charCodeAt(a)))
289
+ a++;
290
+ while (b > a && !isKeyWordChar(head.charCodeAt(b - 1)))
291
+ b--;
292
+ if (b > a)
293
+ pushEdit(edits, src, pos + a, pos + b, REDACTED_SECRET);
294
+ }
268
295
  if (semi !== -1) {
269
296
  let ppos = pos + semi + 1;
270
297
  for (const part of segment.slice(semi + 1).split(';')) {
271
298
  const eq = part.indexOf('=');
272
299
  if (eq !== -1) {
273
300
  const value = part.slice(eq + 1);
274
- if (value.length > 0 && !isMarker(value))
301
+ const at = ppos + eq + 1;
302
+ if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
275
303
  pushEdit(edits, src, ppos + eq + 1, ppos + part.length, REDACTED_QUERY);
276
304
  }
277
305
  ppos += part.length + 1;
@@ -280,7 +308,7 @@ function pathEdits(src, base, path, opaqueSegments, edits) {
280
308
  pos += segment.length + 1;
281
309
  }
282
310
  }
283
- function urlTokenEditsAt(src, start, end, edits) {
311
+ function urlTokenCandidatesAt(src, start, end, edits, soft) {
284
312
  const token = src.slice(start, end);
285
313
  const schemeEnd = token.indexOf('://') + '://'.length;
286
314
  const scheme = token.slice(0, schemeEnd);
@@ -290,23 +318,23 @@ function urlTokenEditsAt(src, start, end, edits) {
290
318
  const tailStart = schemeEnd + authority.length;
291
319
  const tail = token.slice(tailStart);
292
320
  const at = authority.lastIndexOf('@');
293
- if (at !== -1)
321
+ if (at !== -1 && !userinfoIsWholeMarker(authority.slice(0, at)))
294
322
  pushEdit(edits, src, start + schemeEnd, start + schemeEnd + at, REDACTED_USERINFO);
295
323
  const hash = tail.indexOf('#');
296
324
  const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
297
325
  const question = beforeHash.indexOf('?');
298
326
  const path = question === -1 ? beforeHash : beforeHash.slice(0, question);
299
- pathEdits(src, start + tailStart, path, !/^file:/i.test(scheme), edits);
327
+ pathCandidates(src, start + tailStart, path, !/^file:/i.test(scheme), edits, soft);
300
328
  if (question !== -1)
301
- queryEdits(src, start + tailStart + question + 1, beforeHash.slice(question + 1), edits);
329
+ queryCandidates(src, start + tailStart + question + 1, beforeHash.slice(question + 1), edits, soft);
302
330
  if (hash !== -1) {
303
331
  const rawFragment = tail.slice(hash + 1);
304
- if (rawFragment.length > 0 && !isMarker(rawFragment))
332
+ const at = start + tailStart + hash + 1;
333
+ if (rawFragment.length > 0 && !slotIsWholeMarker(rawFragment, soft && ((i) => soft(at + i))))
305
334
  pushEdit(edits, src, start + tailStart + hash + 1, end, REDACTED_FRAGMENT);
306
335
  }
307
336
  }
308
- function urlTokenEdits(text) {
309
- const edits = [];
337
+ function urlCandidates(text, out, regions, unitPos = () => false, soft) {
310
338
  let mark = text.indexOf('://');
311
339
  while (mark !== -1) {
312
340
  let start = mark;
@@ -319,20 +347,47 @@ function urlTokenEdits(text) {
319
347
  continue;
320
348
  }
321
349
  let end = mark + 3;
350
+ let seenPathSep = false;
351
+ let atAhead = -2;
352
+ const colons = new AuthorityColons(text, mark + 3);
353
+ let firstVisible = -1;
354
+ const adjacent = (at) => {
355
+ if (firstVisible === -1) {
356
+ firstVisible = mark + 3;
357
+ while (firstVisible < text.length && unitPos(firstVisible))
358
+ firstVisible++;
359
+ }
360
+ return at <= firstVisible;
361
+ };
322
362
  while (end < text.length) {
323
363
  const ch = text.charAt(end);
324
- if (isWhitespaceOrControl(ch) || TOKEN_STOP.has(ch))
364
+ if (isWhitespaceOrControl(ch))
365
+ break;
366
+ if (TOKEN_STOP.has(ch)) {
367
+ if (!seenPathSep) {
368
+ if (atAhead === -2)
369
+ atAhead = atInAuthority(text, end);
370
+ if (atAhead > end && (adjacent(end) || colons.colonBefore(end))) {
371
+ end++;
372
+ continue;
373
+ }
374
+ }
325
375
  break;
376
+ }
326
377
  const c = ch.charCodeAt(0);
327
378
  if (ch === VIEW_SEPARATOR || isUnicodeBlank(c)) {
328
379
  let p = end;
329
380
  while (p < text.length && (text[p] === VIEW_SEPARATOR || isUnicodeBlank(text.charCodeAt(p))))
330
381
  p++;
331
- if (credentialWordStartsNear(text, p))
382
+ if (nextPairBeyondFragmentAt(text, p))
332
383
  break;
333
384
  end = p;
334
385
  continue;
335
386
  }
387
+ if ((ch === ',' || ch === ';') && nextPairBeyondFragmentAt(text, end + 1))
388
+ break;
389
+ if (ch === '/' || ch === '?' || ch === '#')
390
+ seenPathSep = true;
336
391
  end++;
337
392
  }
338
393
  while (end > mark + 3 && text.charAt(end - 1) === VIEW_SEPARATOR)
@@ -342,10 +397,67 @@ function urlTokenEdits(text) {
342
397
  runStart--;
343
398
  if (runStart < end && !VALUE_BOUNDARY_TAIL.test(text.slice(start, runStart)))
344
399
  end = runStart;
345
- urlTokenEditsAt(text, start, end, edits);
400
+ const tokenEdits = [];
401
+ urlTokenCandidatesAt(text, start, end, tokenEdits, soft);
402
+ editsToCandidates(tokenEdits, 'url', out, end);
403
+ const token = text.slice(start, end);
404
+ const schemeEnd = token.indexOf('://') + 3;
405
+ const rest = token.slice(schemeEnd);
406
+ const authorityEnd = rest.search(/[/?#]/);
407
+ const tailStart = schemeEnd + (authorityEnd === -1 ? rest.length : authorityEnd);
408
+ const tail = token.slice(tailStart);
409
+ const hash = tail.indexOf('#');
410
+ const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
411
+ const question = beforeHash.indexOf('?');
412
+ if (question !== -1)
413
+ regions.push([start + tailStart + question + 1, start + tailStart + beforeHash.length, 'query']);
414
+ if (hash !== -1 && hash + 1 < tail.length)
415
+ regions.push([start + tailStart + hash + 1, end, 'fragment']);
346
416
  mark = text.indexOf('://', end > mark + 3 ? end : mark + 3);
347
417
  }
348
- return edits;
418
+ }
419
+ function atInAuthority(text, from) {
420
+ for (let p = from; p < text.length; p++) {
421
+ const ch = text.charAt(p);
422
+ if (isWhitespaceOrControl(ch) || ch === '/' || ch === '?' || ch === '#')
423
+ return -1;
424
+ if (ch === '@')
425
+ return p;
426
+ }
427
+ return -1;
428
+ }
429
+ class AuthorityColons {
430
+ text;
431
+ scanned;
432
+ colons = 0;
433
+ bracketEnd = -1;
434
+ constructor(text, from) {
435
+ this.text = text;
436
+ this.scanned = from;
437
+ if (text.charCodeAt(from) === 0x5b) {
438
+ let k = from + 1;
439
+ while (k < text.length && isIpv6Char(text.charCodeAt(k)))
440
+ k++;
441
+ if (text.charCodeAt(k) === 0x5d)
442
+ this.bracketEnd = k;
443
+ }
444
+ }
445
+ colonBefore(to) {
446
+ for (; this.scanned < to; this.scanned++) {
447
+ const c = this.text.charCodeAt(this.scanned);
448
+ if (c === 0xab) {
449
+ const re = new RegExp(MARKER_ANYWHERE.source, 'y');
450
+ re.lastIndex = this.scanned;
451
+ if (re.test(this.text) && re.lastIndex <= to) {
452
+ this.scanned = re.lastIndex - 1;
453
+ continue;
454
+ }
455
+ }
456
+ if (c === 0x3a && this.scanned > this.bracketEnd)
457
+ this.colons++;
458
+ }
459
+ return this.colons > 0;
460
+ }
349
461
  }
350
462
  const MARKER_ANYWHERE = /\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb/g;
351
463
  function markerSpans(text) {
@@ -357,18 +469,123 @@ function markerSpans(text) {
357
469
  spans.push(hit.index, hit.index + hit[0].length);
358
470
  return spans;
359
471
  }
360
- function schemelessUserinfoEdits(text, unitAt = NO_UNITS) {
361
- const edits = [];
472
+ const USERNAME_DELIMITERS = new Set(['"', "'", '`', '(', ')', '[', ']', '{', '}', '<', '>', '|', '\\', '^', ',', ';', '=']);
473
+ const PASSWORD_FORBIDDEN = new Set(['"', '`', '<', '>']);
474
+ const forbiddenCloser = (ch) => (ch === '<' ? '>' : ch === '"' || ch === '`' ? ch : '');
475
+ const isBareUserChar = (c) => (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) ||
476
+ c === 0x2e || c === 0x5f || c === 0x2d || c === 0x7e || c === 0x25 || c === 0x2b;
477
+ function userinfoCandidates(S, out, words = true) {
478
+ const text = S.text;
479
+ const unitAt = (k) => S.unitAt(k);
362
480
  const markers = markerSpans(text);
363
- let m = 0;
481
+ const markerStartAt = new Map();
482
+ const markerStartByEnd = markers.length === 0 ? null : new Map();
483
+ const markerEndAt = markers.length === 0 ? null : new Int32Array(text.length).fill(-1);
484
+ for (let m = 0; m + 1 < markers.length; m += 2) {
485
+ const a = markers[m] ?? 0;
486
+ const b = markers[m + 1] ?? 0;
487
+ markerStartAt.set(a, b);
488
+ markerStartByEnd?.set(b, a);
489
+ if (markerEndAt !== null)
490
+ for (let k = a; k < b; k++)
491
+ markerEndAt[k] = b;
492
+ }
493
+ let markerFrom = 0;
494
+ const inMarker = (k) => (markerEndAt === null ? -1 : (markerEndAt[k] ?? -1));
495
+ let forbiddenNext = null;
496
+ const nextForbidden = (k) => {
497
+ if (forbiddenNext === null) {
498
+ const n = text.length;
499
+ const t = new Int32Array(n + 1).fill(n);
500
+ for (let x = n - 1; x >= 0; x--)
501
+ t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x)) ? x : (t[x + 1] ?? n);
502
+ forbiddenNext = t;
503
+ }
504
+ return forbiddenNext[k] ?? text.length;
505
+ };
506
+ let forbiddenPrev = null;
507
+ const prevForbidden = (k) => {
508
+ if (forbiddenPrev === null) {
509
+ const n = text.length;
510
+ const t = new Int32Array(n + 1).fill(-1);
511
+ for (let x = 1; x <= n; x++)
512
+ t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x - 1)) ? x - 1 : (t[x - 1] ?? -1);
513
+ forbiddenPrev = t;
514
+ }
515
+ return forbiddenPrev[k] ?? -1;
516
+ };
517
+ let bareStopNext = null;
518
+ const nextBareStop = (k) => {
519
+ if (bareStopNext === null) {
520
+ const n = text.length;
521
+ const t = new Int32Array(n + 1).fill(n);
522
+ for (let x = n - 1; x >= 0; x--)
523
+ t[x] = isBareValueStop(text.charAt(x)) ? x : (t[x + 1] ?? n);
524
+ bareStopNext = t;
525
+ }
526
+ return bareStopNext[k] ?? text.length;
527
+ };
528
+ const labelValueReaches = (from, at) => {
529
+ let i = from;
530
+ while (i < at && text.charCodeAt(i) === 0x5c)
531
+ i++;
532
+ const q = text[i];
533
+ if (q === '"' || q === "'") {
534
+ if (i > from) {
535
+ const close = text.indexOf('\\' + q, i + 1);
536
+ return close === -1 || close >= at;
537
+ }
538
+ for (let j = i + 1; j < at; j++) {
539
+ if (text[j] === '\\') {
540
+ j++;
541
+ continue;
542
+ }
543
+ if (text[j] === q)
544
+ return false;
545
+ }
546
+ return true;
547
+ }
548
+ return nextBareStop(from) >= at;
549
+ };
364
550
  let floor = 0;
365
- for (let at = text.indexOf('@'), prevAt = -1; at !== -1; prevAt = at, at = text.indexOf('@', at + 1)) {
551
+ for (let at = text.indexOf('@'); at !== -1; at = text.indexOf('@', at + 1)) {
366
552
  let hostEnd = schemelessHostEnd(text, at + 1);
367
553
  const hidden = hostEnd === -1;
368
554
  if (hidden)
369
555
  hostEnd = hiddenHostEnd(text, at + 1, unitAt);
370
556
  if (hostEnd === -1)
371
557
  continue;
558
+ let shaped = null;
559
+ const hostShaped = () => {
560
+ if (shaped === null) {
561
+ const c = text.charCodeAt(hostEnd);
562
+ shaped = !(c === 0x3c || c === 0x3e || c === 0x7c || c === 0x5c || c === 0x5e || c === 0x7b || c === 0x7d);
563
+ if (!shaped)
564
+ for (let k = at + 1; k < hostEnd && !shaped; k++) {
565
+ const h = text.charCodeAt(k);
566
+ if (h === 0x2e || h === 0x3a || h === 0x5b)
567
+ shaped = true;
568
+ }
569
+ }
570
+ return shaped;
571
+ };
572
+ const keptDespiteForbidden = (userStart, colon) => {
573
+ if (labelEndsBefore(text, userStart, floor, '\'"`'))
574
+ return 'label';
575
+ if (hidden || userStart >= colon)
576
+ return null;
577
+ if (labelEndsAt(text, colon + 1) || schemeWordEndsAt(text, colon + 1))
578
+ return words && labelValueReaches(colon + 1, at) ? null : 'label';
579
+ for (let k = userStart; k < colon; k++)
580
+ if (!isBareUserChar(text.charCodeAt(k)))
581
+ return null;
582
+ const q = prevForbidden(at);
583
+ const closer = forbiddenCloser(text.charAt(q));
584
+ if (closer !== '' && q < at - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
585
+ return null;
586
+ return hostShaped() ? 'bare' : null;
587
+ };
588
+ const refused = [];
372
589
  let runStart = at;
373
590
  while (runStart > floor && (!breaksUserinfo(text.charCodeAt(runStart - 1)) || (hidden && unitAt(runStart - 1) !== UNIT_NONE)))
374
591
  runStart--;
@@ -377,67 +594,135 @@ function schemelessUserinfoEdits(text, unitAt = NO_UNITS) {
377
594
  runStart--;
378
595
  }
379
596
  let found = -1;
380
- let colon = -1;
381
- let s = runStart;
382
- while (s + 2 < at) {
383
- if (colon < s) {
384
- colon = s;
385
- while (colon < at && text.charCodeAt(colon) !== 0x3a)
386
- colon++;
597
+ let lo = runStart;
598
+ let keyClose = -2;
599
+ let wholeMarkerAt = -1;
600
+ for (let mk = markerFrom; mk + 1 < markers.length && (markers[mk] ?? 0) < at; mk += 2) {
601
+ const a = markers[mk] ?? 0;
602
+ const b = markers[mk + 1] ?? 0;
603
+ markerFrom = mk;
604
+ if (a < runStart)
605
+ continue;
606
+ let p = b;
607
+ while (p < at && (text[p] === VIEW_SEPARATOR || text[p] === '.'))
608
+ p++;
609
+ if ((p >= at || !isKeyWordChar(text.charCodeAt(p))) && labelWashedBefore(text, a, runStart))
610
+ wholeMarkerAt = a;
611
+ }
612
+ let markerTail = at;
613
+ for (;;) {
614
+ while (markerTail > runStart && (text[markerTail - 1] === VIEW_SEPARATOR || text[markerTail - 1] === '.' || text[markerTail - 1] === ' '))
615
+ markerTail--;
616
+ const ms = markerTail > runStart ? (markerStartByEnd?.get(markerTail) ?? -1) : -1;
617
+ if (ms === -1 || ms < runStart)
618
+ break;
619
+ markerTail = ms;
620
+ }
621
+ let sFrom = runStart;
622
+ for (let colon = runStart; colon < at && found === -1; colon++) {
623
+ const me = inMarker(colon);
624
+ if (me !== -1) {
625
+ colon = me - 1;
626
+ continue;
627
+ }
628
+ const ch = text.charAt(colon);
629
+ if (ch === '[') {
630
+ let k = colon + 1;
631
+ while (k < at && isIpv6Char(text.charCodeAt(k)))
632
+ k++;
633
+ if (k < at && text.charCodeAt(k) === 0x5d && k > colon + 1) {
634
+ colon = k;
635
+ lo = k + 1;
636
+ continue;
637
+ }
638
+ }
639
+ if (USERNAME_DELIMITERS.has(ch)) {
640
+ lo = colon + 1;
641
+ continue;
387
642
  }
643
+ if (text.charCodeAt(colon) !== 0x3a)
644
+ continue;
388
645
  if (colon + 1 >= at)
389
646
  break;
390
- if (colon === s) {
391
- s++;
647
+ const leftOfColon = colon > 0 ? text.charAt(colon - 1) : '';
648
+ if ((leftOfColon === "'" || leftOfColon === '"' || leftOfColon === '`') && colon > 1 && isKeyWordChar(text.charCodeAt(colon - 2))) {
649
+ lo = colon + 1;
650
+ keyClose = colon;
392
651
  continue;
393
652
  }
394
- while (m < markers.length && (markers[m + 1] ?? 0) <= colon)
395
- m += 2;
396
- if (m < markers.length && (markers[m] ?? 0) <= colon) {
397
- s = colon + 1;
653
+ const forbidden = nextForbidden(colon + 1) < at;
654
+ if (!forbidden && text[colon + 1] === "'" && text[hostEnd] === "'") {
655
+ lo = colon + 1;
398
656
  continue;
399
657
  }
400
- const prev = s === 0 ? -1 : text.charCodeAt(s - 1);
401
- const midPair = prev >= 0xd800 && prev <= 0xdbff && (text.charCodeAt(s) & 0xfc00) === 0xdc00;
402
- if ((prev === -1 || !blocksUserinfoStart(prev)) && !midPair) {
403
- found = s;
404
- break;
658
+ if (wholeMarkerAt > colon && colon + 1 < markerTail) {
659
+ lo = colon + 1;
660
+ continue;
661
+ }
662
+ if (wholeMarkerAt === colon + 1 && markerTail === colon + 1 && text.startsWith(REDACTED_USERINFO, colon + 1)) {
663
+ lo = colon + 1;
664
+ continue;
665
+ }
666
+ let userStart = -1;
667
+ if (lo === colon)
668
+ userStart = colon;
669
+ else {
670
+ for (let s = Math.max(lo, sFrom); s < colon; s++) {
671
+ if (markerStartAt.has(s)) {
672
+ userStart = s;
673
+ break;
674
+ }
675
+ const prev = s === 0 ? -1 : text.charCodeAt(s - 1);
676
+ const midPair = prev >= 0xd800 && prev <= 0xdbff && (text.charCodeAt(s) & 0xfc00) === 0xdc00;
677
+ if ((prev === -1 || !blocksUserinfoStart(prev) || s === keyClose + 1) && !midPair) {
678
+ userStart = s;
679
+ break;
680
+ }
681
+ const inner = inMarker(s);
682
+ if (inner !== -1)
683
+ s = inner - 1;
684
+ }
685
+ if (userStart === -1)
686
+ sFrom = colon;
687
+ }
688
+ const kept = userStart === -1 || !forbidden ? null : keptDespiteForbidden(userStart, colon);
689
+ if (userStart === -1 || (forbidden && kept === null)) {
690
+ if (forbidden) {
691
+ lo = colon + 1;
692
+ if (userStart !== -1)
693
+ refused.push(colon, userStart);
694
+ }
695
+ continue;
696
+ }
697
+ found = userStart;
698
+ if (kept === 'bare') {
699
+ for (let r = 0; r + 1 < refused.length; r += 2) {
700
+ if (nextForbidden((refused[r] ?? 0) + 1) >= userStart) {
701
+ found = refused[r + 1] ?? userStart;
702
+ break;
703
+ }
704
+ }
405
705
  }
406
- s++;
407
706
  }
408
707
  if (found === -1)
409
708
  continue;
410
- if (hidden && credentialWordAttached(text, prevAt + 1, found, at, unitAt))
411
- continue;
412
- pushEdit(edits, text, found, at, REDACTED_USERINFO);
709
+ out.push({ a: found, b: at, kind: 'userinfo', cls: hidden ? 'userinfo-hidden' : 'structural', te: hostEnd });
413
710
  floor = hostEnd;
414
711
  }
415
- return edits;
416
712
  }
417
- function credentialWordAttached(text, lo, found, at, unitAt) {
418
- if (credentialWordEndsFrom(text.slice(lo, at), found - lo))
419
- return true;
420
- const kept = [];
421
- let joinedFound = -1;
422
- for (let k = lo; k < at; k++) {
423
- if (k === found)
424
- joinedFound = kept.length;
425
- if (unitAt(k) === UNIT_NONE)
426
- kept.push(text.charAt(k));
427
- }
428
- return joinedFound !== -1 && credentialWordEndsFrom(kept.join(''), joinedFound);
429
- }
430
- function credentialWordEndsFrom(seg, from) {
431
- for (const word of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
432
- const re = new RegExp(word.source, 'gi');
433
- for (let hit = re.exec(seg); hit !== null; hit = re.exec(seg)) {
434
- if (hit.index + hit[0].length >= from)
435
- return true;
436
- if (hit[0].length === 0)
437
- re.lastIndex++;
438
- }
439
- }
440
- return false;
713
+ function labelWashedBefore(text, a, floor) {
714
+ return labelEndsBefore(text, a, floor, "'");
715
+ }
716
+ function labelEndsBefore(text, a, floor, quotes) {
717
+ let e = a;
718
+ while (e > floor && text[e - 1] === VIEW_SEPARATOR)
719
+ e--;
720
+ if (e > floor && quotes.includes(text[e - 1] ?? '\u0000'))
721
+ e--;
722
+ const w = text.slice(Math.max(floor, e - 96), e);
723
+ if (w === '')
724
+ return false;
725
+ return new RegExp(`(?:${SECRET_LABELLED_WORD.source})$`, 'i').test(w) || new RegExp(`(?:${SECRET_SCHEME_WORD.source})$`, 'i').test(w);
441
726
  }
442
727
  function unitRunEnd(text, from, unitAt) {
443
728
  let k = from;
@@ -459,16 +744,6 @@ function hiddenHostEnd(text, from, unitAt) {
459
744
  const c = text.charCodeAt(k);
460
745
  return endsSchemelessHost(c) || c === 0x40 ? k : -1;
461
746
  }
462
- const MAX_URL_WASH_PASSES = 4;
463
- function washUrlNets(target) {
464
- for (let pass = 0; pass < MAX_URL_WASH_PASSES; pass++) {
465
- target.apply(urlTokenEdits(target.text));
466
- const handedOff = schemelessUserinfoEdits(target.text, (k) => target.unitAt(k));
467
- target.apply(handedOff);
468
- if (handedOff.length === 0)
469
- return;
470
- }
471
- }
472
747
  const NAME_BLANK_RANGES = [
473
748
  [0x00a0, 0x00a0],
474
749
  [0x1680, 0x1680],
@@ -484,7 +759,8 @@ const NAME_BLANK_RANGES = [
484
759
  const isNameBlank = (cp) => NAME_BLANK_RANGES.some(([a, b]) => cp >= a && cp <= b);
485
760
  const isFoldedBlank = (c) => isNameBlank(c) && isRegexSpace(c);
486
761
  const isUnicodeBlank = (c) => (c > 0x7f && isRegexSpace(c) && c !== 0xfeff) || isNameBlank(c);
487
- const SECRET_SCHEME_WORD = /(?:\b|(?<=\\[A-Za-z"']))(bearer|basic)[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]+/gi;
762
+ const SECRET_SCHEME_WORD = /(?:\b|(?<=\\[A-Za-z"']))(bearer|basic|token)[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]+/gi;
763
+ const schemeWordIsInner = (word, tokenInner) => tokenInner || word.toLowerCase() !== 'token';
488
764
  const SECRET_LABELLED_WORD = /(?:\b|(?<=_)|(?<=\\[A-Za-z"']))((?:access[-_ ]?|refresh[-_ ]?|id[-_ ]?|client[-_ ]?|api[-_ ]?|x[-_]api[-_ ]?|session[-_ ]?|auth[-_ ]?)?(?:token|key|secret|password|authorization|credential)s?)(?:\\*["'])?[\s\u115F\u1160\u3164\uFFA0\u2800]*[:=\uFF1A\uFF1D][\s\u115F\u1160\u3164\uFFA0\u2800]*/gi;
489
765
  const DIAGNOSTIC_VALUE_WORDS = new Set([
490
766
  'missing', 'required', 'invalid', 'expired', 'unauthorized', 'unauthenticated', 'authentication',
@@ -502,6 +778,37 @@ const SCHEME_FOLLOWER_WORDS = new Set([
502
778
  'setup', 'plan', 'tier', 'info', 'information', 'support', 'level', 'string', 'config', 'configuration', 'settings',
503
779
  'version', 'check', 'validation', 'error', 'errors', 'failed', 'failure', 'credentials', 'user', 'username',
504
780
  ]);
781
+ const TOKEN_FOLLOWER_WORDS = new Set([
782
+ ...SCHEME_FOLLOWER_WORDS,
783
+ 'count', 'counts', 'counting', 'counter', 'counters', 'limit', 'limits', 'budget', 'budgets', 'window', 'windows', 'estimate', 'estimates',
784
+ 'estimation', 'estimator', 'length', 'size', 'cost', 'costs', 'price', 'pricing', 'rate', 'rates', 'quota', 'quotas', 'cap', 'caps', 'ceiling',
785
+ 'threshold', 'total', 'totals', 'accounting', 'allocation', 'consumption', 'overflow', 'savings', 'reduction', 'spend', 'metrics', 'stats',
786
+ 'boundary', 'boundaries', 'stream', 'streaming', 'position', 'positions', 'index', 'offset', 'probability', 'probabilities', 'logprob',
787
+ 'logprobs', 'sequence', 'encoding', 'tokenizer', 'vocabulary', 'id', 'ids', 'maximum', 'minimum', 'max', 'min', 'ratio', 'table', 'line',
788
+ 'lines', 'unit', 'units', 'indices',
789
+ 'refresh', 'refreshed', 'refreshes', 'refreshing', 'exchange', 'exchanged', 'endpoint', 'endpoints', 'request', 'requests', 'response',
790
+ 'responses', 'store', 'stored', 'storage', 'cache', 'cached', 'caching', 'file', 'files', 'path', 'dir', 'directory', 'source', 'provider',
791
+ 'providers', 'supplier', 'helper', 'command', 'env', 'environment', 'variable', 'lifetime', 'ttl', 'expiry', 'expiration', 'expires', 'expiring',
792
+ 'rotation', 'rotated', 'revocation', 'revoke', 'introspection', 'verification', 'verified', 'generation', 'generated', 'issuer', 'issued',
793
+ 'audience', 'claims', 'scope', 'scopes', 'permission', 'permissions', 'owner', 'name', 'label', 'hash', 'signature', 'payload', 'body',
794
+ 'cookie', 'kind', 'types', 'data', 'metadata', 'envelope', 'binding', 'bound', 'renewal', 'minted', 'acquired', 'fetch', 'fetched', 'parsing',
795
+ 'parse', 'parsed', 'persisted', 'url', 'urls', 'client', 'argument', 'flag', 'option', 'field', 'governance', 'target',
796
+ 'has', 'have', 'had', 'will', 'would', 'could', 'may', 'might', 'does', 'did', 'do', 'doesn', 'isn', 'wasn', 'hasn', 'didn', 'won', 'couldn',
797
+ 'wouldn', 'shouldn', 'aren', 'weren', 'haven', 'been', 'being', 'needs', 'need', 'appears', 'seems', 'looks', 'lives', 'stays', 'remains',
798
+ 'starts', 'stopped', 'closed', 'arrived', 'surfaced', 'vanished', 'belongs', 'carries', 'contains', 'lacks', 'matches', 'match', 'changed',
799
+ 'created', 'deleted', 'removed', 'updated', 'exceeded', 'exceeds', 'exhausted', 'reached', 'returned', 'returns', 'received', 'provided',
800
+ 'supplied', 'found', 'used', 'using', 'saved', 'loaded', 'accepted', 'validated', 'passed', 'sent', 'fails', 'works', 'already', 'still',
801
+ 'also', 'only', 'never', 'now', 'here', 'there', 'then', 'than', 'so', 'but', 'as', 'if', 'when', 'while', 'until', 'before', 'after',
802
+ 'since', 'because', 'via', 'into', 'over', 'under', 'within', 'without', 'per', 'each', 'every', 'any', 'all', 'its', 'which', 'whose',
803
+ 'please', 'run', 'try', 'retry', 'see', 'use', 'visit', 'reauthenticate', 'reconnect',
804
+ ]);
805
+ function schemeFollowerReleased(scheme, value, afterPathSlash) {
806
+ const word = value.replace(/[.,;:!?)]+$/, '').toLowerCase();
807
+ const isToken = scheme.toLowerCase() === 'token';
808
+ if (isToken && afterPathSlash && /^\d{3}$/.test(word))
809
+ return true;
810
+ return (isToken ? TOKEN_FOLLOWER_WORDS : SCHEME_FOLLOWER_WORDS).has(word);
811
+ }
505
812
  const AUTH_SCHEME_WORDS = new Set([
506
813
  'bearer', 'basic', 'digest', 'token', 'bot', 'ssws', 'api-key', 'apikey', 'key', 'hmac', 'oauth', 'oauth2', 'jwt',
507
814
  'negotiate', 'ntlm', 'hawk', 'mac', 'dsn', 'splunk', 'sapisidhash', 'googlelogin', 'aws', 'client-id', 'pat',
@@ -511,9 +818,11 @@ const BARE_VALUE_STOP_CHARS = new Set([',', ';', '"', "'", '\\', '(', ')', '[',
511
818
  const isBareValueStop = (ch) => ch <= ' ' || BARE_VALUE_STOP_CHARS.has(ch);
512
819
  class InnerWordIndex {
513
820
  text;
821
+ tokenInner;
514
822
  probes;
515
- constructor(text) {
823
+ constructor(text, tokenInner = true) {
516
824
  this.text = text;
825
+ this.tokenInner = tokenInner;
517
826
  this.probes = [
518
827
  { re: new RegExp(SECRET_LABELLED_WORD.source, SECRET_LABELLED_WORD.flags), scheme: false, from: -1, hit: null },
519
828
  { re: new RegExp(SECRET_SCHEME_WORD.source, SECRET_SCHEME_WORD.flags), scheme: true, from: -1, hit: null },
@@ -525,7 +834,11 @@ class InnerWordIndex {
525
834
  const reusable = p.from !== -1 && q >= p.from && (p.hit === null || q <= p.hit.start);
526
835
  if (!reusable) {
527
836
  p.re.lastIndex = q;
528
- const m = p.re.exec(this.text);
837
+ let m = p.re.exec(this.text);
838
+ while (m !== null && p.scheme && !schemeWordIsInner(m[1] ?? '', this.tokenInner)) {
839
+ p.re.lastIndex = m.index + 1;
840
+ m = p.re.exec(this.text);
841
+ }
529
842
  p.from = q;
530
843
  if (m === null)
531
844
  p.hit = null;
@@ -610,7 +923,7 @@ function scanSecretValue(text, from, idx) {
610
923
  let p = j;
611
924
  while (p < text.length && text[p] === VIEW_SEPARATOR)
612
925
  p++;
613
- if (credentialWordStartsNear(text, p))
926
+ if (credentialWordStartsNear(text, p, idx?.tokenInner ?? true))
614
927
  break;
615
928
  j = p;
616
929
  continue;
@@ -660,6 +973,22 @@ function isTokenCountLabel(text, at, label) {
660
973
  const lastSep = Math.max(head.lastIndexOf('_'), head.lastIndexOf('-'));
661
974
  return TOKEN_COUNT_HEAD_WORDS.has(head.slice(lastSep + 1));
662
975
  }
976
+ function isParserDiagnosticToken(text, at, label) {
977
+ if (label.toLowerCase() !== 'token' || at < 1)
978
+ return false;
979
+ if (text.charAt(at - 1) !== ' ')
980
+ return false;
981
+ const head = 'unexpected';
982
+ const start = at - 1 - head.length;
983
+ if (start < 0 || text.slice(start, at - 1).toLowerCase() !== head)
984
+ return false;
985
+ if (start === 0)
986
+ return true;
987
+ const before = text.charAt(start - 1);
988
+ if (!/[A-Za-z0-9_]/.test(before))
989
+ return true;
990
+ return start >= 2 && text.charAt(start - 2) === '\\' && /[A-Za-z"']/.test(before);
991
+ }
663
992
  const BACKTICKED_CREDENTIAL_NAME = /^\x60[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\x60[.,;:!?)]*$/;
664
993
  const CREDENTIAL_NAME_SEGMENTS = new Set([
665
994
  'KEY', 'KEYS', 'TOKEN', 'TOKENS', 'SECRET', 'SECRETS', 'PASSWORD', 'PASSWD', 'PWD', 'CREDENTIAL', 'CREDENTIALS',
@@ -686,29 +1015,58 @@ function labelStartsAt(text, at) {
686
1015
  re.lastIndex = at;
687
1016
  return re.test(text);
688
1017
  }
689
- function credentialWordStartsAt(text, at) {
1018
+ function credentialWordStartsAt(text, at, tokenInner = true) {
690
1019
  if (at >= text.length)
691
1020
  return false;
692
- if (credentialWordStartsExactlyAt(text, at))
1021
+ if (credentialWordStartsExactlyAt(text, at, tokenInner))
693
1022
  return true;
694
1023
  let k = at;
695
1024
  while (k < text.length && text.charCodeAt(k) === 0x5c)
696
1025
  k++;
697
1026
  if (text[k] === '"' || text[k] === "'")
698
1027
  k++;
699
- return k > at && credentialWordStartsExactlyAt(text, k);
1028
+ return k > at && credentialWordStartsExactlyAt(text, k, tokenInner);
700
1029
  }
701
- function credentialWordStartsExactlyAt(text, at) {
1030
+ function credentialWordStartsExactlyAt(text, at, tokenInner = true) {
702
1031
  if (at >= text.length)
703
1032
  return false;
704
1033
  if (labelStartsAt(text, at))
705
1034
  return true;
706
1035
  const re = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
707
1036
  re.lastIndex = at;
708
- return re.test(text);
709
- }
710
- function credentialWordStartsNear(text, at) {
711
- return credentialWordStartsAt(text, at) || credentialWordStartsAt(text, at + 1);
1037
+ const m = re.exec(text);
1038
+ return m !== null && schemeWordIsInner(m[1] ?? '', tokenInner);
1039
+ }
1040
+ function credentialWordStartsNear(text, at, tokenInner = true) {
1041
+ return credentialWordStartsAt(text, at, tokenInner) || credentialWordStartsAt(text, at + 1, tokenInner);
1042
+ }
1043
+ function innerWordMatchAt(text, at, tokenInner) {
1044
+ const lab = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
1045
+ lab.lastIndex = at;
1046
+ const m = lab.exec(text);
1047
+ if (m !== null)
1048
+ return m;
1049
+ const sch = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
1050
+ sch.lastIndex = at;
1051
+ const s = sch.exec(text);
1052
+ return s !== null && schemeWordIsInner(s[1] ?? '', tokenInner) ? s : null;
1053
+ }
1054
+ function nextPairBeyondFragmentAt(text, at) {
1055
+ for (const p of [at, at + 1]) {
1056
+ let k = p;
1057
+ while (k < text.length && text.charCodeAt(k) === 0x5c)
1058
+ k++;
1059
+ if (text[k] === '"' || text[k] === "'")
1060
+ k++;
1061
+ for (const src of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
1062
+ const re = new RegExp(src.source, 'iy');
1063
+ re.lastIndex = k;
1064
+ const m = re.exec(text);
1065
+ if (m !== null && endsUrlFragmentAt(m[0], m[0].length - 1))
1066
+ return true;
1067
+ }
1068
+ }
1069
+ return false;
712
1070
  }
713
1071
  function percentDecodeForMatching(text) {
714
1072
  const chars = [];
@@ -773,7 +1131,7 @@ function hostTokenIsLabel(afterUserinfo, hostEnd) {
773
1131
  }
774
1132
  return false;
775
1133
  }
776
- function exemptAddressEdits(src, from, to, edits) {
1134
+ function exemptAddressEdits(src, from, to, edits, rawPart = (a, b) => src.slice(a, b)) {
777
1135
  const value = src.slice(from, to);
778
1136
  const q = value.search(/[?#]/);
779
1137
  const prefix = q === -1 ? value : value.slice(0, q);
@@ -790,11 +1148,48 @@ function exemptAddressEdits(src, from, to, edits) {
790
1148
  const pathStart = scheme.length + (at === -1 ? cut : at + 1 + (cut - userinfoLen));
791
1149
  const pathEnd = q === -1 ? value.length : q;
792
1150
  const pathCut = pathCutAfterFirstLabel(value.slice(pathStart, pathEnd));
793
- if (pathCut !== -1)
1151
+ if (pathCut !== -1 && !onlyQuotesAndMarkers(rawPart(from + pathStart + pathCut, from + pathEnd)))
794
1152
  pushEdit(edits, src, from + pathStart + pathCut, from + pathEnd, REDACTED_SECRET);
795
1153
  if (q !== -1)
796
1154
  pushEdit(edits, src, from + q + 1, to, value.charAt(q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY);
797
1155
  }
1156
+ function addressJunkStart(text, from, to) {
1157
+ let authEnd = from;
1158
+ while (authEnd < to && text[authEnd] !== '/' && text[authEnd] !== '?' && text[authEnd] !== '#')
1159
+ authEnd++;
1160
+ const at = text.lastIndexOf('@', authEnd - 1);
1161
+ if (at < from)
1162
+ return -1;
1163
+ let k = at + 1;
1164
+ if (text[k] === '[') {
1165
+ const close = text.indexOf(']', k);
1166
+ if (close === -1 || close >= to)
1167
+ return -1;
1168
+ k = close + 1;
1169
+ }
1170
+ else {
1171
+ while (k < to) {
1172
+ const c = text.charCodeAt(k);
1173
+ if (!(isHostChar(c) || (c >= 0xa1 && c !== 0xab && c !== 0xbb && !isRegexSpace(c))))
1174
+ break;
1175
+ k++;
1176
+ }
1177
+ }
1178
+ if (text.charCodeAt(k) === 0x3a) {
1179
+ let d = k + 1;
1180
+ while (d < to && text.charCodeAt(d) >= 0x30 && text.charCodeAt(d) <= 0x39)
1181
+ d++;
1182
+ if (d > k + 1)
1183
+ k = d;
1184
+ }
1185
+ if (k >= to || text[k] === '/' || text[k] === '?' || text[k] === '#')
1186
+ return -1;
1187
+ return k;
1188
+ }
1189
+ function onlyQuotesAndMarkers(part) {
1190
+ const core = part.replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '').split(RETIRED_MARKER).join('');
1191
+ return /^["'\\]*$/.test(core);
1192
+ }
798
1193
  function isAddressShaped(value, raw = value) {
799
1194
  return value.startsWith(`${REDACTED_USERINFO}@`) || ADDRESS_SCHEME_RE.test(raw);
800
1195
  }
@@ -866,13 +1261,15 @@ function addressExtentEnd(text, from) {
866
1261
  }
867
1262
  return end;
868
1263
  }
869
- function trailingLabelStart(text, from, end) {
1264
+ function trailingLabelStart(text, from, end, tokenInner = true) {
870
1265
  const window = text.slice(from, Math.min(text.length, end + 12));
871
1266
  const segLen = end - from;
872
1267
  let best = -1;
873
1268
  for (const src of [SECRET_SCHEME_WORD, SECRET_LABELLED_WORD]) {
874
1269
  const re = new RegExp(src.source, src.flags);
875
1270
  for (let hit = re.exec(window); hit !== null && hit.index < segLen; hit = re.exec(window)) {
1271
+ if (src === SECRET_SCHEME_WORD && !schemeWordIsInner(hit[1] ?? '', tokenInner))
1272
+ continue;
876
1273
  if (hit.index + hit[0].length >= segLen && (best === -1 || hit.index < best))
877
1274
  best = hit.index;
878
1275
  }
@@ -881,37 +1278,476 @@ function trailingLabelStart(text, from, end) {
881
1278
  }
882
1279
  const withoutSeparators = (s) => (s.includes(VIEW_SEPARATOR) ? s.split(VIEW_SEPARATOR).join('') : s);
883
1280
  const isHorizontalSpace = (ch) => ch === ' ' || ch === '\t' || ch === '\u00a0' || ch === VIEW_SEPARATOR;
884
- function labelEdits(text, re, literalBlanks = false, unitAt = NO_UNITS) {
885
- const edits = [];
886
- const schemePass = re === SECRET_SCHEME_WORD;
887
- const idx = new InnerWordIndex(text);
1281
+ function rawUserinfoAddressAt(text, at, end, gapAfter, stops) {
1282
+ const k = Math.min(end, stops.user[at] ?? text.length);
1283
+ if (k === at || text[k] !== ':')
1284
+ return false;
1285
+ const p = Math.min(end, stops.password[k + 1] ?? text.length);
1286
+ if (p <= k + 1 || text[p] !== '@')
1287
+ return false;
1288
+ if (gapAfter(p))
1289
+ return false;
1290
+ return schemelessHostEnd(text, p + 1) !== -1;
1291
+ }
1292
+ function userinfoStops(text) {
1293
+ const n = text.length;
1294
+ const user = new Int32Array(n + 1).fill(n);
1295
+ const password = new Int32Array(n + 1).fill(n);
1296
+ const brk = new Int32Array(n + 1).fill(n);
1297
+ for (let k = n - 1; k >= 0; k--) {
1298
+ const c = text.charCodeAt(k);
1299
+ const br = breaksUserinfo(c);
1300
+ user[k] = br || c === 0x3a || USERNAME_DELIMITERS.has(text.charAt(k)) ? k : (user[k + 1] ?? n);
1301
+ brk[k] = br && text[k] !== VIEW_SEPARATOR ? k : (brk[k + 1] ?? n);
1302
+ password[k] = (br && text[k] !== VIEW_SEPARATOR) || PASSWORD_FORBIDDEN.has(text.charAt(k)) ? k : (password[k + 1] ?? n);
1303
+ }
1304
+ return { user, password, brk };
1305
+ }
1306
+ function forbiddenUserinfoAt(text, at, end, gapAfter, stops, unitAt) {
1307
+ const k = Math.min(end, stops.user[at] ?? text.length);
1308
+ if (k >= end || text[k] !== ':')
1309
+ return -1;
1310
+ for (let u = at; u < k; u++)
1311
+ if (unitAt(u) !== UNIT_NONE || text[u] === VIEW_SEPARATOR)
1312
+ return -1;
1313
+ const p = Math.min(end, stops.password[k + 1] ?? text.length);
1314
+ if (p >= end || !PASSWORD_FORBIDDEN.has(text.charAt(p)))
1315
+ return -1;
1316
+ const a = Math.min(end, stops.brk[k + 1] ?? text.length);
1317
+ if (a >= end || text[a] !== '@' || gapAfter(a))
1318
+ return -1;
1319
+ return schemelessHostEnd(text, a + 1) === -1 ? -1 : a;
1320
+ }
1321
+ function isTokenCountAt(text, at) {
1322
+ const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
1323
+ re.lastIndex = at;
1324
+ const m = re.exec(text);
1325
+ return m !== null && isTokenCountLabel(text, at, m[1] ?? '');
1326
+ }
1327
+ function innerWordHasValue(text, at, idx) {
1328
+ const m = innerWordMatchAt(text, at, idx.tokenInner);
1329
+ return m !== null && scanSecretValue(text, at + m[0].length, idx) !== null;
1330
+ }
1331
+ function quoteSwallowedBefore(S, at) {
1332
+ let k = at - 1;
1333
+ while (k >= 0 && S.dropped(k))
1334
+ k--;
1335
+ const lo = k >= 0 ? S.rawEnd(k) : 0;
1336
+ const hi = at < S.text.length ? S.rawStart(at) : S.raw.length;
1337
+ if (hi <= lo)
1338
+ return '';
1339
+ const gap = S.raw.slice(lo, hi);
1340
+ const dq = gap.indexOf('"');
1341
+ const sq = gap.indexOf("'");
1342
+ if (dq === -1 && sq === -1)
1343
+ return '';
1344
+ return dq === -1 ? "'" : sq === -1 ? '"' : dq < sq ? '"' : "'";
1345
+ }
1346
+ function labelEndsAt(text, end, atLeast = false) {
1347
+ let k = end;
1348
+ while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
1349
+ k--;
1350
+ if (k === 0)
1351
+ return false;
1352
+ const sep = text.charAt(k - 1);
1353
+ if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
1354
+ return false;
1355
+ k--;
1356
+ while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
1357
+ k--;
1358
+ if (k > 0 && (text[k - 1] === '"' || text[k - 1] === "'"))
1359
+ k--;
1360
+ while (k > 0 && text.charCodeAt(k - 1) === 0x5c)
1361
+ k--;
1362
+ let w = k;
1363
+ while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
1364
+ w--;
1365
+ const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
1366
+ for (let s = w; s < k; s++) {
1367
+ re.lastIndex = s;
1368
+ const m = re.exec(text);
1369
+ if (m !== null && (atLeast ? s + m[0].length >= end : s + m[0].length === end) && !isTokenCountLabel(text, s, m[1] ?? ''))
1370
+ return true;
1371
+ }
1372
+ return false;
1373
+ }
1374
+ function schemeWordEndsAt(text, end) {
1375
+ const sep = text.charAt(end - 1);
1376
+ if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
1377
+ return false;
1378
+ let w = end - 1;
1379
+ while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
1380
+ w--;
1381
+ const re = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
1382
+ for (let s = w; s < end - 1; s++) {
1383
+ re.lastIndex = s;
1384
+ const m = re.exec(text);
1385
+ if (m !== null && s + m[0].length === end)
1386
+ return true;
1387
+ }
1388
+ return false;
1389
+ }
1390
+ function quotedExtensionEnd(text, close, cache) {
1391
+ const escaped = text.charCodeAt(close) === 0x5c;
1392
+ const q = escaped ? text.charAt(close + 1) : text.charAt(close);
1393
+ if (q !== '"' && q !== "'")
1394
+ return close;
1395
+ const token = escaped ? `\\${q}` : q;
1396
+ const after = (at) => at + token.length;
1397
+ let memo = cache.get(token);
1398
+ if (memo === undefined) {
1399
+ memo = new Map();
1400
+ cache.set(token, memo);
1401
+ }
1402
+ const done = memo;
1403
+ const step = (cur) => {
1404
+ let n = after(cur);
1405
+ while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
1406
+ n++;
1407
+ const sepNext = text[n] === ':' || text[n] === '=' || text[n] === '\uFF1A' || text[n] === '\uFF1D';
1408
+ if (sepNext && labelEndsAt(text, n + 1, true) && !labelEndsAt(text, cur)) {
1409
+ const next = text.indexOf(token, after(cur));
1410
+ return next === -1 ? { kind: 'open', inner: false } : { kind: 'next', at: next };
1411
+ }
1412
+ if (labelEndsAt(text, cur)) {
1413
+ const innerClose = text.indexOf(token, after(cur));
1414
+ if (innerClose === -1)
1415
+ return { kind: 'open', inner: true };
1416
+ const next = text.indexOf(token, after(innerClose));
1417
+ return next === -1 ? { kind: 'open', inner: true } : { kind: 'next', at: next };
1418
+ }
1419
+ return { kind: 'end' };
1420
+ };
1421
+ const first = step(close);
1422
+ if (first.kind === 'end')
1423
+ return close;
1424
+ if (first.kind === 'open')
1425
+ return first.inner ? close : text.length;
1426
+ const path = [];
1427
+ let cur = first.at;
1428
+ let result;
1429
+ for (;;) {
1430
+ const hit = done.get(cur);
1431
+ if (hit !== undefined) {
1432
+ result = hit;
1433
+ break;
1434
+ }
1435
+ path.push(cur);
1436
+ const s = step(cur);
1437
+ if (s.kind === 'end') {
1438
+ result = cur;
1439
+ break;
1440
+ }
1441
+ if (s.kind === 'open') {
1442
+ result = text.length;
1443
+ break;
1444
+ }
1445
+ cur = s.at;
1446
+ }
1447
+ for (const p of path)
1448
+ done.set(p, result);
1449
+ return result;
1450
+ }
1451
+ function extensionMaskStart(text, close) {
1452
+ let n = close + (text.charCodeAt(close) === 0x5c ? 2 : 1);
1453
+ while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
1454
+ n++;
1455
+ const sep = text[n];
1456
+ if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
1457
+ return close;
1458
+ if (!labelEndsAt(text, n + 1, true) || labelEndsAt(text, close))
1459
+ return close;
1460
+ n++;
1461
+ while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
1462
+ n++;
1463
+ return n;
1464
+ }
1465
+ function quotedAddressEnd(S, from, to) {
1466
+ const text = S.text;
1467
+ for (let k = from; k < to; k++) {
1468
+ const c = text.charCodeAt(k);
1469
+ if ((isRegexSpace(c) && c !== 0xfeff) || S.unitAt(k) !== UNIT_NONE)
1470
+ return k;
1471
+ if (k > from && S.spaceGapBefore(k))
1472
+ return k;
1473
+ }
1474
+ return to;
1475
+ }
1476
+ function labelCandidates(S, pass, regions, out) {
1477
+ const text = S.text;
1478
+ const unitAt = (k) => S.unitAt(k);
1479
+ const re = pass === 'label' ? SECRET_LABELLED_WORD : SECRET_SCHEME_WORD;
1480
+ const schemePass = pass !== 'label';
1481
+ const tokenPass = pass === 'token';
1482
+ const idx = new InnerWordIndex(text, tokenPass);
1483
+ const extensionCache = new Map();
1484
+ let stops;
1485
+ const push = (start, end, kindText, te = -1, cls) => {
1486
+ let a = start;
1487
+ let b = end;
1488
+ while (a < b && text[a] === VIEW_SEPARATOR)
1489
+ a++;
1490
+ while (b > a && text[b - 1] === VIEW_SEPARATOR)
1491
+ b--;
1492
+ if (a >= b)
1493
+ return;
1494
+ if (text.slice(a, b) === kindText)
1495
+ return;
1496
+ const kind = KIND_BY_TEXT.get(kindText);
1497
+ if (kind === undefined)
1498
+ return;
1499
+ out.push({ a, b, kind, cls: cls ?? (kindText === REDACTED_SECRET ? 'word' : 'structural'), te });
1500
+ };
1501
+ const pushEdits = (edits, te) => {
1502
+ for (const e of edits) {
1503
+ const part = text.slice(e.start, e.end);
1504
+ if (e.text === REDACTED_USERINFO ? userinfoIsWholeMarker(part) : e.text !== REDACTED_SECRET && slotIsWholeMarker(part))
1505
+ continue;
1506
+ push(e.start, e.end, e.text, e.text === REDACTED_SECRET ? -1 : te);
1507
+ }
1508
+ };
1509
+ let markerCache = -2;
1510
+ const nextMarkerFrom = (p) => {
1511
+ if (markerCache === -1 || markerCache >= p)
1512
+ return markerCache;
1513
+ markerCache = text.indexOf(REDACTION_MARKER_PREFIX, p);
1514
+ return markerCache;
1515
+ };
1516
+ const maskGaps = (r0, end) => {
1517
+ let p = r0;
1518
+ while (p < end) {
1519
+ while (p < end && (isRegexSpace(text.charCodeAt(p)) || unitAt(p) !== UNIT_NONE))
1520
+ p++;
1521
+ if (p >= end)
1522
+ return;
1523
+ let inner = idx.next(p);
1524
+ while (inner !== null && inner.start < end && isTokenCountAt(text, inner.start))
1525
+ inner = idx.next(inner.start + 1);
1526
+ const hasInner = inner !== null && inner.start < end;
1527
+ let stop = hasInner && inner !== null ? inner.start : end;
1528
+ const head = withoutSeparators(text.slice(p, Math.min(stop, p + 80)));
1529
+ const addressEnd = ADDRESS_SCHEME_RE.test(head) || head.startsWith(`${REDACTED_USERINFO}@`) ? Math.min(stop, quotedAddressEnd(S, p, end)) : -1;
1530
+ if (addressEnd !== -1 && addressEnd < stop) {
1531
+ p = addressEnd;
1532
+ continue;
1533
+ }
1534
+ if (addressEnd === -1) {
1535
+ let marker = null;
1536
+ for (let at = nextMarkerFrom(p); at !== -1 && at < stop && marker === null; at = nextMarkerFrom(at + 1)) {
1537
+ const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
1538
+ mk.lastIndex = at;
1539
+ marker = mk.exec(text);
1540
+ }
1541
+ const markerHit = marker !== null && marker.index < stop;
1542
+ if (markerHit && marker !== null)
1543
+ stop = marker.index;
1544
+ let cut = stop;
1545
+ while (cut > p && (isCutDelimiter(text.charAt(cut - 1)) || isRegexSpace(text.charCodeAt(cut - 1))))
1546
+ cut--;
1547
+ if (cut > p) {
1548
+ const rest = withoutSeparators(text.slice(p, cut));
1549
+ if (rest.trim() !== '' && !valueIsWholeMarker(S, p, cut, idx.tokenInner) && !isDiagnosticValue(rest))
1550
+ push(p, cut, REDACTED_SECRET);
1551
+ }
1552
+ if (markerHit && marker !== null) {
1553
+ p = marker.index + marker[0].length;
1554
+ continue;
1555
+ }
1556
+ }
1557
+ if (!hasInner || inner === null)
1558
+ return;
1559
+ const m = innerWordMatchAt(text, inner.start, idx.tokenInner);
1560
+ if (m === null)
1561
+ return;
1562
+ const after = inner.start + m[0].length;
1563
+ const iv = scanSecretValue(text, after, idx);
1564
+ let next = iv === null ? after : iv.end;
1565
+ if (next < end && (text[next] === '"' || text[next] === "'"))
1566
+ next++;
1567
+ if (next <= p)
1568
+ return;
1569
+ p = next;
1570
+ }
1571
+ };
1572
+ const pushInnerRemainder = (from) => {
1573
+ let open = from;
1574
+ while (text.charCodeAt(open) === 0x5c)
1575
+ open++;
1576
+ let q = text.charAt(open);
1577
+ let at = open + 1;
1578
+ let outerEnd;
1579
+ if (q === '"' || q === "'") {
1580
+ const outer = scanSecretValue(text, from);
1581
+ if (outer === null)
1582
+ return;
1583
+ outerEnd = outer.end;
1584
+ }
1585
+ else {
1586
+ q = quoteSwallowedBefore(S, from);
1587
+ if (q === '')
1588
+ return;
1589
+ at = from;
1590
+ const close = text.indexOf(q, from);
1591
+ outerEnd = close === -1 ? text.length : close;
1592
+ }
1593
+ const m = innerWordMatchAt(text, at, idx.tokenInner);
1594
+ if (m === null)
1595
+ return;
1596
+ const inner = { start: at };
1597
+ const iv = scanSecretValue(text, inner.start + m[0].length, idx);
1598
+ if (iv === null)
1599
+ return;
1600
+ let r0 = iv.end;
1601
+ if (r0 < text.length && (text[r0] === '"' || text[r0] === "'") && text[r0] !== q)
1602
+ r0++;
1603
+ maskGaps(r0, outerEnd);
1604
+ };
1605
+ const regionKind = regions.length === 0 ? null : new Uint8Array(text.length);
1606
+ if (regionKind !== null)
1607
+ for (const r of regions)
1608
+ for (let k = r[0]; k < r[1] && k < text.length; k++)
1609
+ regionKind[k] = r[2] === 'query' ? 1 : 2;
1610
+ const inQuery = (k) => regionKind !== null && k < text.length && regionKind[k] === 1;
1611
+ const forbiddenTailAt = (p) => {
1612
+ if (!PASSWORD_FORBIDDEN.has(text.charAt(p)))
1613
+ return -1;
1614
+ const st = (stops ??= userinfoStops(text));
1615
+ const a = st.brk[p] ?? text.length;
1616
+ if (a >= text.length || text[a] !== '@' || S.gapAfter(a))
1617
+ return -1;
1618
+ const inner = idx.next(p);
1619
+ if (inner !== null && inner.start < a)
1620
+ return -1;
1621
+ const hostEnd = schemelessHostEnd(text, a + 1);
1622
+ if (hostEnd === -1)
1623
+ return -1;
1624
+ let q = -1;
1625
+ for (let k = a - 1; k >= p && q === -1; k--)
1626
+ if (PASSWORD_FORBIDDEN.has(text.charAt(k)))
1627
+ q = k;
1628
+ const closer = forbiddenCloser(text.charAt(q));
1629
+ if (closer !== '' && q < a - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
1630
+ return -1;
1631
+ return a;
1632
+ };
888
1633
  re.lastIndex = 0;
889
1634
  for (let hit = re.exec(text); hit !== null; hit = re.exec(text)) {
890
1635
  const label = hit[1] ?? '';
1636
+ if (schemePass && (label.toLowerCase() === 'token') !== tokenPass)
1637
+ continue;
891
1638
  if (isTokenCountLabel(text, hit.index, label))
892
1639
  continue;
1640
+ if (schemePass && isParserDiagnosticToken(text, hit.index, label))
1641
+ continue;
1642
+ if (schemePass && labelStartsAt(text, hit.index))
1643
+ continue;
893
1644
  const from = hit.index + hit[0].length;
894
- if (credentialWordStartsAt(text, from)) {
1645
+ if (credentialWordStartsAt(text, from, idx.tokenInner)) {
1646
+ pushInnerRemainder(from);
895
1647
  re.lastIndex = from;
896
1648
  continue;
897
1649
  }
898
- const v = scanSecretValue(text, from, idx);
899
- if (v === null)
900
- continue;
901
- if (text.startsWith('-----BEGIN ', v.start)) {
902
- re.lastIndex = v.end;
1650
+ let v0s = scanSecretValue(text, from, idx);
1651
+ if (v0s === null) {
1652
+ let k = from;
1653
+ while (k < text.length && text[k] === VIEW_SEPARATOR)
1654
+ k++;
1655
+ const a = inQuery(k) ? -1 : forbiddenTailAt(k);
1656
+ if (a === -1)
1657
+ continue;
1658
+ v0s = { start: k, end: a };
1659
+ }
1660
+ const v0 = { start: v0s.start, end: v0s.end };
1661
+ let open = from;
1662
+ while (text.charCodeAt(open) === 0x5c)
1663
+ open++;
1664
+ let quoted = text[open] === '"' || text[open] === "'";
1665
+ if (!quoted) {
1666
+ const q = quoteSwallowedBefore(S, v0.start);
1667
+ if (q !== '') {
1668
+ quoted = true;
1669
+ let j = v0.start;
1670
+ while (j < text.length && text[j] !== q)
1671
+ j += text[j] === '\\' ? 2 : 1;
1672
+ v0.end = Math.min(j, text.length);
1673
+ }
1674
+ }
1675
+ let keptSepEnd = -1;
1676
+ if (quoted) {
1677
+ let close = v0.end;
1678
+ while (close < text.length && text[close] === VIEW_SEPARATOR)
1679
+ close++;
1680
+ const ext = close < text.length ? quotedExtensionEnd(text, close, extensionCache) : close;
1681
+ if (ext > close) {
1682
+ const from = extensionMaskStart(text, close);
1683
+ if (from > close)
1684
+ keptSepEnd = from;
1685
+ push(from, ext, REDACTED_SECRET);
1686
+ }
1687
+ }
1688
+ const v = { start: v0.start, end: v0.end };
1689
+ if (quoted) {
1690
+ while (v.start < v.end && unitAt(v.start) !== UNIT_NONE)
1691
+ v.start++;
1692
+ while (v.end > v.start && unitAt(v.end - 1) !== UNIT_NONE)
1693
+ v.end--;
1694
+ if (v.start >= v.end) {
1695
+ re.lastIndex = from;
1696
+ continue;
1697
+ }
1698
+ }
1699
+ if (valueIsWholeMarker(S, v.start, v.end, idx.tokenInner)) {
1700
+ re.lastIndex = quoted ? from : v.end;
903
1701
  continue;
904
1702
  }
905
- if (literalBlanks) {
906
- if (literalBlankLeadJudgedAsBlank(text, v, label, schemePass)) {
907
- re.lastIndex = v.end;
1703
+ if (!quoted && inQuery(v.start)) {
1704
+ for (let k = v.start; k < v.end; k++) {
1705
+ const ch = text[k];
1706
+ if (ch === '&' || ch === '#') {
1707
+ v.end = k;
1708
+ break;
1709
+ }
1710
+ }
1711
+ if (v.end <= v.start) {
1712
+ re.lastIndex = v0.end;
908
1713
  continue;
909
1714
  }
910
1715
  }
1716
+ let forbiddenAt = -1;
1717
+ if (!quoted && !inQuery(v.start)) {
1718
+ forbiddenAt = forbiddenUserinfoAt(text, v.start, text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
1719
+ if (forbiddenAt !== -1)
1720
+ v.end = Math.max(v.end, addressExtentEnd(text, forbiddenAt));
1721
+ }
1722
+ else if (quoted) {
1723
+ forbiddenAt = forbiddenUserinfoAt(text, v.start, v.end, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
1724
+ }
1725
+ if (!quoted && forbiddenAt === -1 && !inQuery(v.start)) {
1726
+ const a = forbiddenTailAt(v.end);
1727
+ if (a !== -1)
1728
+ v.end = a;
1729
+ }
1730
+ if (!quoted) {
1731
+ const trailing = trailingLabelStart(text, v.start, v.end, idx.tokenInner);
1732
+ if (trailing > 0 && innerWordHasValue(text, v.start + trailing, idx)) {
1733
+ let e = v.start + trailing;
1734
+ while (e > v.start && (isCutDelimiter(text.charAt(e - 1)) || text[e - 1] === VIEW_SEPARATOR))
1735
+ e--;
1736
+ if (e > v.start)
1737
+ v.end = e;
1738
+ }
1739
+ }
1740
+ if (pemBeginAt(S, v.start)) {
1741
+ re.lastIndex = v.end;
1742
+ continue;
1743
+ }
911
1744
  const rawValue = text.slice(v.start, v.end);
912
1745
  const value = withoutSeparators(rawValue);
913
- const quoted = v.start > 0 && (text[v.start - 1] === '"' || text[v.start - 1] === "'");
914
- if (schemePass && !quoted && SCHEME_FOLLOWER_WORDS.has(value.replace(/[.,;:!?)]+$/, '').toLowerCase())) {
1746
+ if (MARKER_WITH_DOTS.test(value)) {
1747
+ re.lastIndex = quoted ? from : v.end;
1748
+ continue;
1749
+ }
1750
+ if (schemePass && !quoted && schemeFollowerReleased(label, value, text.charCodeAt(hit.index - 1) === 0x2f)) {
915
1751
  re.lastIndex = v.end;
916
1752
  continue;
917
1753
  }
@@ -927,75 +1763,69 @@ function labelEdits(text, re, literalBlanks = false, unitAt = NO_UNITS) {
927
1763
  re.lastIndex = v.end;
928
1764
  continue;
929
1765
  }
930
- const cred = j > v.end && !labelStartsAt(text, j) ? scanSecretValue(text, j, idx) : null;
1766
+ let cred = j > v.end && !labelStartsAt(text, j) ? scanSecretValue(text, j, idx) : null;
1767
+ if (j > v.end && !inQuery(j)) {
1768
+ const a = forbiddenTailAt(cred === null ? j : cred.end);
1769
+ if (a !== -1)
1770
+ cred = { start: cred === null ? j : cred.start, end: a };
1771
+ }
931
1772
  if (cred !== null) {
932
1773
  const credValue = withoutSeparators(text.slice(cred.start, cred.end));
933
1774
  const keep = isMarker(credValue) || DIAGNOSTIC_VALUE_WORDS.has(credValue.replace(/[.,;:!?)\]}\x27\x22]+$/, '').toLowerCase());
934
1775
  if (!keep)
935
- pushEdit(edits, text, cred.start, cred.end, REDACTED_SECRET);
1776
+ push(cred.start, cred.end, REDACTED_SECRET);
936
1777
  re.lastIndex = cred.end;
937
1778
  continue;
938
1779
  }
939
1780
  re.lastIndex = v.end;
940
1781
  continue;
941
1782
  }
942
- if (isDiagnosticValue(value, rawValue)) {
943
- if (isAddressShaped(value, rawValue)) {
944
- let end = quoted ? v.end : addressExtentEnd(text, v.start);
945
- const trailing = trailingLabelStart(text, v.start, end);
1783
+ const forbiddenAddress = forbiddenAt !== -1 && text[v.start] !== ':';
1784
+ const rawAddress = forbiddenAddress || rawUserinfoAddressAt(text, v.start, quoted ? v.end : text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text));
1785
+ if (isDiagnosticValue(value, rawValue) || rawAddress) {
1786
+ if (isAddressShaped(value, rawValue) || rawAddress) {
1787
+ const extentFrom = forbiddenAddress ? forbiddenAt : v.start;
1788
+ let end = quoted ? quotedAddressEnd(S, extentFrom, v.end) : addressExtentEnd(text, extentFrom);
1789
+ const trailing = trailingLabelStart(text, v.start, end, idx.tokenInner);
946
1790
  if (trailing !== -1)
947
1791
  end = v.start + trailing;
948
1792
  if (gluedToSingleUnit(v.start, hit.index, unitAt) || userinfoHostHidden(text, v.start, unitAt)) {
949
1793
  const cut = trailing !== -1 ? end : Math.max(end, v.end);
950
- const keep = ADDRESS_SCHEME_RE.exec(text.slice(v.start, v.start + 20))?.[0].length ?? 0;
1794
+ const schemeHit = ADDRESS_SCHEME_RE.exec(text.slice(v.start, v.start + 20))?.[0] ?? '';
1795
+ const keep = schemeHit !== '' && schemeHit.length - 3 <= 12 ? schemeHit.length : 0;
951
1796
  const q = text.slice(v.start, cut).search(/[?#]/);
952
1797
  const bodyEnd = q === -1 ? cut : v.start + q;
953
1798
  if (withoutSeparators(text.slice(v.start + keep, bodyEnd)) !== '')
954
- pushEdit(edits, text, v.start + keep, bodyEnd, REDACTED_SECRET);
1799
+ push(v.start + keep, bodyEnd, REDACTED_SECRET, -1, 'glued');
955
1800
  if (q !== -1 && v.start + q + 1 < cut)
956
- pushEdit(edits, text, v.start + q + 1, cut, text.charAt(v.start + q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY);
957
- re.lastIndex = cut;
1801
+ push(v.start + q + 1, cut, text.charAt(v.start + q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY, cut, 'glued');
1802
+ re.lastIndex = quoted ? from : cut;
958
1803
  continue;
959
1804
  }
960
- exemptAddressEdits(text, v.start, end, edits);
961
- re.lastIndex = end;
1805
+ const edits = [];
1806
+ exemptAddressEdits(text, v.start, end, edits, (a, b) => (b > a ? S.raw.slice(S.rawStart(a), S.rawEnd(b - 1)) : ''));
1807
+ pushEdits(edits, end);
1808
+ if (rawAddress) {
1809
+ const limit = quoted ? end : Math.min(end, v.end);
1810
+ const junk = addressJunkStart(text, v.start, limit);
1811
+ if (junk !== -1) {
1812
+ const rest = withoutSeparators(text.slice(junk, limit)).replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '');
1813
+ if (/[A-Za-z0-9]/.test(rest))
1814
+ push(junk, limit, REDACTED_SECRET);
1815
+ }
1816
+ }
1817
+ if (quoted && end < v.end) {
1818
+ maskGaps(end, v.end);
1819
+ }
1820
+ re.lastIndex = quoted ? from : end;
962
1821
  continue;
963
1822
  }
964
1823
  }
965
1824
  else {
966
- pushEdit(edits, text, v.start, v.end, REDACTED_SECRET);
1825
+ push(v.start, keptSepEnd !== -1 && withoutSeparators(text.slice(v.end, keptSepEnd)).search(/[^"'\\:=\uFF1A\uFF1D \t]/) === -1 ? keptSepEnd : v.end, REDACTED_SECRET);
967
1826
  }
968
- re.lastIndex = v.end;
1827
+ re.lastIndex = quoted ? from : v.end;
969
1828
  }
970
- return edits;
971
- }
972
- function literalBlankLeadJudgedAsBlank(text, v, label, schemePass) {
973
- let restStart;
974
- if (v.start > 0 && text.charCodeAt(v.start - 1) === 0x5c && nameBlankTokenAt(text, v.start - 1) !== -1)
975
- restStart = v.start + 5;
976
- else if (nameBlankTokenAt(text, v.start) !== -1)
977
- restStart = v.start + 6;
978
- else if (eatenBlankTokenTailAt(text, v.start))
979
- restStart = v.start + 5;
980
- else
981
- return false;
982
- const rest = withoutSeparators(text.slice(restStart, v.end));
983
- const bare = rest.replace(/[.,;:!?)\]}'"]+$/, '');
984
- if (bare === '')
985
- return true;
986
- if (credentialWordStartsAt(' ' + text.slice(restStart, Math.min(text.length, v.end + 64)), 1))
987
- return true;
988
- if (schemePass && SCHEME_FOLLOWER_WORDS.has(bare.toLowerCase()))
989
- return true;
990
- if (!schemePass && isNamedNotValue(label, rest))
991
- return true;
992
- if (!schemePass && /authorizations?$/i.test(label) && isAuthScheme(bare))
993
- return true;
994
- return !isAddressShaped(rest) && isDiagnosticValue(rest);
995
- }
996
- function eatenBlankTokenTailAt(text, at) {
997
- const hit = /^u([0-9A-Fa-f]{4})/.exec(text.slice(at, at + 5));
998
- return hit !== null && isNameBlank(Number.parseInt(hit[1] ?? '', 16));
999
1829
  }
1000
1830
  function isAuthParamAt(text, j, scheme) {
1001
1831
  const p = /^[A-Za-z][A-Za-z0-9_-]*([ \t\ufeff]*)=([ \t\ufeff]*)("|[^\s=,"]+)([ \t\ufeff]*,)?/.exec(text.slice(j, j + 512));
@@ -1087,13 +1917,15 @@ function pemBodyEndWithoutTrailer(text, from) {
1087
1917
  }
1088
1918
  return end;
1089
1919
  }
1090
- function pemEdits(text) {
1091
- const edits = [];
1092
- if (!text.includes('-----BEGIN '))
1093
- return edits;
1920
+ function pemBlocks(text, soft = () => false) {
1921
+ const out = [];
1922
+ if (!text.includes('-----BEGIN'))
1923
+ return out;
1924
+ const head = pemHeadView(text, soft);
1094
1925
  const begin = new RegExp(PEM_BEGIN_SOURCE, 'g');
1926
+ const trailer = /-----END[A-Z0-9 ]*-----/y;
1095
1927
  let nextEnd = -2;
1096
- for (let b = begin.exec(text); b !== null; b = begin.exec(text)) {
1928
+ for (let b = begin.exec(head); b !== null; b = begin.exec(head)) {
1097
1929
  let bodyStart = b.index + b[0].length;
1098
1930
  const lead = lineBreakLengthAt(text, bodyStart);
1099
1931
  if (lead > 0)
@@ -1104,18 +1936,56 @@ function pemEdits(text) {
1104
1936
  if (nextEnd !== -1 && nextEnd < bodyStart)
1105
1937
  nextEnd = text.indexOf(PEM_END_MARK, bodyStart);
1106
1938
  let bodyEnd;
1107
- if (nextEnd === -1)
1939
+ let regionEnd;
1940
+ if (nextEnd === -1) {
1108
1941
  bodyEnd = pemBodyEndWithoutTrailer(text, bodyStart);
1942
+ regionEnd = bodyEnd + lineBreakLengthAt(text, bodyEnd);
1943
+ }
1109
1944
  else {
1110
1945
  bodyEnd = nextEnd;
1111
1946
  const back = lineBreakLengthBefore(text, bodyEnd);
1112
1947
  if (back > 0 && bodyEnd - back >= bodyStart)
1113
1948
  bodyEnd -= back;
1949
+ trailer.lastIndex = nextEnd;
1950
+ regionEnd = trailer.test(head) ? trailer.lastIndex : nextEnd + PEM_END_MARK.length;
1114
1951
  }
1115
- if (bodyEnd > bodyStart)
1116
- pushEdit(edits, text, bodyStart, bodyEnd, REDACTED_SECRET);
1952
+ out.push({ begin: b.index, bodyStart, bodyEnd, regionEnd });
1117
1953
  begin.lastIndex = Math.max(begin.lastIndex, bodyEnd);
1118
1954
  }
1955
+ return out;
1956
+ }
1957
+ function pemHeadView(text, soft) {
1958
+ let chars = null;
1959
+ for (let k = 0; k < text.length; k++) {
1960
+ if (!soft(k) || text[k] === ' ')
1961
+ continue;
1962
+ chars ??= text.split('');
1963
+ chars[k] = ' ';
1964
+ }
1965
+ return chars === null ? text : chars.join('');
1966
+ }
1967
+ function pemBeginAt(S, at) {
1968
+ const k = at + '-----BEGIN'.length;
1969
+ return S.text.startsWith('-----BEGIN', at) && (S.text[k] === ' ' || (k < S.text.length && pemSoft(S, k)));
1970
+ }
1971
+ const pemSoft = (S, k) => S.dropped(k) || S.painted(k);
1972
+ function pemBlockToRaw(S, blk) {
1973
+ const n = S.text.length;
1974
+ const at = (k) => (k < n ? S.rawStart(k) : S.raw.length);
1975
+ const begin = at(blk.begin);
1976
+ const bodyStart = at(blk.bodyStart);
1977
+ const bodyEnd = blk.bodyEnd > blk.bodyStart ? Math.max(bodyStart, S.rawEnd(blk.bodyEnd - 1)) : bodyStart;
1978
+ const regionEnd = blk.regionEnd > blk.begin ? Math.max(begin, S.rawEnd(blk.regionEnd - 1)) : begin;
1979
+ return { begin, bodyStart, bodyEnd, regionEnd };
1980
+ }
1981
+ function pemEdits(S, onBlock) {
1982
+ const text = S.text;
1983
+ const edits = [];
1984
+ for (const blk of pemBlocks(text, (k) => pemSoft(S, k))) {
1985
+ onBlock(blk);
1986
+ if (blk.bodyEnd > blk.bodyStart)
1987
+ pushEdit(edits, text, blk.bodyStart, blk.bodyEnd, REDACTED_SECRET);
1988
+ }
1119
1989
  return edits;
1120
1990
  }
1121
1991
  function authParamEdits(text) {
@@ -1188,21 +2058,6 @@ function proseLiteralEdits(text) {
1188
2058
  }
1189
2059
  return edits;
1190
2060
  }
1191
- function washWordNets(target, literalBlanks = false) {
1192
- const unitAt = (k) => target.unitAt(k);
1193
- target.apply(pemEdits(target.text));
1194
- target.apply(labelEdits(target.text, SECRET_SCHEME_WORD, literalBlanks, unitAt));
1195
- target.apply(labelEdits(target.text, SECRET_LABELLED_WORD, literalBlanks, unitAt));
1196
- target.apply(authParamEdits(target.text));
1197
- target.apply(proseLiteralEdits(target.text));
1198
- }
1199
- function runCredentialNets(target, urls, words, literalBlanks = false) {
1200
- if (urls)
1201
- washUrlNets(target);
1202
- if (words)
1203
- washWordNets(target, literalBlanks);
1204
- }
1205
- const NEEDS_VIEW = /[\u0000-\u0008\u000E-\u001F\u007F-\u009F\uD800-\uDFFF]|\p{Cf}|\\u/u;
1206
2061
  const FORMAT_CHAR = /^\p{Cf}$/u;
1207
2062
  const isStrippableControl = (c) => (c <= 0x1f && (c < 0x09 || c > 0x0d)) || (c >= 0x7f && c <= 0x9f);
1208
2063
  class SequenceReader {
@@ -1215,10 +2070,12 @@ class SequenceReader {
1215
2070
  termHit = null;
1216
2071
  breakFrom = -1;
1217
2072
  breakAt = -1;
1218
- constructor(text, atomicBlanks, foldBlanks = false) {
2073
+ foldSpaces;
2074
+ constructor(text, atomicBlanks, foldBlanks = false, foldSpaces = false) {
1219
2075
  this.text = text;
1220
2076
  this.atomicBlanks = atomicBlanks;
1221
2077
  this.foldBlanks = foldBlanks;
2078
+ this.foldSpaces = foldSpaces;
1222
2079
  this.terminator = /[\u0007\u009c]|\u001b\\|\\u(?:0007|009[cC]|001[bB]\\)/g;
1223
2080
  this.lineBreak = /[\n\r]/g;
1224
2081
  }
@@ -1236,6 +2093,8 @@ class SequenceReader {
1236
2093
  const c = this.text.charCodeAt(at);
1237
2094
  if (c >= 0x20 && c <= 0x2f)
1238
2095
  return 1;
2096
+ if (this.foldSpaces && ((c >= 0x09 && c <= 0x0d) || c === 0x2028 || c === 0x2029))
2097
+ return 1;
1239
2098
  if (!this.foldBlanks)
1240
2099
  return 0;
1241
2100
  if (isFoldedBlank(c))
@@ -1351,41 +2210,38 @@ function nameBlankTokenAt(text, at) {
1351
2210
  const cp = Number.parseInt(hit[1] ?? '', 16);
1352
2211
  return isNameBlank(cp) ? cp : -1;
1353
2212
  }
1354
- function buildCredentialView(text, separated, wordLetters = false, blankTokens = 'off') {
1355
- const folding = blankTokens === 'fold' || blankTokens === 'foldlit';
1356
- if (!NEEDS_VIEW.test(text) && !(folding && hasRawFoldedBlank(text)))
1357
- return null;
2213
+ function buildScanView(text, o) {
1358
2214
  const chars = [];
1359
2215
  const starts = [];
1360
2216
  const ends = [];
1361
2217
  const units = [];
1362
- let changed = false;
1363
- let splitAny = false;
1364
- const seq = new SequenceReader(text, blankTokens === 'blank' || blankTokens === 'fold' ? 'all' : blankTokens === 'foldlit' ? 'folded' : 'none', folding);
2218
+ const dropped = [];
2219
+ const spaceUnits = [];
2220
+ const contents = [];
2221
+ const seq = new SequenceReader(text, o.blanks ? 'all' : 'none', o.blanks || o.fold, o.fold);
1365
2222
  const keep = (ch, s, e, unit = UNIT_NONE) => {
1366
2223
  chars.push(ch);
1367
2224
  starts.push(s);
1368
2225
  ends.push(e);
1369
2226
  units.push(unit);
2227
+ dropped.push(0);
2228
+ spaceUnits.push(0);
1370
2229
  };
1371
- const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE) => {
1372
- changed = true;
1373
- if (separated)
1374
- keep(placeholder, s, e, unit);
2230
+ const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE, space = 0) => {
2231
+ chars.push(placeholder);
2232
+ starts.push(s);
2233
+ ends.push(e);
2234
+ units.push(unit);
2235
+ dropped.push(1);
2236
+ spaceUnits.push(space);
1375
2237
  };
1376
2238
  const dropUnit = (s, unit) => {
1377
2239
  const last = unit.end - 1;
1378
- if ((blankTokens === 'literal' || blankTokens === 'foldlit') && last > s && text.charCodeAt(last) === 0x5c && nameBlankTokenAt(text, last) !== -1) {
1379
- drop(s, last, VIEW_SEPARATOR, UNIT_SEQUENCE);
1380
- keep(text.charAt(last + 1), last, last + 6);
1381
- for (let u = 2; u < 6; u++)
1382
- keep(text.charAt(last + u), last + 6, last + 6);
1383
- return last + 6;
1384
- }
1385
- if (wordLetters && unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
2240
+ if (unit.content !== undefined)
2241
+ contents.push([unit.content[0], unit.content[1]]);
2242
+ if (unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
1386
2243
  drop(s, last, VIEW_SEPARATOR, UNIT_SEQUENCE);
1387
2244
  keep(text.charAt(last), last, unit.end);
1388
- splitAny = true;
1389
2245
  return unit.end;
1390
2246
  }
1391
2247
  drop(s, unit.end, VIEW_SEPARATOR, UNIT_SEQUENCE);
@@ -1394,12 +2250,6 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
1394
2250
  let i = 0;
1395
2251
  while (i < text.length) {
1396
2252
  const c = text.charCodeAt(i);
1397
- if (folding && isFoldedBlank(c)) {
1398
- changed = true;
1399
- keep(' ', i, i + 1);
1400
- i++;
1401
- continue;
1402
- }
1403
2253
  if (c === 0x1b || c === 0x9b || c === 0x9d || c === 0x90 || c === 0x98 || c === 0x9e || c === 0x9f) {
1404
2254
  const unit = c === 0x1b ? escapeSequenceAfterEsc(text, i + 1, seq) : c1SequenceAfter(text, c, i + 1, seq);
1405
2255
  if (unit !== null) {
@@ -1422,30 +2272,33 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
1422
2272
  i = dropUnit(i, unit);
1423
2273
  continue;
1424
2274
  }
1425
- drop(i, tok.end, ' ', isRegexSpace(tok.cp) && tok.cp !== 0xfeff ? UNIT_NONE : UNIT_SINGLE);
2275
+ if (isRegexSpace(tok.cp) && tok.cp !== 0xfeff) {
2276
+ if (o.escapeSpaces !== null && o.escapeSpaces.has(tok.cp))
2277
+ drop(i, tok.end, VIEW_SEPARATOR, UNIT_SINGLE, 1);
2278
+ else
2279
+ drop(i, tok.end, ' ', UNIT_NONE);
2280
+ }
2281
+ else
2282
+ drop(i, tok.end);
1426
2283
  i = tok.end;
1427
2284
  continue;
1428
2285
  }
1429
- if (blankTokens !== 'off') {
1430
- const cp = nameBlankTokenAt(text, i);
1431
- if (cp !== -1) {
1432
- changed = true;
1433
- if (blankTokens === 'blank')
1434
- keep(String.fromCharCode(cp), i, i + 6);
1435
- else if (blankTokens === 'fold')
1436
- keep(isFoldedBlank(cp) ? ' ' : String.fromCharCode(cp), i, i + 6);
1437
- else if (blankTokens === 'foldlit' && isFoldedBlank(cp))
1438
- keep(' ', i, i + 6);
1439
- else {
1440
- keep(text.charAt(i), i, i + 6);
1441
- for (let u = 1; u < 6; u++)
1442
- keep(text.charAt(i + u), i + 6, i + 6);
1443
- }
1444
- i += 6;
1445
- continue;
1446
- }
2286
+ if (nameBlankTokenAt(text, i) !== -1) {
2287
+ drop(i, i + 6);
2288
+ i += 6;
2289
+ continue;
1447
2290
  }
1448
2291
  }
2292
+ if (o.blanks && isNameBlank(c)) {
2293
+ drop(i, i + 1);
2294
+ i++;
2295
+ continue;
2296
+ }
2297
+ if (o.escapeSpaces !== null && o.escapeSpaces.has(c)) {
2298
+ drop(i, i + 1, VIEW_SEPARATOR, UNIT_SINGLE, 1);
2299
+ i++;
2300
+ continue;
2301
+ }
1449
2302
  if (isStrippableControl(c)) {
1450
2303
  drop(i, i + 1);
1451
2304
  i++;
@@ -1467,9 +2320,195 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
1467
2320
  keep(text.charAt(i), i, i + 1);
1468
2321
  i++;
1469
2322
  }
1470
- if (!changed || (wordLetters && !splitAny))
2323
+ const rawUnit = new Uint8Array(text.length);
2324
+ for (let k = 0; k < chars.length; k++) {
2325
+ if (dropped[k] !== 1)
2326
+ continue;
2327
+ for (let x = starts[k] ?? 0; x < (ends[k] ?? 0); x++)
2328
+ rawUnit[x] = 1;
2329
+ }
2330
+ return {
2331
+ view: chars.join(''),
2332
+ starts: Int32Array.from(starts),
2333
+ ends: Int32Array.from(ends),
2334
+ units: Uint8Array.from(units),
2335
+ dropped: Uint8Array.from(dropped),
2336
+ spaceUnits: Uint8Array.from(spaceUnits),
2337
+ contents,
2338
+ rawUnit,
2339
+ };
2340
+ }
2341
+ const viewRawStart = (V, M, k) => {
2342
+ const s = V.starts[k] ?? 0;
2343
+ return M === null ? s : (M.ps[s] ?? 0);
2344
+ };
2345
+ const viewRawEnd = (V, M, k) => {
2346
+ const e = V.ends[k] ?? 0;
2347
+ return M === null ? e : (M.pe[Math.max(0, e - 1)] ?? 0);
2348
+ };
2349
+ const viewPainted = (V, M, k) => {
2350
+ if (M === null)
2351
+ return false;
2352
+ for (let x = V.starts[k] ?? 0; x < (V.ends[k] ?? 0); x++)
2353
+ if (M.pdot[x] === 1)
2354
+ return true;
2355
+ return false;
2356
+ };
2357
+ function separatedSurface(raw, V, M = null) {
2358
+ return {
2359
+ text: V.view,
2360
+ raw,
2361
+ unitAt: (k) => V.units[k] ?? UNIT_NONE,
2362
+ rawStart: (k) => viewRawStart(V, M, k),
2363
+ rawEnd: (k) => viewRawEnd(V, M, k),
2364
+ dropped: (k) => V.dropped[k] === 1,
2365
+ painted: (k) => viewPainted(V, M, k),
2366
+ gapAfter: (k) => V.dropped[k + 1] === 1,
2367
+ spaceGapBefore: () => false,
2368
+ rawDot: M === null ? null : M.rawDot,
2369
+ };
2370
+ }
2371
+ function joinedSurface(raw, V, spacesKept = false, M = null) {
2372
+ const map = [];
2373
+ const chars = [];
2374
+ const spaceBefore = [];
2375
+ let pendingSpace = 0;
2376
+ let keptAny = false;
2377
+ for (let k = 0; k < V.view.length; k++) {
2378
+ if (V.dropped[k] === 1) {
2379
+ if (V.spaceUnits[k] === 1) {
2380
+ if (spacesKept) {
2381
+ map.push(k);
2382
+ chars.push(' ');
2383
+ spaceBefore.push(0);
2384
+ keptAny = true;
2385
+ continue;
2386
+ }
2387
+ pendingSpace = 1;
2388
+ }
2389
+ continue;
2390
+ }
2391
+ map.push(k);
2392
+ chars.push(V.view.charAt(k));
2393
+ spaceBefore.push(pendingSpace);
2394
+ pendingSpace = 0;
2395
+ }
2396
+ if (map.length === V.view.length || (spacesKept && !keptAny))
1471
2397
  return null;
1472
- return { view: chars.join(''), starts: Int32Array.from(starts), ends: Int32Array.from(ends), units: Uint8Array.from(units) };
2398
+ const arr = Int32Array.from(map);
2399
+ const gapSpace = Uint8Array.from(spaceBefore);
2400
+ return {
2401
+ text: chars.join(''),
2402
+ raw,
2403
+ unitAt: () => UNIT_NONE,
2404
+ rawStart: (k) => viewRawStart(V, M, arr[k] ?? 0),
2405
+ rawEnd: (k) => viewRawEnd(V, M, arr[k] ?? 0),
2406
+ dropped: () => false,
2407
+ painted: (k) => viewPainted(V, M, arr[k] ?? 0),
2408
+ gapAfter: (k) => k + 1 < arr.length && arr[k + 1] !== (arr[k] ?? 0) + 1,
2409
+ spaceGapBefore: (k) => gapSpace[k] === 1,
2410
+ rawDot: M === null ? null : M.rawDot,
2411
+ };
2412
+ }
2413
+ function screenSurfaces(text, o) {
2414
+ const face = o.face;
2415
+ if (face === null)
2416
+ return { surfaces: [], rawSoft: null, rawPlain: null, contents: [] };
2417
+ const unsafe = face.unsafe === null ? null : new RegExp(face.unsafe.source, 'u');
2418
+ const keepBlanks = face.keepBlanks === true;
2419
+ const chars = [];
2420
+ const ps = [];
2421
+ const pe = [];
2422
+ const pdot = [];
2423
+ const rawDot = new Uint8Array(text.length);
2424
+ const rawPlain = new Uint8Array(text.length);
2425
+ let changed = false;
2426
+ const foldable = (at) => {
2427
+ const c = text.codePointAt(at) ?? 0;
2428
+ return isRegexSpace(c) && !(keepBlanks && isNameBlank(c));
2429
+ };
2430
+ let lo = 0;
2431
+ let hi = text.length;
2432
+ if (face.fold) {
2433
+ while (lo < hi && foldable(lo))
2434
+ lo++;
2435
+ while (hi > lo && foldable(hi - 1))
2436
+ hi--;
2437
+ if (lo > 0 || hi < text.length)
2438
+ changed = true;
2439
+ }
2440
+ for (let i = lo; i < hi;) {
2441
+ const cp = text.codePointAt(i) ?? 0;
2442
+ const w = cp > 0xffff ? 2 : 1;
2443
+ const ch = text.slice(i, i + w);
2444
+ let form = null;
2445
+ let dotted = false;
2446
+ if (face.fold && foldable(i)) {
2447
+ let j = i + 1;
2448
+ while (j < hi && foldable(j))
2449
+ j++;
2450
+ if (j - i > 1 || cp !== 0x20)
2451
+ changed = true;
2452
+ chars.push(' ');
2453
+ ps.push(i);
2454
+ pe.push(j);
2455
+ pdot.push(cp === 0xfeff || j - i > 1 ? 1 : 0);
2456
+ for (let x = i; x < j; x++)
2457
+ if (text.charCodeAt(x) === 0xfeff)
2458
+ rawDot[x] = 1;
2459
+ i = j;
2460
+ continue;
2461
+ }
2462
+ else if (unsafe !== null && unsafe.test(ch) && !faceHitKept(ch, i, text)) {
2463
+ form = markOf(ch, face.mark);
2464
+ dotted = face.mark !== 'escape';
2465
+ }
2466
+ if (form === null || form === ch) {
2467
+ for (let u = 0; u < w; u++) {
2468
+ chars.push(text.charAt(i + u));
2469
+ ps.push(i + u);
2470
+ pe.push(i + u + 1);
2471
+ pdot.push(0);
2472
+ rawPlain[i + u] = 1;
2473
+ }
2474
+ }
2475
+ else {
2476
+ changed = true;
2477
+ for (let u = 0; u < form.length; u++) {
2478
+ chars.push(form.charAt(u));
2479
+ ps.push(i);
2480
+ pe.push(i + w);
2481
+ pdot.push(dotted ? 1 : 0);
2482
+ }
2483
+ if (dotted)
2484
+ for (let x = i; x < i + w; x++)
2485
+ rawDot[x] = 1;
2486
+ }
2487
+ i += w;
2488
+ }
2489
+ if (!changed)
2490
+ return { surfaces: [], rawSoft: null, rawPlain: null, contents: [] };
2491
+ const screen = chars.join('');
2492
+ const M = { ps: Int32Array.from(ps), pe: Int32Array.from(pe), pdot: Uint8Array.from(pdot), rawDot };
2493
+ const V = buildScanView(screen, o);
2494
+ const rawSoft = Uint8Array.from(rawDot);
2495
+ for (let k = 0; k < V.view.length; k++) {
2496
+ if (V.dropped[k] !== 1)
2497
+ continue;
2498
+ for (let x = viewRawStart(V, M, k); x < viewRawEnd(V, M, k); x++) {
2499
+ rawSoft[x] = 1;
2500
+ rawPlain[x] = 0;
2501
+ }
2502
+ }
2503
+ const out = [separatedSurface(text, V, M)];
2504
+ const joined = joinedSurface(text, V, false, M);
2505
+ if (joined !== null)
2506
+ out.push(joined);
2507
+ const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true, M) : null;
2508
+ if (joinedSpaced !== null)
2509
+ out.push(joinedSpaced);
2510
+ const contents = V.contents.map(([a, b]) => ({ sub: screen.slice(a, b), rawAt: (k) => M.ps[a + k] ?? 0, rawEndAt: (k) => M.pe[a + k] ?? 0 }));
2511
+ return { surfaces: out, rawSoft, rawPlain, contents };
1473
2512
  }
1474
2513
  function finalIsStructural(text, at) {
1475
2514
  if (text.charCodeAt(at) !== 0x5c)
@@ -1497,260 +2536,350 @@ function credentialWordBeginsWith(text, at) {
1497
2536
  let end = Math.min(text.length, at + 64);
1498
2537
  while (end < text.length && isProbeFiller(text.charCodeAt(end - 1)))
1499
2538
  end++;
1500
- const probe = ' ' + text.slice(at, Math.min(text.length, end + 2));
2539
+ const probe = ' ' + text.slice(at, Math.min(text.length, end + 2)) + (end + 2 >= text.length ? ' ' : '');
1501
2540
  if (letter)
1502
2541
  return credentialWordStartsAt(probe, 1);
1503
2542
  return credentialWordStartsAt(probe, 2) || credentialWordStartsAt(probe, 3);
1504
2543
  }
1505
- function rebuildFromView(original, view, target, trace = false) {
2544
+ function editsToCandidates(edits, cls, out, te = -1) {
2545
+ for (const e of edits) {
2546
+ const kind = KIND_BY_TEXT.get(e.text);
2547
+ if (kind === undefined || e.end <= e.start)
2548
+ continue;
2549
+ out.push({ a: e.start, b: e.end, kind, cls, te });
2550
+ }
2551
+ }
2552
+ function pemRegions(recognized) {
2553
+ const seen = new Set();
1506
2554
  const out = [];
1507
- const origin = trace ? [] : null;
1508
- const fromOriginal = (a, b) => {
1509
- out.push(original.slice(a, b));
1510
- if (origin !== null)
1511
- for (let x = a; x < b; x++)
1512
- origin.push(x);
1513
- };
1514
- const inserted = (s) => {
1515
- out.push(s);
1516
- if (origin !== null)
1517
- for (let x = 0; x < s.length; x++)
1518
- origin.push(-1);
1519
- };
1520
- const viewLength = view.view.length;
1521
- let cursor = 0;
1522
- let k = 0;
1523
- const text = target.text;
1524
- while (k < text.length) {
1525
- const s = target.src[k] ?? 0;
1526
- if (s >= 0) {
1527
- const end = view.ends[s] ?? cursor;
1528
- if ((view.starts[s] ?? 0) >= cursor) {
1529
- fromOriginal(cursor, end);
1530
- cursor = end;
1531
- }
1532
- else
1533
- inserted(text.charAt(k));
1534
- k++;
2555
+ for (const blk of recognized) {
2556
+ if (blk.regionEnd <= blk.begin)
2557
+ continue;
2558
+ const key = `${blk.begin}:${blk.bodyStart}:${blk.bodyEnd}:${blk.regionEnd}`;
2559
+ if (seen.has(key))
1535
2560
  continue;
2561
+ seen.add(key);
2562
+ out.push(blk);
2563
+ }
2564
+ return out;
2565
+ }
2566
+ function resolveCandidates(text, cands, rawUnit, rawDot, pems, rawSoft = rawDot, rawPlain = null) {
2567
+ const items = [];
2568
+ for (const c of cands) {
2569
+ let a = c.start;
2570
+ let b = c.end;
2571
+ if (c.cls === 'pem') {
1536
2572
  }
1537
- let j = k + 1;
1538
- while (j < text.length && (target.src[j] ?? 0) < 0)
1539
- j++;
1540
- let a = target.rangeStart(k);
1541
- let b = target.rangeEnd(k);
1542
- for (let t = k + 1; t < j; t++) {
1543
- a = Math.min(a, target.rangeStart(t));
1544
- b = Math.max(b, target.rangeEnd(t));
1545
- }
1546
- const from = a < viewLength ? (view.starts[a] ?? original.length) : original.length;
1547
- const to = b > a ? (view.ends[b - 1] ?? from) : from;
1548
- if (from > cursor)
1549
- fromOriginal(cursor, from);
1550
- inserted(text.slice(k, j));
1551
- cursor = Math.max(cursor, to);
1552
- k = j;
2573
+ else if (c.dotted === true) {
2574
+ while (a < b && rawDot !== null && rawDot[a] === 1)
2575
+ a++;
2576
+ if (a >= b)
2577
+ a = c.start;
2578
+ else if (rawDot !== null && b < text.length && isKeyWordChar(text.charCodeAt(b))) {
2579
+ while (b > a && rawDot[b - 1] === 1)
2580
+ b--;
2581
+ }
2582
+ }
2583
+ else if (c.pretrimmed !== true) {
2584
+ while (a < b && rawUnit[a] === 1)
2585
+ a++;
2586
+ while (b > a && rawUnit[b - 1] === 1)
2587
+ b--;
2588
+ if (rawPlain !== null && rawDot !== null) {
2589
+ let k = a;
2590
+ while (k > 0 && rawUnit[k - 1] === 1 && rawPlain[k - 1] === 1 && isKeyWordChar(text.charCodeAt(k - 1)))
2591
+ k--;
2592
+ if (k < a && k > 0 && (rawDot[k - 1] === 1 || isRegexSpace(text.charCodeAt(k - 1))))
2593
+ a = k;
2594
+ }
2595
+ }
2596
+ if (a >= b)
2597
+ continue;
2598
+ const body = text.slice(a, b);
2599
+ if (isMarker(body) || MARKER_WITH_DOTS.test(body))
2600
+ continue;
2601
+ items.push({ start: a, end: b, kind: c.kind, cls: c.cls, tokenEnd: c.tokenEnd, origin: c.dotted === true ? c.start : a });
1553
2602
  }
1554
- fromOriginal(cursor, original.length);
1555
- const rebuilt = out.join('');
1556
- return origin === null ? { text: rebuilt } : { text: rebuilt, origin: Int32Array.from(origin) };
1557
- }
1558
- function washThroughView(original, view, urls, words, trace = false, literalBlanks = false) {
1559
- const target = new TrackedTarget(view.view, view.view.length, view.units);
1560
- runCredentialNets(target, urls, words, literalBlanks);
1561
- return target.text === view.view ? { text: original } : rebuildFromView(original, view, target, trace);
1562
- }
1563
- function washCredentialNetsOnce(text, urls, words, nested, blankTokens, trace) {
1564
- const contents = nested ? { text } : washControlStringContents(text, urls, words, blankTokens, trace);
1565
- if (blankTokens === 'off')
1566
- return washThroughViews(contents, urls, words, 'off', trace);
1567
- const tokens = hasNameBlankToken(contents.text);
1568
- const readings = [];
1569
- if (tokens)
1570
- readings.push('literal');
1571
- const rawFolded = blankTokens === 'fold' && hasRawFoldedBlank(contents.text);
1572
- if (blankTokens === 'fold' && (tokens || rawFolded))
1573
- readings.push('fold');
1574
- if (blankTokens === 'fold' && tokens && hasNameBlankToken(contents.text, false) && (rawFolded || hasNameBlankToken(contents.text, true)))
1575
- readings.push('foldlit');
1576
- readings.push('blank');
1577
- if (readings.length === 1)
1578
- return washThroughViews(contents, urls, words, 'blank', trace);
1579
- const base = { text: contents.text };
1580
- const union = unionMaskedRanges(base.text, readings.map((r) => washThroughViews(base, urls, words, r, true)));
1581
- return traced(union.text, composeOrigin(contents.origin, union.origin));
1582
- }
1583
- function unionMaskedRanges(text, results) {
1584
- const ranges = [];
1585
- for (const r of results) {
1586
- if (r.text === text || r.origin === undefined)
2603
+ items.sort((x, y) => (x.start !== y.start ? x.start - y.start : y.end - x.end));
2604
+ let maxTokenEnd = null;
2605
+ for (const x of items) {
2606
+ if (!isStructuralKind(x.kind) || x.cls !== 'url')
1587
2607
  continue;
1588
- const origin = r.origin;
1589
- let lastKept = -1;
1590
- let k = 0;
1591
- while (k < origin.length) {
1592
- if ((origin[k] ?? -1) >= 0) {
1593
- lastKept = origin[k] ?? lastKept;
1594
- k++;
2608
+ if (maxTokenEnd === null)
2609
+ maxTokenEnd = new Int32Array(text.length).fill(-1);
2610
+ for (let k = x.start; k < x.end && k < text.length; k++)
2611
+ if (x.tokenEnd > (maxTokenEnd[k] ?? -1))
2612
+ maxTokenEnd[k] = x.tokenEnd;
2613
+ }
2614
+ const p2Drops = (x) => !isStructuralKind(x.kind) && x.cls !== 'pem' && x.cls !== 'glued' && maxTokenEnd !== null && x.end <= (maxTokenEnd[x.start] ?? -1);
2615
+ let pemCover = null;
2616
+ if (pems.length > 0) {
2617
+ const reachBefore = new Int32Array(text.length + 1).fill(-1);
2618
+ for (const x of items)
2619
+ if (x.cls !== 'pem' && !p2Drops(x) && x.origin < text.length && x.end > (reachBefore[x.origin + 1] ?? -1))
2620
+ reachBefore[x.origin + 1] = x.end;
2621
+ for (let k = 1; k <= text.length; k++)
2622
+ if ((reachBefore[k - 1] ?? -1) > (reachBefore[k] ?? -1))
2623
+ reachBefore[k] = reachBefore[k - 1] ?? -1;
2624
+ let pemMask = null;
2625
+ for (const x of items) {
2626
+ if (x.cls !== 'pem')
1595
2627
  continue;
2628
+ pemMask ??= new Uint8Array(text.length);
2629
+ for (let k = x.start; k < x.end && k < text.length; k++)
2630
+ pemMask[k] = 1;
2631
+ }
2632
+ for (const blk of pems) {
2633
+ const a = blk.begin;
2634
+ const b = blk.regionEnd;
2635
+ if ((reachBefore[a] ?? -1) > a)
2636
+ continue;
2637
+ pemCover ??= new Uint8Array(text.length);
2638
+ for (let k = a; k < b && k < text.length; k++) {
2639
+ if (k >= blk.bodyStart && k < blk.bodyEnd && rawUnit[k] === 1 && pemMask?.[k] !== 1)
2640
+ continue;
2641
+ pemCover[k] = 1;
1596
2642
  }
1597
- let j = k;
1598
- while (j < origin.length && (origin[j] ?? -1) < 0)
1599
- j++;
1600
- const next = j < origin.length ? (origin[j] ?? text.length) : text.length;
1601
- ranges.push({ start: lastKept + 1, end: next, text: r.text.slice(k, j) });
1602
- k = j;
1603
2643
  }
1604
2644
  }
1605
- if (ranges.length === 0)
1606
- return { text };
1607
- ranges.sort((a, b) => (a.start !== b.start ? a.start - b.start : b.end - a.end));
1608
- const merged = [];
1609
- for (const m of ranges) {
1610
- const last = merged[merged.length - 1];
1611
- if (last !== undefined && (m.start < last.end || (m.start === last.start && m.end === last.end))) {
1612
- if (m.end > last.end) {
1613
- if (m.end - m.start > last.end - last.start)
1614
- last.text = m.text;
1615
- last.end = m.end;
1616
- }
2645
+ const kept = items.filter((x) => {
2646
+ if (x.cls !== 'pem' && pemCover !== null && pemCover[x.origin] === 1)
2647
+ return false;
2648
+ if (p2Drops(x))
2649
+ return false;
2650
+ return true;
2651
+ });
2652
+ const groups = [];
2653
+ for (const it of kept) {
2654
+ const g = groups[groups.length - 1];
2655
+ if (g !== undefined && it.start < g.end) {
2656
+ if (it.end > g.end)
2657
+ g.end = it.end;
2658
+ g.members.push(it);
1617
2659
  continue;
1618
2660
  }
1619
- if (last !== undefined && m.start === m.end && m.start > last.start && m.start < last.end)
2661
+ groups.push({ start: it.start, end: it.end, members: [it] });
2662
+ }
2663
+ const out = [];
2664
+ for (const g of groups) {
2665
+ let structStart = Infinity;
2666
+ let structEnd = -Infinity;
2667
+ let firstStructKind = null;
2668
+ let firstStructStart = Infinity;
2669
+ let anySecret = false;
2670
+ const secretLike = (m) => !isStructuralKind(m.kind) || m.cls === 'userinfo-hidden';
2671
+ for (const m of g.members) {
2672
+ if (!secretLike(m)) {
2673
+ if (m.start < structStart)
2674
+ structStart = m.start;
2675
+ if (m.end > structEnd)
2676
+ structEnd = m.end;
2677
+ if (m.start < firstStructStart) {
2678
+ firstStructStart = m.start;
2679
+ firstStructKind = m.kind;
2680
+ }
2681
+ }
2682
+ else
2683
+ anySecret = true;
2684
+ }
2685
+ let secretWider = false;
2686
+ if (anySecret)
2687
+ for (const m of g.members)
2688
+ if (secretLike(m) && (m.start < structStart || m.end > structEnd))
2689
+ secretWider = true;
2690
+ const onlyHidden = g.members.every((m) => m.cls === 'userinfo-hidden');
2691
+ const kind = onlyHidden ? 'userinfo' : firstStructKind === null || secretWider ? 'secret' : firstStructKind;
2692
+ const value = g.members.some((m) => m.cls === 'word' || m.cls === 'glued' || m.cls === 'param' || (m.kind !== 'userinfo' && m.kind !== 'secret' && (m.cls === 'url' || m.cls === 'structural')));
2693
+ out.push({ start: g.start, end: g.end, kind, value });
2694
+ }
2695
+ return continueValues(text, out, rawSoft ?? rawUnit);
2696
+ }
2697
+ function continueValues(text, drafts, rawSoft) {
2698
+ const out = [];
2699
+ const soft = (k) => rawSoft[k] === 1;
2700
+ let i = 0;
2701
+ while (i < drafts.length) {
2702
+ const cur = drafts[i];
2703
+ i++;
2704
+ if (cur === undefined)
1620
2705
  continue;
1621
- merged.push({ ...m });
2706
+ const d = { start: cur.start, end: cur.end, kind: cur.kind, value: cur.value };
2707
+ for (;;) {
2708
+ let k = d.end;
2709
+ if (!d.value || k >= text.length || !soft(k))
2710
+ break;
2711
+ while (k < text.length && soft(k))
2712
+ k++;
2713
+ if (k >= text.length || !isKeyWordChar(text.charCodeAt(k)) || credentialWordStartsAt(text, k))
2714
+ break;
2715
+ let e = k;
2716
+ while (e < text.length && (soft(e) || !isBareValueStop(text.charAt(e))))
2717
+ e++;
2718
+ d.end = e;
2719
+ while (i < drafts.length && (drafts[i]?.start ?? Infinity) < d.end) {
2720
+ const next = drafts[i];
2721
+ i++;
2722
+ if (next === undefined)
2723
+ continue;
2724
+ if (next.end > d.end)
2725
+ d.end = next.end;
2726
+ if (next.kind !== d.kind)
2727
+ d.kind = 'secret';
2728
+ }
2729
+ }
2730
+ out.push(d);
1622
2731
  }
2732
+ return out;
2733
+ }
2734
+ function materialize(text, intervals, trace) {
2735
+ if (intervals.length === 0)
2736
+ return { text };
1623
2737
  const parts = [];
1624
- const origin = [];
2738
+ const origin = trace ? [] : null;
1625
2739
  let cursor = 0;
1626
- for (const m of merged) {
1627
- parts.push(text.slice(cursor, m.start), m.text);
1628
- for (let x = cursor; x < m.start; x++)
1629
- origin.push(x);
1630
- for (let x = 0; x < m.text.length; x++)
1631
- origin.push(-1);
1632
- cursor = m.end;
2740
+ for (const iv of intervals) {
2741
+ const token = TEXT_BY_KIND[iv.kind];
2742
+ parts.push(text.slice(cursor, iv.start), token);
2743
+ if (origin !== null) {
2744
+ for (let x = cursor; x < iv.start; x++)
2745
+ origin.push(x);
2746
+ for (let x = 0; x < token.length; x++)
2747
+ origin.push(-1);
2748
+ }
2749
+ cursor = iv.end;
1633
2750
  }
1634
2751
  parts.push(text.slice(cursor));
1635
- for (let x = cursor; x < text.length; x++)
1636
- origin.push(x);
1637
- return { text: parts.join(''), origin: Int32Array.from(origin) };
1638
- }
1639
- function hasRawFoldedBlank(text) {
1640
- for (let k = 0; k < text.length; k++)
1641
- if (isFoldedBlank(text.charCodeAt(k)))
1642
- return true;
1643
- return false;
2752
+ if (origin !== null)
2753
+ for (let x = cursor; x < text.length; x++)
2754
+ origin.push(x);
2755
+ return origin === null ? { text: parts.join('') } : { text: parts.join(''), origin: Int32Array.from(origin) };
1644
2756
  }
1645
- function hasNameBlankToken(text, folded) {
1646
- if (!text.includes('\\u'))
1647
- return false;
1648
- for (const hit of text.matchAll(/\\u([0-9A-Fa-f]{4})/g)) {
1649
- const cp = Number.parseInt(hit[1] ?? '', 16);
1650
- if (isNameBlank(cp) && (folded === undefined || isFoldedBlank(cp) === folded))
1651
- return true;
2757
+ function toRaw(S, c) {
2758
+ let a = c.a;
2759
+ let b = c.b;
2760
+ while (a < b && S.dropped(a))
2761
+ a++;
2762
+ while (b > a && S.dropped(b - 1))
2763
+ b--;
2764
+ if (a >= b)
2765
+ return null;
2766
+ const te = c.te > b ? S.rawEnd(c.te - 1) : S.rawEnd(b - 1);
2767
+ return { start: S.rawStart(a), end: S.rawEnd(b - 1), kind: c.kind, cls: c.cls, tokenEnd: te };
2768
+ }
2769
+ function collectCandidates(text, o, nested) {
2770
+ const V = buildScanView(text, o);
2771
+ const separated = separatedSurface(text, V);
2772
+ const surfaces = [separated];
2773
+ const joined = joinedSurface(text, V);
2774
+ if (joined !== null)
2775
+ surfaces.push(joined);
2776
+ const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true) : null;
2777
+ if (joinedSpaced !== null)
2778
+ surfaces.push(joinedSpaced);
2779
+ const screen = screenSurfaces(text, o);
2780
+ surfaces.push(...screen.surfaces);
2781
+ const raw = [];
2782
+ const regionSurface = screen.surfaces[0] ?? separated;
2783
+ const pemBlocksSeen = [];
2784
+ for (const S of surfaces) {
2785
+ const cands = [];
2786
+ const regions = [];
2787
+ if (o.urls) {
2788
+ const unitPos = (k) => S.dropped(k) || V.rawUnit[S.rawStart(k)] === 1;
2789
+ urlCandidates(S.text, cands, regions, unitPos, (k) => unitPos(k) || S.painted(k));
2790
+ userinfoCandidates(S, cands, o.words);
2791
+ }
2792
+ if (o.words) {
2793
+ editsToCandidates(pemEdits(S, (blk) => { if (S === regionSurface)
2794
+ pemBlocksSeen.push(pemBlockToRaw(S, blk)); }), 'pem', cands);
2795
+ labelCandidates(S, 'scheme', regions, cands);
2796
+ labelCandidates(S, 'token', regions, cands);
2797
+ labelCandidates(S, 'label', regions, cands);
2798
+ editsToCandidates(authParamEdits(S.text), 'param', cands);
2799
+ editsToCandidates(proseLiteralEdits(S.text), 'prose', cands);
2800
+ }
2801
+ for (const c of cands) {
2802
+ const r = toRaw(S, c);
2803
+ if (r !== null)
2804
+ raw.push(S.rawDot !== null ? { ...r, dotted: true } : r);
2805
+ }
1652
2806
  }
1653
- return false;
1654
- }
1655
- function washThroughViews(input, urls, words, blankRead, trace) {
1656
- const start = input.text;
1657
- const literal = blankRead === 'literal' || blankRead === 'foldlit';
1658
- const lettered = buildCredentialView(start, true, true, blankRead);
1659
- const letteredWashed = lettered === null ? { text: start } : washThroughView(start, lettered, urls, words, trace, literal);
1660
- const pre = letteredWashed.text;
1661
- let origin = composeOrigin(input.origin, letteredWashed.origin);
1662
- const separated = buildCredentialView(pre, true, false, blankRead);
1663
- if (separated === null) {
1664
- const target = new PlainTarget(pre, trace);
1665
- runCredentialNets(target, urls, words, literal);
1666
- return traced(target.text, composeOrigin(origin, target.origin));
1667
- }
1668
- const afterSeparated = washThroughView(pre, separated, urls, words, trace, literal);
1669
- origin = composeOrigin(origin, afterSeparated.origin);
1670
- const joined = buildCredentialView(afterSeparated.text, false, false, blankRead);
1671
- if (joined === null)
1672
- return traced(afterSeparated.text, origin);
1673
- const afterJoined = washThroughView(afterSeparated.text, joined, urls, words, trace, literal);
1674
- return traced(afterJoined.text, composeOrigin(origin, afterJoined.origin));
1675
- }
1676
- function washControlStringContents(text, urls, words, blankTokens, trace) {
1677
- if (!/[\u001b\u0090\u0098\u009d-\u009f]|\\u/.test(text))
1678
- return { text };
1679
- const ends = new SequenceReader(text, 'none');
1680
- const parts = [];
1681
- const origin = trace ? [] : null;
1682
- const keptUpTo = (a, b) => {
1683
- if (origin !== null)
1684
- for (let x = a; x < b; x++)
1685
- origin.push(x);
1686
- };
1687
- let cursor = 0;
1688
- let i = 0;
1689
- while (i < text.length) {
1690
- const c = text.charCodeAt(i);
1691
- let unit = null;
1692
- let next = i + 1;
1693
- if (c === 0x1b)
1694
- unit = escapeSequenceAfterEsc(text, i + 1, ends);
1695
- else if (c >= 0x90 && c <= 0x9f)
1696
- unit = c1SequenceAfter(text, c, i + 1, ends);
1697
- else if (c === 0x5c && text.charCodeAt(i + 1) === 0x75) {
1698
- const tok = escapeTokenAt(text, i);
1699
- if (tok !== null) {
1700
- next = tok.end;
1701
- unit = tok.cp === 0x1b ? escapeSequenceAfterEsc(text, tok.end, ends) : tok.cp >= 0x90 && tok.cp <= 0x9f ? c1SequenceAfter(text, tok.cp, tok.end, ends) : null;
2807
+ if (!nested) {
2808
+ for (const [a, b] of V.contents) {
2809
+ const sub = text.slice(a, b);
2810
+ const inner = collectCandidates(sub, o, true);
2811
+ for (const iv of resolveCandidates(sub, inner.cands, inner.rawUnit, inner.rawDot, o.words ? pemRegions(inner.pemBlocksSeen) : [], inner.rawSoft, inner.rawPlain)) {
2812
+ raw.push({ start: a + iv.start, end: a + iv.end, kind: iv.kind, cls: 'nested', tokenEnd: a + iv.end, pretrimmed: true });
1702
2813
  }
1703
2814
  }
1704
- if (unit === null) {
1705
- i = next;
1706
- continue;
1707
- }
1708
- if (unit.content !== undefined) {
1709
- const [a, b] = unit.content;
1710
- const inner = text.slice(a, b);
1711
- const washed = washNets(inner, urls, words, true, blankTokens, trace);
1712
- if (washed.text !== inner) {
1713
- parts.push(text.slice(cursor, a), washed.text);
1714
- keptUpTo(cursor, a);
1715
- if (origin !== null) {
1716
- const w = washed.origin;
1717
- for (let x = 0; x < washed.text.length; x++) {
1718
- const s = w === undefined ? x : (w[x] ?? -1);
1719
- origin.push(s < 0 ? -1 : a + s);
1720
- }
1721
- }
1722
- cursor = b;
2815
+ for (const c of screen.contents) {
2816
+ const inner = collectCandidates(c.sub, o, true);
2817
+ for (const iv of resolveCandidates(c.sub, inner.cands, inner.rawUnit, inner.rawDot, o.words ? pemRegions(inner.pemBlocksSeen) : [], inner.rawSoft, inner.rawPlain)) {
2818
+ const start = c.rawAt(iv.start);
2819
+ const end = c.rawEndAt(iv.end - 1);
2820
+ if (end > start)
2821
+ raw.push({ start, end, kind: iv.kind, cls: 'nested', tokenEnd: end, pretrimmed: true });
1723
2822
  }
1724
2823
  }
1725
- i = Math.max(unit.end, i + 1);
1726
2824
  }
1727
- if (cursor === 0)
1728
- return { text };
1729
- parts.push(text.slice(cursor));
1730
- keptUpTo(cursor, text.length);
1731
- const joined = parts.join('');
1732
- return origin === null ? { text: joined } : { text: joined, origin: Int32Array.from(origin) };
2825
+ return { cands: raw, rawUnit: V.rawUnit, rawDot: screen.surfaces[0]?.rawDot ?? null, rawSoft: screen.rawSoft, rawPlain: screen.rawPlain, pemBlocksSeen };
1733
2826
  }
1734
- const MAX_SETTLE_ROUNDS = 4;
1735
- function washNets(text, urls, words, nested, blankTokens, trace) {
1736
- if (!urls && !words)
1737
- return { text };
1738
- let current = { text };
1739
- for (let round = 0; round < MAX_SETTLE_ROUNDS; round++) {
1740
- const next = washCredentialNetsOnce(current.text, urls, words, nested, blankTokens, trace);
1741
- if (next.text === current.text)
1742
- return current;
1743
- current = traced(next.text, composeOrigin(current.origin, next.origin));
2827
+ function scanOnce(text, o, trace) {
2828
+ const { cands, rawUnit, rawDot, rawSoft, rawPlain, pemBlocksSeen } = collectCandidates(text, o, false);
2829
+ const intervals = resolveCandidates(text, cands, rawUnit, rawDot, o.words ? pemRegions(pemBlocksSeen) : [], rawSoft, rawPlain);
2830
+ return materialize(text, intervals, trace);
2831
+ }
2832
+ const MAX_WASH_REWRITES = 8;
2833
+ function composeOrigin(first, second) {
2834
+ if (second === undefined)
2835
+ return first;
2836
+ if (first === undefined)
2837
+ return second;
2838
+ const out = new Int32Array(second.length);
2839
+ for (let k = 0; k < second.length; k++) {
2840
+ const s = second[k] ?? -1;
2841
+ out[k] = s < 0 ? -1 : (first[s] ?? -1);
1744
2842
  }
1745
- return current;
2843
+ return out;
1746
2844
  }
1747
- function washCredentialNets(text, urls, words, nested = false, blankTokens = 'off') {
1748
- return washNets(text, urls, words, nested, blankTokens, false).text;
2845
+ function scanAndMask(text, o, trace, stats) {
2846
+ if (!o.urls && !o.words) {
2847
+ if (stats !== undefined) {
2848
+ stats.passes = 0;
2849
+ stats.capped = false;
2850
+ }
2851
+ return { text };
2852
+ }
2853
+ let cur = scanOnce(text, o, trace);
2854
+ let passes = 1;
2855
+ let rewrites = cur.text !== text ? 1 : 0;
2856
+ let changed = rewrites > 0;
2857
+ while (changed && rewrites < MAX_WASH_REWRITES) {
2858
+ const next = scanOnce(cur.text, o, trace);
2859
+ passes++;
2860
+ changed = next.text !== cur.text;
2861
+ if (!changed)
2862
+ break;
2863
+ rewrites++;
2864
+ const origin = trace ? composeOrigin(cur.origin, next.origin) : undefined;
2865
+ cur = origin === undefined ? { text: next.text } : { text: next.text, origin };
2866
+ }
2867
+ if (stats !== undefined) {
2868
+ let capped = false;
2869
+ if (changed) {
2870
+ passes++;
2871
+ capped = scanOnce(cur.text, o, false).text !== cur.text;
2872
+ }
2873
+ stats.passes = passes;
2874
+ stats.capped = capped;
2875
+ }
2876
+ return cur;
1749
2877
  }
2878
+ const WASH_OPTIONS = Object.freeze({ urls: true, words: true, blanks: false, fold: false, face: null, escapeSpaces: null });
1750
2879
  export function washCredentials(text) {
1751
2880
  if (typeof text !== 'string')
1752
2881
  return text;
1753
- return washCredentialNets(text, true, true);
2882
+ return scanAndMask(text, WASH_OPTIONS, false).text;
1754
2883
  }
1755
2884
  const C0_ALL = '\\u0000-\\u001F';
1756
2885
  const C0_KEEP_LAYOUT = '\\u0000-\\u0008\\u000B-\\u001F';
@@ -2089,42 +3218,64 @@ function capDisplay(text, max, mark) {
2089
3218
  }
2090
3219
  const MAX_CAP_SETTLE_ROUNDS = 4;
2091
3220
  function runDisplaySteps(start, steps) {
2092
- const settle = (s) => {
2093
- let current = steps.paint(steps.wash(s));
2094
- if (steps.text(current) === steps.text(s))
2095
- return current;
2096
- for (let round = 1; round < MAX_SETTLE_ROUNDS; round++) {
2097
- const next = steps.paint(steps.wash(current));
2098
- if (steps.text(next) === steps.text(current))
2099
- return current;
2100
- current = next;
2101
- }
2102
- return current;
2103
- };
2104
- const settled = settle(start);
3221
+ const once = (s) => steps.paint(steps.wash(s));
3222
+ const settled = once(start);
2105
3223
  let out = steps.cap(settled);
2106
3224
  if (steps.text(out) === steps.text(settled))
2107
3225
  return out;
2108
3226
  for (let round = 0; round < MAX_CAP_SETTLE_ROUNDS; round++) {
2109
- const again = steps.cap(settle(out));
3227
+ const again = steps.cap(once(out));
2110
3228
  if (steps.text(again) === steps.text(out))
2111
3229
  break;
2112
3230
  out = again;
2113
3231
  }
2114
3232
  return out;
2115
3233
  }
3234
+ function escapeSpacesFor(o, face) {
3235
+ if (face.mark !== 'escape' || face.fold)
3236
+ return null;
3237
+ const set = new Set();
3238
+ if (o.controls !== false) {
3239
+ for (const c of [0x09, 0x0a, 0x0b, 0x0c, 0x0d])
3240
+ set.add(c);
3241
+ if (o.keepLayout === true) {
3242
+ set.delete(0x09);
3243
+ set.delete(0x0a);
3244
+ }
3245
+ }
3246
+ if (o.bidi !== false) {
3247
+ set.add(0x2028);
3248
+ set.add(0x2029);
3249
+ }
3250
+ return set.size === 0 ? null : set;
3251
+ }
3252
+ function scanOptionsOf(o, face) {
3253
+ return {
3254
+ urls: o.credentialUrls !== false,
3255
+ words: o.credentialWords !== false,
3256
+ blanks: face.keepBlanks === true,
3257
+ fold: face.fold,
3258
+ face,
3259
+ escapeSpaces: escapeSpacesFor(o, face),
3260
+ };
3261
+ }
3262
+ export function credentialWashPasses(text, opts = {}) {
3263
+ const input = typeof text === 'string' ? text : String(text ?? '');
3264
+ const stats = { passes: 0, capped: false };
3265
+ const scan = opts === null ? WASH_OPTIONS : scanOptionsOf(opts, publicFace(opts));
3266
+ scanAndMask(input, scan, false, stats);
3267
+ return { passes: stats.passes, capped: stats.capped };
3268
+ }
2116
3269
  function renderUntrusted(text, opts, ledger) {
2117
3270
  const input = typeof text === 'string' ? text : String(text ?? '');
2118
3271
  const o = opts ?? {};
2119
- const urls = o.credentialUrls !== false;
2120
- const words = o.credentialWords !== false;
2121
3272
  const face = publicFace(o);
2122
- const blankTokens = face.keepBlanks !== true ? 'off' : face.fold ? 'fold' : 'keep';
3273
+ const scan = scanOptionsOf(o, face);
2123
3274
  const pairBackslashes = o.escapeBackslashes === true && face.mark !== 'dot' && face.mark !== 'space';
2124
3275
  const book = ledger ?? (pairBackslashes ? { cps: [], lens: [] } : null);
2125
3276
  if (book === null) {
2126
3277
  return runDisplaySteps(input, {
2127
- wash: (s) => washCredentialNets(s, urls, words, false, blankTokens),
3278
+ wash: (s) => scanAndMask(s, scan, false).text,
2128
3279
  paint: (s) => applyDisplayFace(s, face),
2129
3280
  cap: (s) => capDisplay(s, o.max, face.mark),
2130
3281
  text: (s) => s,
@@ -2132,7 +3283,7 @@ function renderUntrusted(text, opts, ledger) {
2132
3283
  }
2133
3284
  const out = runDisplaySteps({ text: input, ids: new Int32Array(input.length).fill(-1) }, {
2134
3285
  wash: (t) => {
2135
- const r = washNets(t.text, urls, words, false, blankTokens, true);
3286
+ const r = scanAndMask(t.text, scan, true);
2136
3287
  return r.text === t.text ? t : { text: r.text, ids: carryIds(t.ids, r.origin) };
2137
3288
  },
2138
3289
  paint: (t) => faceTracked(t, face, book),