@sema-agent/client-core 0.86.0 → 0.88.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +128 -0
- package/README.md +18 -12
- package/dist/adapt/arms.js +9 -0
- package/dist/adapter/downstream/eventToSdkMessage.js +14 -1
- package/dist/adapter/downstream/terminalToSdkResult.js +4 -1
- package/dist/adapter/downstream/wiringManifestView.d.ts +2 -0
- package/dist/adapter/downstream/wiringManifestView.js +1 -0
- package/dist/adapter/runStream.js +75 -2
- package/dist/adapter/types.d.ts +3 -0
- package/dist/decideReceipt.d.ts +1 -1
- package/dist/decideReceipt.js +1 -1
- package/dist/detachWire.d.ts +6 -1
- package/dist/detachWire.js +22 -1
- package/dist/displayUntrusted.d.ts +4 -0
- package/dist/displayUntrusted.js +1657 -573
- package/dist/engineErrorCodes.d.ts +2 -0
- package/dist/engineErrorCodes.js +1 -0
- package/dist/engineNoticeCodes.d.ts +20 -0
- package/dist/engineNoticeCodes.js +71 -0
- package/dist/engineWireFor.d.ts +1 -0
- package/dist/engineWireFor.js +1 -1
- package/dist/gateOutcome.d.ts +4 -0
- package/dist/gateOutcome.js +5 -0
- package/dist/generated/engineFactTables.d.ts +1 -0
- package/dist/generated/engineFactTables.js +3 -0
- package/dist/generated/engineNoticeTables.js +2 -0
- package/dist/hitl/crashConverged.d.ts +2 -1
- package/dist/hitl/crashConverged.js +23 -3
- package/dist/hitl/hitlBridge.d.ts +4 -0
- package/dist/hitl/hitlBridge.js +6 -0
- package/dist/hitl/parkResolver.js +17 -12
- package/dist/hitl/planReviewWire.js +30 -10
- package/dist/hitl/suspendedReopen.d.ts +7 -0
- package/dist/hitl/suspendedReopen.js +38 -7
- package/dist/hitl/toolApprovalWire.js +4 -5
- package/dist/index.d.ts +7 -0
- package/dist/index.js +4 -0
- package/dist/promptDelivery.d.ts +20 -0
- package/dist/promptDelivery.js +153 -0
- package/dist/registryConflict.d.ts +3 -0
- package/dist/registryConflict.js +44 -0
- package/dist/request/taskRequest.d.ts +1 -0
- package/dist/request/taskRequest.js +25 -1
- package/dist/resumeRefusalCopy.d.ts +8 -6
- package/dist/resumeRefusalCopy.js +41 -19
- package/dist/rewindArchiveCapability.js +5 -0
- package/dist/seam.d.ts +12 -1
- package/dist/seam.js +7 -0
- package/dist/sessionMemoryErase.d.ts +76 -0
- package/dist/sessionMemoryErase.js +432 -0
- package/dist/subagent/engineSubagentResume.js +2 -1
- package/dist/toolHistoryMismatch.d.ts +63 -0
- package/dist/toolHistoryMismatch.js +425 -0
- package/dist/workflowClient.d.ts +3 -2
- package/dist/workflowClient.js +3 -3
- package/docs/INTEGRATION-CLIENTS.md +598 -13
- package/package.json +2 -2
package/dist/displayUntrusted.js
CHANGED
|
@@ -26,135 +26,6 @@ function pushEdit(edits, src, start, end, text) {
|
|
|
26
26
|
const UNIT_NONE = 0;
|
|
27
27
|
const UNIT_SEQUENCE = 1;
|
|
28
28
|
const UNIT_SINGLE = 2;
|
|
29
|
-
const NO_UNITS = () => UNIT_NONE;
|
|
30
|
-
function applyEdits(text, edits) {
|
|
31
|
-
if (edits.length === 0)
|
|
32
|
-
return text;
|
|
33
|
-
const parts = [];
|
|
34
|
-
let cursor = 0;
|
|
35
|
-
for (const e of edits) {
|
|
36
|
-
parts.push(text.slice(cursor, e.start), e.text);
|
|
37
|
-
cursor = e.end;
|
|
38
|
-
}
|
|
39
|
-
parts.push(text.slice(cursor));
|
|
40
|
-
return parts.join('');
|
|
41
|
-
}
|
|
42
|
-
class PlainTarget {
|
|
43
|
-
text;
|
|
44
|
-
origin;
|
|
45
|
-
trace;
|
|
46
|
-
constructor(text, trace = false) {
|
|
47
|
-
this.text = text;
|
|
48
|
-
this.trace = trace;
|
|
49
|
-
}
|
|
50
|
-
apply(edits) {
|
|
51
|
-
if (this.trace && edits.length > 0)
|
|
52
|
-
this.origin = editedOrigin(this.origin, this.text.length, edits);
|
|
53
|
-
this.text = applyEdits(this.text, edits);
|
|
54
|
-
}
|
|
55
|
-
unitAt() {
|
|
56
|
-
return UNIT_NONE;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
const traced = (text, origin) => (origin === undefined ? { text } : { text, origin });
|
|
60
|
-
function editedOrigin(prev, length, edits) {
|
|
61
|
-
let n = length;
|
|
62
|
-
for (const e of edits)
|
|
63
|
-
n += e.text.length - (e.end - e.start);
|
|
64
|
-
const out = new Int32Array(n);
|
|
65
|
-
let w = 0;
|
|
66
|
-
let cursor = 0;
|
|
67
|
-
const copy = (a, b) => {
|
|
68
|
-
for (let k = a; k < b; k++)
|
|
69
|
-
out[w++] = prev === undefined ? k : (prev[k] ?? -1);
|
|
70
|
-
};
|
|
71
|
-
for (const e of edits) {
|
|
72
|
-
copy(cursor, e.start);
|
|
73
|
-
out.fill(-1, w, w + e.text.length);
|
|
74
|
-
w += e.text.length;
|
|
75
|
-
cursor = e.end;
|
|
76
|
-
}
|
|
77
|
-
copy(cursor, length);
|
|
78
|
-
return out;
|
|
79
|
-
}
|
|
80
|
-
function composeOrigin(outer, inner) {
|
|
81
|
-
if (inner === undefined)
|
|
82
|
-
return outer;
|
|
83
|
-
if (outer === undefined)
|
|
84
|
-
return inner;
|
|
85
|
-
const out = new Int32Array(inner.length);
|
|
86
|
-
for (let k = 0; k < inner.length; k++) {
|
|
87
|
-
const s = inner[k] ?? -1;
|
|
88
|
-
out[k] = s < 0 ? -1 : (outer[s] ?? -1);
|
|
89
|
-
}
|
|
90
|
-
return out;
|
|
91
|
-
}
|
|
92
|
-
class TrackedTarget {
|
|
93
|
-
text;
|
|
94
|
-
src;
|
|
95
|
-
spans = [];
|
|
96
|
-
viewLength;
|
|
97
|
-
units;
|
|
98
|
-
constructor(text, viewLength, units) {
|
|
99
|
-
this.text = text;
|
|
100
|
-
this.viewLength = viewLength;
|
|
101
|
-
this.units = units;
|
|
102
|
-
this.src = new Int32Array(text.length);
|
|
103
|
-
for (let k = 0; k < text.length; k++)
|
|
104
|
-
this.src[k] = k;
|
|
105
|
-
}
|
|
106
|
-
unitAt(k) {
|
|
107
|
-
const s = this.src[k] ?? -1;
|
|
108
|
-
return s >= 0 && this.units !== undefined ? (this.units[s] ?? UNIT_NONE) : UNIT_NONE;
|
|
109
|
-
}
|
|
110
|
-
rangeStart(k) {
|
|
111
|
-
const s = this.src[k] ?? 0;
|
|
112
|
-
return s >= 0 ? s : (this.spans[2 * (-1 - s)] ?? 0);
|
|
113
|
-
}
|
|
114
|
-
rangeEnd(k) {
|
|
115
|
-
const s = this.src[k] ?? 0;
|
|
116
|
-
return s >= 0 ? s + 1 : (this.spans[2 * (-1 - s) + 1] ?? 0);
|
|
117
|
-
}
|
|
118
|
-
apply(edits) {
|
|
119
|
-
if (edits.length === 0)
|
|
120
|
-
return;
|
|
121
|
-
let length = this.text.length;
|
|
122
|
-
for (const e of edits)
|
|
123
|
-
length += e.text.length - (e.end - e.start);
|
|
124
|
-
const src = new Int32Array(length);
|
|
125
|
-
const parts = [];
|
|
126
|
-
let cursor = 0;
|
|
127
|
-
let w = 0;
|
|
128
|
-
for (const e of edits) {
|
|
129
|
-
src.set(this.src.subarray(cursor, e.start), w);
|
|
130
|
-
w += e.start - cursor;
|
|
131
|
-
parts.push(this.text.slice(cursor, e.start), e.text);
|
|
132
|
-
let a;
|
|
133
|
-
let b;
|
|
134
|
-
if (e.start === e.end) {
|
|
135
|
-
a = e.start < this.text.length ? this.rangeStart(e.start) : this.viewLength;
|
|
136
|
-
b = a;
|
|
137
|
-
}
|
|
138
|
-
else {
|
|
139
|
-
a = this.rangeStart(e.start);
|
|
140
|
-
b = this.rangeEnd(e.start);
|
|
141
|
-
for (let k = e.start + 1; k < e.end; k++) {
|
|
142
|
-
a = Math.min(a, this.rangeStart(k));
|
|
143
|
-
b = Math.max(b, this.rangeEnd(k));
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
const id = this.spans.length / 2;
|
|
147
|
-
this.spans.push(a, b);
|
|
148
|
-
src.fill(-1 - id, w, w + e.text.length);
|
|
149
|
-
w += e.text.length;
|
|
150
|
-
cursor = e.end;
|
|
151
|
-
}
|
|
152
|
-
src.set(this.src.subarray(cursor), w);
|
|
153
|
-
parts.push(this.text.slice(cursor));
|
|
154
|
-
this.text = parts.join('');
|
|
155
|
-
this.src = src;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
29
|
const SECRET_LITERAL_RE = /^(sk-|gh[opus]_|github_pat_|xox[baprse]-|xapp-|bearer\s|basic\s|eyJ[A-Za-z0-9_-]{10,}\.|AKIA[0-9A-Z]{12,}|AIza[0-9A-Za-z_-]{20,}|-----BEGIN[A-Z0-9 ]*PRIVATE KEY)/i;
|
|
159
30
|
const SCHEME_TOKEN_RE = /(?:^|[^A-Za-z0-9_-])(?:bearer|basic)[ \t]+\S{8,4096}/i;
|
|
160
31
|
const JWT_RE = /(?:^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{4,2048}/i;
|
|
@@ -181,6 +52,147 @@ export function looksLikeKnownSecret(s) {
|
|
|
181
52
|
}
|
|
182
53
|
return false;
|
|
183
54
|
}
|
|
55
|
+
const KIND_BY_TEXT = new Map([
|
|
56
|
+
[REDACTED_USERINFO, 'userinfo'],
|
|
57
|
+
[REDACTED_QUERY, 'query'],
|
|
58
|
+
[REDACTED_FRAGMENT, 'fragment'],
|
|
59
|
+
[REDACTED_SECRET, 'secret'],
|
|
60
|
+
]);
|
|
61
|
+
const TEXT_BY_KIND = {
|
|
62
|
+
userinfo: REDACTED_USERINFO,
|
|
63
|
+
query: REDACTED_QUERY,
|
|
64
|
+
fragment: REDACTED_FRAGMENT,
|
|
65
|
+
secret: REDACTED_SECRET,
|
|
66
|
+
};
|
|
67
|
+
const isStructuralKind = (kind) => kind !== 'secret';
|
|
68
|
+
const MARKER_WITH_DOTS = /^[. ]*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ]*)+$/;
|
|
69
|
+
function valueIsWholeMarker(S, start, end) {
|
|
70
|
+
const text = S.text;
|
|
71
|
+
let k = start;
|
|
72
|
+
while (k < end && (S.dropped(k) || text[k] === '.' || text[k] === ' '))
|
|
73
|
+
k++;
|
|
74
|
+
let m;
|
|
75
|
+
const retired = text.startsWith(RETIRED_MARKER, k);
|
|
76
|
+
if (retired)
|
|
77
|
+
m = k + RETIRED_MARKER.length;
|
|
78
|
+
else {
|
|
79
|
+
if (!text.startsWith(REDACTION_MARKER_PREFIX, k))
|
|
80
|
+
return false;
|
|
81
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
82
|
+
re.lastIndex = k;
|
|
83
|
+
if (!re.test(text))
|
|
84
|
+
return false;
|
|
85
|
+
m = re.lastIndex;
|
|
86
|
+
}
|
|
87
|
+
let j = m;
|
|
88
|
+
while (j < end && S.dropped(j))
|
|
89
|
+
j++;
|
|
90
|
+
if (text[j] === '@' && text.startsWith(REDACTED_USERINFO, k))
|
|
91
|
+
return false;
|
|
92
|
+
if (retired) {
|
|
93
|
+
while (j < end && (S.dropped(j) || S.painted(j) || text[j] === '.' || text[j] === ' '))
|
|
94
|
+
j++;
|
|
95
|
+
return j >= end || credentialWordStartsExactlyAt(text, j) || credentialPairAt(S, j);
|
|
96
|
+
}
|
|
97
|
+
while (j < end && (S.dropped(j) || S.painted(j)))
|
|
98
|
+
j++;
|
|
99
|
+
return j >= end || !isKeyWordChar(text.charCodeAt(j)) || credentialWordStartsExactlyAt(text, j) || credentialPairAt(S, j);
|
|
100
|
+
}
|
|
101
|
+
const CREDENTIAL_NAME = /(?:(?:access|refresh|id|client|api|x[-_]api|session|auth)[-_]?)?(?:token|key|secret|password|authorization|credential)s?|bearer|basic/iy;
|
|
102
|
+
const LABEL_SEPARATOR = /(?:\\*["'])?[\s\u115F\u1160\u3164\uFFA0\u2800]*[:=\uFF1A\uFF1D]/y;
|
|
103
|
+
const SCHEME_SEPARATOR = /[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]/y;
|
|
104
|
+
function credentialPairAt(S, at) {
|
|
105
|
+
const text = S.text;
|
|
106
|
+
const re = new RegExp(CREDENTIAL_NAME.source, 'iy');
|
|
107
|
+
re.lastIndex = at;
|
|
108
|
+
const m = re.exec(text);
|
|
109
|
+
if (m === null)
|
|
110
|
+
return false;
|
|
111
|
+
let k = at + m[0].length;
|
|
112
|
+
if (isKeyWordChar(text.charCodeAt(k)))
|
|
113
|
+
return false;
|
|
114
|
+
const skippable = (p) => S.dropped(p) || S.painted(p) || text[p] === '.' || text[p] === ' ';
|
|
115
|
+
const skipped = k < text.length && skippable(k);
|
|
116
|
+
while (k < text.length && skippable(k))
|
|
117
|
+
k++;
|
|
118
|
+
if (k >= text.length)
|
|
119
|
+
return true;
|
|
120
|
+
const scheme = /^(?:bearer|basic)$/i.test(m[0]);
|
|
121
|
+
if (!(scheme && skipped)) {
|
|
122
|
+
const sep = new RegExp((scheme ? SCHEME_SEPARATOR : LABEL_SEPARATOR).source, 'y');
|
|
123
|
+
sep.lastIndex = k;
|
|
124
|
+
if (!sep.test(text))
|
|
125
|
+
return false;
|
|
126
|
+
k = sep.lastIndex;
|
|
127
|
+
}
|
|
128
|
+
while (k < text.length && (skippable(k) || isRegexSpace(text.charCodeAt(k)) || text[k] === '"' || text[k] === "'" || text[k] === '\\'))
|
|
129
|
+
k++;
|
|
130
|
+
if (k >= text.length || isBareValueStop(text.charAt(k)))
|
|
131
|
+
return true;
|
|
132
|
+
if (text.startsWith(RETIRED_MARKER, k))
|
|
133
|
+
return true;
|
|
134
|
+
const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
135
|
+
mk.lastIndex = k;
|
|
136
|
+
return mk.test(text);
|
|
137
|
+
}
|
|
138
|
+
const MARKER_LEAD = /^[. ,;:!?']*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ,;:!?']*)+/;
|
|
139
|
+
function nextPairOrNameAt(v, at) {
|
|
140
|
+
const name = new RegExp(CREDENTIAL_NAME.source, 'iy');
|
|
141
|
+
name.lastIndex = at;
|
|
142
|
+
if (name.test(v)) {
|
|
143
|
+
let k = name.lastIndex;
|
|
144
|
+
while (k < v.length && TRAILING_PUNCTUATION.has(v.charAt(k)))
|
|
145
|
+
k++;
|
|
146
|
+
if (k >= v.length)
|
|
147
|
+
return true;
|
|
148
|
+
}
|
|
149
|
+
return credentialWordStartsExactlyAt(v, at);
|
|
150
|
+
}
|
|
151
|
+
function userinfoIsWholeMarker(part) {
|
|
152
|
+
const v = withoutSeparators(part);
|
|
153
|
+
if (v.replace(/^[. ]+/, '').startsWith(`${REDACTED_USERINFO}@`))
|
|
154
|
+
return true;
|
|
155
|
+
const lead = MARKER_LEAD.exec(v);
|
|
156
|
+
if (lead === null)
|
|
157
|
+
return false;
|
|
158
|
+
return lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length);
|
|
159
|
+
}
|
|
160
|
+
function slotIsWholeMarker(part, soft) {
|
|
161
|
+
if (soft !== undefined && markerPunctSoftGlue(part, soft))
|
|
162
|
+
return false;
|
|
163
|
+
const v = withoutSeparators(part);
|
|
164
|
+
let k = 0;
|
|
165
|
+
while (k < v.length && (v[k] === '.' || v[k] === ' '))
|
|
166
|
+
k++;
|
|
167
|
+
if (v.startsWith(RETIRED_MARKER, k)) {
|
|
168
|
+
const lead = MARKER_LEAD.exec(v);
|
|
169
|
+
return lead !== null && (lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length));
|
|
170
|
+
}
|
|
171
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
172
|
+
re.lastIndex = k;
|
|
173
|
+
if (!re.test(v))
|
|
174
|
+
return false;
|
|
175
|
+
let j = re.lastIndex;
|
|
176
|
+
while (j < v.length && (v[j] === '.' || v[j] === ' '))
|
|
177
|
+
j++;
|
|
178
|
+
return j >= v.length || !isKeyWordChar(v.charCodeAt(j)) || nextPairOrNameAt(v, j);
|
|
179
|
+
}
|
|
180
|
+
function markerPunctSoftGlue(part, soft) {
|
|
181
|
+
let k = 0;
|
|
182
|
+
while (k < part.length && (part[k] === VIEW_SEPARATOR || part[k] === '.' || part[k] === ' '))
|
|
183
|
+
k++;
|
|
184
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
185
|
+
re.lastIndex = k;
|
|
186
|
+
if (!re.test(part))
|
|
187
|
+
return false;
|
|
188
|
+
let j = re.lastIndex;
|
|
189
|
+
while (j < part.length && part[j] !== VIEW_SEPARATOR && !soft(j) && TRAILING_PUNCTUATION.has(part.charAt(j)))
|
|
190
|
+
j++;
|
|
191
|
+
const s0 = j;
|
|
192
|
+
while (j < part.length && (part[j] === VIEW_SEPARATOR || soft(j)))
|
|
193
|
+
j++;
|
|
194
|
+
return j > s0 && j < part.length && isKeyWordChar(part.charCodeAt(j)) && !nextPairOrNameAt(part, j);
|
|
195
|
+
}
|
|
184
196
|
const TOKEN_STOP = new Set(['"', '<', '>', '`', '|', '\\', '^', '{', '}']);
|
|
185
197
|
const TRAILING_PUNCTUATION = new Set(['.', ',', ';', ':', '!', '?', "'"]);
|
|
186
198
|
const VALUE_BOUNDARY_TAIL = /[?#&=;]$/;
|
|
@@ -188,6 +200,10 @@ const isWhitespaceOrControl = (ch) => {
|
|
|
188
200
|
const c = ch.charCodeAt(0);
|
|
189
201
|
return c <= 0x20 || c === 0x7f;
|
|
190
202
|
};
|
|
203
|
+
const endsUrlFragmentAt = (s, at) => {
|
|
204
|
+
const ch = s.charAt(at);
|
|
205
|
+
return ch !== '' && (isWhitespaceOrControl(ch) || TOKEN_STOP.has(ch));
|
|
206
|
+
};
|
|
191
207
|
const isSchemeChar = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch === '+' || ch === '.' || ch === '-';
|
|
192
208
|
const isAlpha = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
|
|
193
209
|
const isHexDigit = (ch) => (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
|
|
@@ -226,7 +242,8 @@ function schemelessHostEnd(text, from) {
|
|
|
226
242
|
if (d > k + 1)
|
|
227
243
|
k = d;
|
|
228
244
|
}
|
|
229
|
-
|
|
245
|
+
const c = text.charCodeAt(k);
|
|
246
|
+
return endsSchemelessHost(c) || c === 0x27 ? k : -1;
|
|
230
247
|
}
|
|
231
248
|
function valueCoreForJudging(value) {
|
|
232
249
|
if (value.length === 0 || isMarker(value))
|
|
@@ -240,38 +257,49 @@ function isKnownSecretSegment(value) {
|
|
|
240
257
|
const core = valueCoreForJudging(value);
|
|
241
258
|
return core.length > 0 && looksLikeKnownSecret(core);
|
|
242
259
|
}
|
|
243
|
-
function
|
|
260
|
+
function queryCandidates(src, base, query, edits, soft) {
|
|
244
261
|
let pos = base;
|
|
245
262
|
for (const part of query.split('&')) {
|
|
246
263
|
if (part.length > 0) {
|
|
247
264
|
const eq = part.indexOf('=');
|
|
248
265
|
if (eq === -1) {
|
|
249
|
-
|
|
266
|
+
const at = pos;
|
|
267
|
+
if (!slotIsWholeMarker(part, soft && ((i) => soft(at + i))))
|
|
250
268
|
pushEdit(edits, src, pos, pos + part.length, REDACTED_QUERY);
|
|
251
269
|
}
|
|
252
270
|
else {
|
|
253
271
|
const value = part.slice(eq + 1);
|
|
254
|
-
|
|
272
|
+
const at = pos + eq + 1;
|
|
273
|
+
if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
|
|
255
274
|
pushEdit(edits, src, pos + eq + 1, pos + part.length, REDACTED_QUERY);
|
|
256
275
|
}
|
|
257
276
|
}
|
|
258
277
|
pos += part.length + 1;
|
|
259
278
|
}
|
|
260
279
|
}
|
|
261
|
-
function
|
|
280
|
+
function pathCandidates(src, base, path, opaqueSegments, edits, soft) {
|
|
262
281
|
let pos = base;
|
|
263
282
|
for (const segment of path.split('/')) {
|
|
264
283
|
const semi = segment.indexOf(';');
|
|
265
284
|
const head = semi === -1 ? segment : segment.slice(0, semi);
|
|
266
|
-
if (opaqueSegments && isKnownSecretSegment(head))
|
|
267
|
-
|
|
285
|
+
if (opaqueSegments && isKnownSecretSegment(head)) {
|
|
286
|
+
let a = 0;
|
|
287
|
+
let b = head.length;
|
|
288
|
+
while (a < b && !isKeyWordChar(head.charCodeAt(a)))
|
|
289
|
+
a++;
|
|
290
|
+
while (b > a && !isKeyWordChar(head.charCodeAt(b - 1)))
|
|
291
|
+
b--;
|
|
292
|
+
if (b > a)
|
|
293
|
+
pushEdit(edits, src, pos + a, pos + b, REDACTED_SECRET);
|
|
294
|
+
}
|
|
268
295
|
if (semi !== -1) {
|
|
269
296
|
let ppos = pos + semi + 1;
|
|
270
297
|
for (const part of segment.slice(semi + 1).split(';')) {
|
|
271
298
|
const eq = part.indexOf('=');
|
|
272
299
|
if (eq !== -1) {
|
|
273
300
|
const value = part.slice(eq + 1);
|
|
274
|
-
|
|
301
|
+
const at = ppos + eq + 1;
|
|
302
|
+
if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
|
|
275
303
|
pushEdit(edits, src, ppos + eq + 1, ppos + part.length, REDACTED_QUERY);
|
|
276
304
|
}
|
|
277
305
|
ppos += part.length + 1;
|
|
@@ -280,7 +308,7 @@ function pathEdits(src, base, path, opaqueSegments, edits) {
|
|
|
280
308
|
pos += segment.length + 1;
|
|
281
309
|
}
|
|
282
310
|
}
|
|
283
|
-
function
|
|
311
|
+
function urlTokenCandidatesAt(src, start, end, edits, soft) {
|
|
284
312
|
const token = src.slice(start, end);
|
|
285
313
|
const schemeEnd = token.indexOf('://') + '://'.length;
|
|
286
314
|
const scheme = token.slice(0, schemeEnd);
|
|
@@ -290,23 +318,23 @@ function urlTokenEditsAt(src, start, end, edits) {
|
|
|
290
318
|
const tailStart = schemeEnd + authority.length;
|
|
291
319
|
const tail = token.slice(tailStart);
|
|
292
320
|
const at = authority.lastIndexOf('@');
|
|
293
|
-
if (at !== -1)
|
|
321
|
+
if (at !== -1 && !userinfoIsWholeMarker(authority.slice(0, at)))
|
|
294
322
|
pushEdit(edits, src, start + schemeEnd, start + schemeEnd + at, REDACTED_USERINFO);
|
|
295
323
|
const hash = tail.indexOf('#');
|
|
296
324
|
const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
|
|
297
325
|
const question = beforeHash.indexOf('?');
|
|
298
326
|
const path = question === -1 ? beforeHash : beforeHash.slice(0, question);
|
|
299
|
-
|
|
327
|
+
pathCandidates(src, start + tailStart, path, !/^file:/i.test(scheme), edits, soft);
|
|
300
328
|
if (question !== -1)
|
|
301
|
-
|
|
329
|
+
queryCandidates(src, start + tailStart + question + 1, beforeHash.slice(question + 1), edits, soft);
|
|
302
330
|
if (hash !== -1) {
|
|
303
331
|
const rawFragment = tail.slice(hash + 1);
|
|
304
|
-
|
|
332
|
+
const at = start + tailStart + hash + 1;
|
|
333
|
+
if (rawFragment.length > 0 && !slotIsWholeMarker(rawFragment, soft && ((i) => soft(at + i))))
|
|
305
334
|
pushEdit(edits, src, start + tailStart + hash + 1, end, REDACTED_FRAGMENT);
|
|
306
335
|
}
|
|
307
336
|
}
|
|
308
|
-
function
|
|
309
|
-
const edits = [];
|
|
337
|
+
function urlCandidates(text, out, regions, unitPos = () => false, soft) {
|
|
310
338
|
let mark = text.indexOf('://');
|
|
311
339
|
while (mark !== -1) {
|
|
312
340
|
let start = mark;
|
|
@@ -319,10 +347,33 @@ function urlTokenEdits(text) {
|
|
|
319
347
|
continue;
|
|
320
348
|
}
|
|
321
349
|
let end = mark + 3;
|
|
350
|
+
let seenPathSep = false;
|
|
351
|
+
let atAhead = -2;
|
|
352
|
+
const colons = new AuthorityColons(text, mark + 3);
|
|
353
|
+
let firstVisible = -1;
|
|
354
|
+
const adjacent = (at) => {
|
|
355
|
+
if (firstVisible === -1) {
|
|
356
|
+
firstVisible = mark + 3;
|
|
357
|
+
while (firstVisible < text.length && unitPos(firstVisible))
|
|
358
|
+
firstVisible++;
|
|
359
|
+
}
|
|
360
|
+
return at <= firstVisible;
|
|
361
|
+
};
|
|
322
362
|
while (end < text.length) {
|
|
323
363
|
const ch = text.charAt(end);
|
|
324
|
-
if (isWhitespaceOrControl(ch)
|
|
364
|
+
if (isWhitespaceOrControl(ch))
|
|
325
365
|
break;
|
|
366
|
+
if (TOKEN_STOP.has(ch)) {
|
|
367
|
+
if (!seenPathSep) {
|
|
368
|
+
if (atAhead === -2)
|
|
369
|
+
atAhead = atInAuthority(text, end);
|
|
370
|
+
if (atAhead > end && (adjacent(end) || colons.colonBefore(end))) {
|
|
371
|
+
end++;
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
break;
|
|
376
|
+
}
|
|
326
377
|
const c = ch.charCodeAt(0);
|
|
327
378
|
if (ch === VIEW_SEPARATOR || isUnicodeBlank(c)) {
|
|
328
379
|
let p = end;
|
|
@@ -333,6 +384,10 @@ function urlTokenEdits(text) {
|
|
|
333
384
|
end = p;
|
|
334
385
|
continue;
|
|
335
386
|
}
|
|
387
|
+
if ((ch === ',' || ch === ';') && nextPairBeyondFragmentAt(text, end + 1))
|
|
388
|
+
break;
|
|
389
|
+
if (ch === '/' || ch === '?' || ch === '#')
|
|
390
|
+
seenPathSep = true;
|
|
336
391
|
end++;
|
|
337
392
|
}
|
|
338
393
|
while (end > mark + 3 && text.charAt(end - 1) === VIEW_SEPARATOR)
|
|
@@ -342,10 +397,67 @@ function urlTokenEdits(text) {
|
|
|
342
397
|
runStart--;
|
|
343
398
|
if (runStart < end && !VALUE_BOUNDARY_TAIL.test(text.slice(start, runStart)))
|
|
344
399
|
end = runStart;
|
|
345
|
-
|
|
400
|
+
const tokenEdits = [];
|
|
401
|
+
urlTokenCandidatesAt(text, start, end, tokenEdits, soft);
|
|
402
|
+
editsToCandidates(tokenEdits, 'url', out, end);
|
|
403
|
+
const token = text.slice(start, end);
|
|
404
|
+
const schemeEnd = token.indexOf('://') + 3;
|
|
405
|
+
const rest = token.slice(schemeEnd);
|
|
406
|
+
const authorityEnd = rest.search(/[/?#]/);
|
|
407
|
+
const tailStart = schemeEnd + (authorityEnd === -1 ? rest.length : authorityEnd);
|
|
408
|
+
const tail = token.slice(tailStart);
|
|
409
|
+
const hash = tail.indexOf('#');
|
|
410
|
+
const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
|
|
411
|
+
const question = beforeHash.indexOf('?');
|
|
412
|
+
if (question !== -1)
|
|
413
|
+
regions.push([start + tailStart + question + 1, start + tailStart + beforeHash.length, 'query']);
|
|
414
|
+
if (hash !== -1 && hash + 1 < tail.length)
|
|
415
|
+
regions.push([start + tailStart + hash + 1, end, 'fragment']);
|
|
346
416
|
mark = text.indexOf('://', end > mark + 3 ? end : mark + 3);
|
|
347
417
|
}
|
|
348
|
-
|
|
418
|
+
}
|
|
419
|
+
function atInAuthority(text, from) {
|
|
420
|
+
for (let p = from; p < text.length; p++) {
|
|
421
|
+
const ch = text.charAt(p);
|
|
422
|
+
if (isWhitespaceOrControl(ch) || ch === '/' || ch === '?' || ch === '#')
|
|
423
|
+
return -1;
|
|
424
|
+
if (ch === '@')
|
|
425
|
+
return p;
|
|
426
|
+
}
|
|
427
|
+
return -1;
|
|
428
|
+
}
|
|
429
|
+
class AuthorityColons {
|
|
430
|
+
text;
|
|
431
|
+
scanned;
|
|
432
|
+
colons = 0;
|
|
433
|
+
bracketEnd = -1;
|
|
434
|
+
constructor(text, from) {
|
|
435
|
+
this.text = text;
|
|
436
|
+
this.scanned = from;
|
|
437
|
+
if (text.charCodeAt(from) === 0x5b) {
|
|
438
|
+
let k = from + 1;
|
|
439
|
+
while (k < text.length && isIpv6Char(text.charCodeAt(k)))
|
|
440
|
+
k++;
|
|
441
|
+
if (text.charCodeAt(k) === 0x5d)
|
|
442
|
+
this.bracketEnd = k;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
colonBefore(to) {
|
|
446
|
+
for (; this.scanned < to; this.scanned++) {
|
|
447
|
+
const c = this.text.charCodeAt(this.scanned);
|
|
448
|
+
if (c === 0xab) {
|
|
449
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
450
|
+
re.lastIndex = this.scanned;
|
|
451
|
+
if (re.test(this.text) && re.lastIndex <= to) {
|
|
452
|
+
this.scanned = re.lastIndex - 1;
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
if (c === 0x3a && this.scanned > this.bracketEnd)
|
|
457
|
+
this.colons++;
|
|
458
|
+
}
|
|
459
|
+
return this.colons > 0;
|
|
460
|
+
}
|
|
349
461
|
}
|
|
350
462
|
const MARKER_ANYWHERE = /\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb/g;
|
|
351
463
|
function markerSpans(text) {
|
|
@@ -357,18 +469,123 @@ function markerSpans(text) {
|
|
|
357
469
|
spans.push(hit.index, hit.index + hit[0].length);
|
|
358
470
|
return spans;
|
|
359
471
|
}
|
|
360
|
-
|
|
361
|
-
|
|
472
|
+
const USERNAME_DELIMITERS = new Set(['"', "'", '`', '(', ')', '[', ']', '{', '}', '<', '>', '|', '\\', '^', ',', ';', '=']);
|
|
473
|
+
const PASSWORD_FORBIDDEN = new Set(['"', '`', '<', '>']);
|
|
474
|
+
const forbiddenCloser = (ch) => (ch === '<' ? '>' : ch === '"' || ch === '`' ? ch : '');
|
|
475
|
+
const isBareUserChar = (c) => (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) ||
|
|
476
|
+
c === 0x2e || c === 0x5f || c === 0x2d || c === 0x7e || c === 0x25 || c === 0x2b;
|
|
477
|
+
function userinfoCandidates(S, out, words = true) {
|
|
478
|
+
const text = S.text;
|
|
479
|
+
const unitAt = (k) => S.unitAt(k);
|
|
362
480
|
const markers = markerSpans(text);
|
|
363
|
-
|
|
481
|
+
const markerStartAt = new Map();
|
|
482
|
+
const markerStartByEnd = markers.length === 0 ? null : new Map();
|
|
483
|
+
const markerEndAt = markers.length === 0 ? null : new Int32Array(text.length).fill(-1);
|
|
484
|
+
for (let m = 0; m + 1 < markers.length; m += 2) {
|
|
485
|
+
const a = markers[m] ?? 0;
|
|
486
|
+
const b = markers[m + 1] ?? 0;
|
|
487
|
+
markerStartAt.set(a, b);
|
|
488
|
+
markerStartByEnd?.set(b, a);
|
|
489
|
+
if (markerEndAt !== null)
|
|
490
|
+
for (let k = a; k < b; k++)
|
|
491
|
+
markerEndAt[k] = b;
|
|
492
|
+
}
|
|
493
|
+
let markerFrom = 0;
|
|
494
|
+
const inMarker = (k) => (markerEndAt === null ? -1 : (markerEndAt[k] ?? -1));
|
|
495
|
+
let forbiddenNext = null;
|
|
496
|
+
const nextForbidden = (k) => {
|
|
497
|
+
if (forbiddenNext === null) {
|
|
498
|
+
const n = text.length;
|
|
499
|
+
const t = new Int32Array(n + 1).fill(n);
|
|
500
|
+
for (let x = n - 1; x >= 0; x--)
|
|
501
|
+
t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x)) ? x : (t[x + 1] ?? n);
|
|
502
|
+
forbiddenNext = t;
|
|
503
|
+
}
|
|
504
|
+
return forbiddenNext[k] ?? text.length;
|
|
505
|
+
};
|
|
506
|
+
let forbiddenPrev = null;
|
|
507
|
+
const prevForbidden = (k) => {
|
|
508
|
+
if (forbiddenPrev === null) {
|
|
509
|
+
const n = text.length;
|
|
510
|
+
const t = new Int32Array(n + 1).fill(-1);
|
|
511
|
+
for (let x = 1; x <= n; x++)
|
|
512
|
+
t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x - 1)) ? x - 1 : (t[x - 1] ?? -1);
|
|
513
|
+
forbiddenPrev = t;
|
|
514
|
+
}
|
|
515
|
+
return forbiddenPrev[k] ?? -1;
|
|
516
|
+
};
|
|
517
|
+
let bareStopNext = null;
|
|
518
|
+
const nextBareStop = (k) => {
|
|
519
|
+
if (bareStopNext === null) {
|
|
520
|
+
const n = text.length;
|
|
521
|
+
const t = new Int32Array(n + 1).fill(n);
|
|
522
|
+
for (let x = n - 1; x >= 0; x--)
|
|
523
|
+
t[x] = isBareValueStop(text.charAt(x)) ? x : (t[x + 1] ?? n);
|
|
524
|
+
bareStopNext = t;
|
|
525
|
+
}
|
|
526
|
+
return bareStopNext[k] ?? text.length;
|
|
527
|
+
};
|
|
528
|
+
const labelValueReaches = (from, at) => {
|
|
529
|
+
let i = from;
|
|
530
|
+
while (i < at && text.charCodeAt(i) === 0x5c)
|
|
531
|
+
i++;
|
|
532
|
+
const q = text[i];
|
|
533
|
+
if (q === '"' || q === "'") {
|
|
534
|
+
if (i > from) {
|
|
535
|
+
const close = text.indexOf('\\' + q, i + 1);
|
|
536
|
+
return close === -1 || close >= at;
|
|
537
|
+
}
|
|
538
|
+
for (let j = i + 1; j < at; j++) {
|
|
539
|
+
if (text[j] === '\\') {
|
|
540
|
+
j++;
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
if (text[j] === q)
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
return true;
|
|
547
|
+
}
|
|
548
|
+
return nextBareStop(from) >= at;
|
|
549
|
+
};
|
|
364
550
|
let floor = 0;
|
|
365
|
-
for (let at = text.indexOf('@')
|
|
551
|
+
for (let at = text.indexOf('@'); at !== -1; at = text.indexOf('@', at + 1)) {
|
|
366
552
|
let hostEnd = schemelessHostEnd(text, at + 1);
|
|
367
553
|
const hidden = hostEnd === -1;
|
|
368
554
|
if (hidden)
|
|
369
555
|
hostEnd = hiddenHostEnd(text, at + 1, unitAt);
|
|
370
556
|
if (hostEnd === -1)
|
|
371
557
|
continue;
|
|
558
|
+
let shaped = null;
|
|
559
|
+
const hostShaped = () => {
|
|
560
|
+
if (shaped === null) {
|
|
561
|
+
const c = text.charCodeAt(hostEnd);
|
|
562
|
+
shaped = !(c === 0x3c || c === 0x3e || c === 0x7c || c === 0x5c || c === 0x5e || c === 0x7b || c === 0x7d);
|
|
563
|
+
if (!shaped)
|
|
564
|
+
for (let k = at + 1; k < hostEnd && !shaped; k++) {
|
|
565
|
+
const h = text.charCodeAt(k);
|
|
566
|
+
if (h === 0x2e || h === 0x3a || h === 0x5b)
|
|
567
|
+
shaped = true;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
return shaped;
|
|
571
|
+
};
|
|
572
|
+
const keptDespiteForbidden = (userStart, colon) => {
|
|
573
|
+
if (labelEndsBefore(text, userStart, floor, '\'"`'))
|
|
574
|
+
return 'label';
|
|
575
|
+
if (hidden || userStart >= colon)
|
|
576
|
+
return null;
|
|
577
|
+
if (labelEndsAt(text, colon + 1) || schemeWordEndsAt(text, colon + 1))
|
|
578
|
+
return words && labelValueReaches(colon + 1, at) ? null : 'label';
|
|
579
|
+
for (let k = userStart; k < colon; k++)
|
|
580
|
+
if (!isBareUserChar(text.charCodeAt(k)))
|
|
581
|
+
return null;
|
|
582
|
+
const q = prevForbidden(at);
|
|
583
|
+
const closer = forbiddenCloser(text.charAt(q));
|
|
584
|
+
if (closer !== '' && q < at - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
|
|
585
|
+
return null;
|
|
586
|
+
return hostShaped() ? 'bare' : null;
|
|
587
|
+
};
|
|
588
|
+
const refused = [];
|
|
372
589
|
let runStart = at;
|
|
373
590
|
while (runStart > floor && (!breaksUserinfo(text.charCodeAt(runStart - 1)) || (hidden && unitAt(runStart - 1) !== UNIT_NONE)))
|
|
374
591
|
runStart--;
|
|
@@ -377,67 +594,135 @@ function schemelessUserinfoEdits(text, unitAt = NO_UNITS) {
|
|
|
377
594
|
runStart--;
|
|
378
595
|
}
|
|
379
596
|
let found = -1;
|
|
380
|
-
let
|
|
381
|
-
let
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
597
|
+
let lo = runStart;
|
|
598
|
+
let keyClose = -2;
|
|
599
|
+
let wholeMarkerAt = -1;
|
|
600
|
+
for (let mk = markerFrom; mk + 1 < markers.length && (markers[mk] ?? 0) < at; mk += 2) {
|
|
601
|
+
const a = markers[mk] ?? 0;
|
|
602
|
+
const b = markers[mk + 1] ?? 0;
|
|
603
|
+
markerFrom = mk;
|
|
604
|
+
if (a < runStart)
|
|
605
|
+
continue;
|
|
606
|
+
let p = b;
|
|
607
|
+
while (p < at && (text[p] === VIEW_SEPARATOR || text[p] === '.'))
|
|
608
|
+
p++;
|
|
609
|
+
if ((p >= at || !isKeyWordChar(text.charCodeAt(p))) && labelWashedBefore(text, a, runStart))
|
|
610
|
+
wholeMarkerAt = a;
|
|
611
|
+
}
|
|
612
|
+
let markerTail = at;
|
|
613
|
+
for (;;) {
|
|
614
|
+
while (markerTail > runStart && (text[markerTail - 1] === VIEW_SEPARATOR || text[markerTail - 1] === '.' || text[markerTail - 1] === ' '))
|
|
615
|
+
markerTail--;
|
|
616
|
+
const ms = markerTail > runStart ? (markerStartByEnd?.get(markerTail) ?? -1) : -1;
|
|
617
|
+
if (ms === -1 || ms < runStart)
|
|
618
|
+
break;
|
|
619
|
+
markerTail = ms;
|
|
620
|
+
}
|
|
621
|
+
let sFrom = runStart;
|
|
622
|
+
for (let colon = runStart; colon < at && found === -1; colon++) {
|
|
623
|
+
const me = inMarker(colon);
|
|
624
|
+
if (me !== -1) {
|
|
625
|
+
colon = me - 1;
|
|
626
|
+
continue;
|
|
387
627
|
}
|
|
628
|
+
const ch = text.charAt(colon);
|
|
629
|
+
if (ch === '[') {
|
|
630
|
+
let k = colon + 1;
|
|
631
|
+
while (k < at && isIpv6Char(text.charCodeAt(k)))
|
|
632
|
+
k++;
|
|
633
|
+
if (k < at && text.charCodeAt(k) === 0x5d && k > colon + 1) {
|
|
634
|
+
colon = k;
|
|
635
|
+
lo = k + 1;
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (USERNAME_DELIMITERS.has(ch)) {
|
|
640
|
+
lo = colon + 1;
|
|
641
|
+
continue;
|
|
642
|
+
}
|
|
643
|
+
if (text.charCodeAt(colon) !== 0x3a)
|
|
644
|
+
continue;
|
|
388
645
|
if (colon + 1 >= at)
|
|
389
646
|
break;
|
|
390
|
-
|
|
391
|
-
|
|
647
|
+
const leftOfColon = colon > 0 ? text.charAt(colon - 1) : '';
|
|
648
|
+
if ((leftOfColon === "'" || leftOfColon === '"' || leftOfColon === '`') && colon > 1 && isKeyWordChar(text.charCodeAt(colon - 2))) {
|
|
649
|
+
lo = colon + 1;
|
|
650
|
+
keyClose = colon;
|
|
392
651
|
continue;
|
|
393
652
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
s = colon + 1;
|
|
653
|
+
const forbidden = nextForbidden(colon + 1) < at;
|
|
654
|
+
if (!forbidden && text[colon + 1] === "'" && text[hostEnd] === "'") {
|
|
655
|
+
lo = colon + 1;
|
|
398
656
|
continue;
|
|
399
657
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
658
|
+
if (wholeMarkerAt > colon && colon + 1 < markerTail) {
|
|
659
|
+
lo = colon + 1;
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
if (wholeMarkerAt === colon + 1 && markerTail === colon + 1 && text.startsWith(REDACTED_USERINFO, colon + 1)) {
|
|
663
|
+
lo = colon + 1;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
let userStart = -1;
|
|
667
|
+
if (lo === colon)
|
|
668
|
+
userStart = colon;
|
|
669
|
+
else {
|
|
670
|
+
for (let s = Math.max(lo, sFrom); s < colon; s++) {
|
|
671
|
+
if (markerStartAt.has(s)) {
|
|
672
|
+
userStart = s;
|
|
673
|
+
break;
|
|
674
|
+
}
|
|
675
|
+
const prev = s === 0 ? -1 : text.charCodeAt(s - 1);
|
|
676
|
+
const midPair = prev >= 0xd800 && prev <= 0xdbff && (text.charCodeAt(s) & 0xfc00) === 0xdc00;
|
|
677
|
+
if ((prev === -1 || !blocksUserinfoStart(prev) || s === keyClose + 1) && !midPair) {
|
|
678
|
+
userStart = s;
|
|
679
|
+
break;
|
|
680
|
+
}
|
|
681
|
+
const inner = inMarker(s);
|
|
682
|
+
if (inner !== -1)
|
|
683
|
+
s = inner - 1;
|
|
684
|
+
}
|
|
685
|
+
if (userStart === -1)
|
|
686
|
+
sFrom = colon;
|
|
687
|
+
}
|
|
688
|
+
const kept = userStart === -1 || !forbidden ? null : keptDespiteForbidden(userStart, colon);
|
|
689
|
+
if (userStart === -1 || (forbidden && kept === null)) {
|
|
690
|
+
if (forbidden) {
|
|
691
|
+
lo = colon + 1;
|
|
692
|
+
if (userStart !== -1)
|
|
693
|
+
refused.push(colon, userStart);
|
|
694
|
+
}
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
found = userStart;
|
|
698
|
+
if (kept === 'bare') {
|
|
699
|
+
for (let r = 0; r + 1 < refused.length; r += 2) {
|
|
700
|
+
if (nextForbidden((refused[r] ?? 0) + 1) >= userStart) {
|
|
701
|
+
found = refused[r + 1] ?? userStart;
|
|
702
|
+
break;
|
|
703
|
+
}
|
|
704
|
+
}
|
|
405
705
|
}
|
|
406
|
-
s++;
|
|
407
706
|
}
|
|
408
707
|
if (found === -1)
|
|
409
708
|
continue;
|
|
410
|
-
|
|
411
|
-
continue;
|
|
412
|
-
pushEdit(edits, text, found, at, REDACTED_USERINFO);
|
|
709
|
+
out.push({ a: found, b: at, kind: 'userinfo', cls: hidden ? 'userinfo-hidden' : 'structural', te: hostEnd });
|
|
413
710
|
floor = hostEnd;
|
|
414
711
|
}
|
|
415
|
-
return edits;
|
|
416
712
|
}
|
|
417
|
-
function
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
let
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
}
|
|
430
|
-
function credentialWordEndsFrom(seg, from) {
|
|
431
|
-
for (const word of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
|
|
432
|
-
const re = new RegExp(word.source, 'gi');
|
|
433
|
-
for (let hit = re.exec(seg); hit !== null; hit = re.exec(seg)) {
|
|
434
|
-
if (hit.index + hit[0].length >= from)
|
|
435
|
-
return true;
|
|
436
|
-
if (hit[0].length === 0)
|
|
437
|
-
re.lastIndex++;
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
return false;
|
|
713
|
+
function labelWashedBefore(text, a, floor) {
|
|
714
|
+
return labelEndsBefore(text, a, floor, "'");
|
|
715
|
+
}
|
|
716
|
+
function labelEndsBefore(text, a, floor, quotes) {
|
|
717
|
+
let e = a;
|
|
718
|
+
while (e > floor && text[e - 1] === VIEW_SEPARATOR)
|
|
719
|
+
e--;
|
|
720
|
+
if (e > floor && quotes.includes(text[e - 1] ?? '\u0000'))
|
|
721
|
+
e--;
|
|
722
|
+
const w = text.slice(Math.max(floor, e - 96), e);
|
|
723
|
+
if (w === '')
|
|
724
|
+
return false;
|
|
725
|
+
return new RegExp(`(?:${SECRET_LABELLED_WORD.source})$`, 'i').test(w) || new RegExp(`(?:${SECRET_SCHEME_WORD.source})$`, 'i').test(w);
|
|
441
726
|
}
|
|
442
727
|
function unitRunEnd(text, from, unitAt) {
|
|
443
728
|
let k = from;
|
|
@@ -459,16 +744,6 @@ function hiddenHostEnd(text, from, unitAt) {
|
|
|
459
744
|
const c = text.charCodeAt(k);
|
|
460
745
|
return endsSchemelessHost(c) || c === 0x40 ? k : -1;
|
|
461
746
|
}
|
|
462
|
-
const MAX_URL_WASH_PASSES = 4;
|
|
463
|
-
function washUrlNets(target) {
|
|
464
|
-
for (let pass = 0; pass < MAX_URL_WASH_PASSES; pass++) {
|
|
465
|
-
target.apply(urlTokenEdits(target.text));
|
|
466
|
-
const handedOff = schemelessUserinfoEdits(target.text, (k) => target.unitAt(k));
|
|
467
|
-
target.apply(handedOff);
|
|
468
|
-
if (handedOff.length === 0)
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
747
|
const NAME_BLANK_RANGES = [
|
|
473
748
|
[0x00a0, 0x00a0],
|
|
474
749
|
[0x1680, 0x1680],
|
|
@@ -710,6 +985,23 @@ function credentialWordStartsExactlyAt(text, at) {
|
|
|
710
985
|
function credentialWordStartsNear(text, at) {
|
|
711
986
|
return credentialWordStartsAt(text, at) || credentialWordStartsAt(text, at + 1);
|
|
712
987
|
}
|
|
988
|
+
function nextPairBeyondFragmentAt(text, at) {
|
|
989
|
+
for (const p of [at, at + 1]) {
|
|
990
|
+
let k = p;
|
|
991
|
+
while (k < text.length && text.charCodeAt(k) === 0x5c)
|
|
992
|
+
k++;
|
|
993
|
+
if (text[k] === '"' || text[k] === "'")
|
|
994
|
+
k++;
|
|
995
|
+
for (const src of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
|
|
996
|
+
const re = new RegExp(src.source, 'iy');
|
|
997
|
+
re.lastIndex = k;
|
|
998
|
+
const m = re.exec(text);
|
|
999
|
+
if (m !== null && endsUrlFragmentAt(m[0], m[0].length - 1))
|
|
1000
|
+
return true;
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
return false;
|
|
1004
|
+
}
|
|
713
1005
|
function percentDecodeForMatching(text) {
|
|
714
1006
|
const chars = [];
|
|
715
1007
|
const offsets = [];
|
|
@@ -773,7 +1065,7 @@ function hostTokenIsLabel(afterUserinfo, hostEnd) {
|
|
|
773
1065
|
}
|
|
774
1066
|
return false;
|
|
775
1067
|
}
|
|
776
|
-
function exemptAddressEdits(src, from, to, edits) {
|
|
1068
|
+
function exemptAddressEdits(src, from, to, edits, rawPart = (a, b) => src.slice(a, b)) {
|
|
777
1069
|
const value = src.slice(from, to);
|
|
778
1070
|
const q = value.search(/[?#]/);
|
|
779
1071
|
const prefix = q === -1 ? value : value.slice(0, q);
|
|
@@ -790,11 +1082,48 @@ function exemptAddressEdits(src, from, to, edits) {
|
|
|
790
1082
|
const pathStart = scheme.length + (at === -1 ? cut : at + 1 + (cut - userinfoLen));
|
|
791
1083
|
const pathEnd = q === -1 ? value.length : q;
|
|
792
1084
|
const pathCut = pathCutAfterFirstLabel(value.slice(pathStart, pathEnd));
|
|
793
|
-
if (pathCut !== -1)
|
|
1085
|
+
if (pathCut !== -1 && !onlyQuotesAndMarkers(rawPart(from + pathStart + pathCut, from + pathEnd)))
|
|
794
1086
|
pushEdit(edits, src, from + pathStart + pathCut, from + pathEnd, REDACTED_SECRET);
|
|
795
1087
|
if (q !== -1)
|
|
796
1088
|
pushEdit(edits, src, from + q + 1, to, value.charAt(q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY);
|
|
797
1089
|
}
|
|
1090
|
+
function addressJunkStart(text, from, to) {
|
|
1091
|
+
let authEnd = from;
|
|
1092
|
+
while (authEnd < to && text[authEnd] !== '/' && text[authEnd] !== '?' && text[authEnd] !== '#')
|
|
1093
|
+
authEnd++;
|
|
1094
|
+
const at = text.lastIndexOf('@', authEnd - 1);
|
|
1095
|
+
if (at < from)
|
|
1096
|
+
return -1;
|
|
1097
|
+
let k = at + 1;
|
|
1098
|
+
if (text[k] === '[') {
|
|
1099
|
+
const close = text.indexOf(']', k);
|
|
1100
|
+
if (close === -1 || close >= to)
|
|
1101
|
+
return -1;
|
|
1102
|
+
k = close + 1;
|
|
1103
|
+
}
|
|
1104
|
+
else {
|
|
1105
|
+
while (k < to) {
|
|
1106
|
+
const c = text.charCodeAt(k);
|
|
1107
|
+
if (!(isHostChar(c) || (c >= 0xa1 && c !== 0xab && c !== 0xbb && !isRegexSpace(c))))
|
|
1108
|
+
break;
|
|
1109
|
+
k++;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
if (text.charCodeAt(k) === 0x3a) {
|
|
1113
|
+
let d = k + 1;
|
|
1114
|
+
while (d < to && text.charCodeAt(d) >= 0x30 && text.charCodeAt(d) <= 0x39)
|
|
1115
|
+
d++;
|
|
1116
|
+
if (d > k + 1)
|
|
1117
|
+
k = d;
|
|
1118
|
+
}
|
|
1119
|
+
if (k >= to || text[k] === '/' || text[k] === '?' || text[k] === '#')
|
|
1120
|
+
return -1;
|
|
1121
|
+
return k;
|
|
1122
|
+
}
|
|
1123
|
+
function onlyQuotesAndMarkers(part) {
|
|
1124
|
+
const core = part.replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '').split(RETIRED_MARKER).join('');
|
|
1125
|
+
return /^["'\\]*$/.test(core);
|
|
1126
|
+
}
|
|
798
1127
|
function isAddressShaped(value, raw = value) {
|
|
799
1128
|
return value.startsWith(`${REDACTED_USERINFO}@`) || ADDRESS_SCHEME_RE.test(raw);
|
|
800
1129
|
}
|
|
@@ -881,10 +1210,376 @@ function trailingLabelStart(text, from, end) {
|
|
|
881
1210
|
}
|
|
882
1211
|
const withoutSeparators = (s) => (s.includes(VIEW_SEPARATOR) ? s.split(VIEW_SEPARATOR).join('') : s);
|
|
883
1212
|
const isHorizontalSpace = (ch) => ch === ' ' || ch === '\t' || ch === '\u00a0' || ch === VIEW_SEPARATOR;
|
|
884
|
-
function
|
|
885
|
-
const
|
|
1213
|
+
function rawUserinfoAddressAt(text, at, end, gapAfter, stops) {
|
|
1214
|
+
const k = Math.min(end, stops.user[at] ?? text.length);
|
|
1215
|
+
if (k === at || text[k] !== ':')
|
|
1216
|
+
return false;
|
|
1217
|
+
const p = Math.min(end, stops.password[k + 1] ?? text.length);
|
|
1218
|
+
if (p <= k + 1 || text[p] !== '@')
|
|
1219
|
+
return false;
|
|
1220
|
+
if (gapAfter(p))
|
|
1221
|
+
return false;
|
|
1222
|
+
return schemelessHostEnd(text, p + 1) !== -1;
|
|
1223
|
+
}
|
|
1224
|
+
function userinfoStops(text) {
|
|
1225
|
+
const n = text.length;
|
|
1226
|
+
const user = new Int32Array(n + 1).fill(n);
|
|
1227
|
+
const password = new Int32Array(n + 1).fill(n);
|
|
1228
|
+
const brk = new Int32Array(n + 1).fill(n);
|
|
1229
|
+
for (let k = n - 1; k >= 0; k--) {
|
|
1230
|
+
const c = text.charCodeAt(k);
|
|
1231
|
+
const br = breaksUserinfo(c);
|
|
1232
|
+
user[k] = br || c === 0x3a || USERNAME_DELIMITERS.has(text.charAt(k)) ? k : (user[k + 1] ?? n);
|
|
1233
|
+
brk[k] = br && text[k] !== VIEW_SEPARATOR ? k : (brk[k + 1] ?? n);
|
|
1234
|
+
password[k] = (br && text[k] !== VIEW_SEPARATOR) || PASSWORD_FORBIDDEN.has(text.charAt(k)) ? k : (password[k + 1] ?? n);
|
|
1235
|
+
}
|
|
1236
|
+
return { user, password, brk };
|
|
1237
|
+
}
|
|
1238
|
+
function forbiddenUserinfoAt(text, at, end, gapAfter, stops, unitAt) {
|
|
1239
|
+
const k = Math.min(end, stops.user[at] ?? text.length);
|
|
1240
|
+
if (k >= end || text[k] !== ':')
|
|
1241
|
+
return -1;
|
|
1242
|
+
for (let u = at; u < k; u++)
|
|
1243
|
+
if (unitAt(u) !== UNIT_NONE || text[u] === VIEW_SEPARATOR)
|
|
1244
|
+
return -1;
|
|
1245
|
+
const p = Math.min(end, stops.password[k + 1] ?? text.length);
|
|
1246
|
+
if (p >= end || !PASSWORD_FORBIDDEN.has(text.charAt(p)))
|
|
1247
|
+
return -1;
|
|
1248
|
+
const a = Math.min(end, stops.brk[k + 1] ?? text.length);
|
|
1249
|
+
if (a >= end || text[a] !== '@' || gapAfter(a))
|
|
1250
|
+
return -1;
|
|
1251
|
+
return schemelessHostEnd(text, a + 1) === -1 ? -1 : a;
|
|
1252
|
+
}
|
|
1253
|
+
function isTokenCountAt(text, at) {
|
|
1254
|
+
const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1255
|
+
re.lastIndex = at;
|
|
1256
|
+
const m = re.exec(text);
|
|
1257
|
+
return m !== null && isTokenCountLabel(text, at, m[1] ?? '');
|
|
1258
|
+
}
|
|
1259
|
+
function innerWordHasValue(text, at, idx) {
|
|
1260
|
+
for (const src of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
|
|
1261
|
+
const re = new RegExp(src.source, 'iy');
|
|
1262
|
+
re.lastIndex = at;
|
|
1263
|
+
const m = re.exec(text);
|
|
1264
|
+
if (m !== null)
|
|
1265
|
+
return scanSecretValue(text, at + m[0].length, idx) !== null;
|
|
1266
|
+
}
|
|
1267
|
+
return false;
|
|
1268
|
+
}
|
|
1269
|
+
function quoteSwallowedBefore(S, at) {
|
|
1270
|
+
let k = at - 1;
|
|
1271
|
+
while (k >= 0 && S.dropped(k))
|
|
1272
|
+
k--;
|
|
1273
|
+
const lo = k >= 0 ? S.rawEnd(k) : 0;
|
|
1274
|
+
const hi = at < S.text.length ? S.rawStart(at) : S.raw.length;
|
|
1275
|
+
if (hi <= lo)
|
|
1276
|
+
return '';
|
|
1277
|
+
const gap = S.raw.slice(lo, hi);
|
|
1278
|
+
const dq = gap.indexOf('"');
|
|
1279
|
+
const sq = gap.indexOf("'");
|
|
1280
|
+
if (dq === -1 && sq === -1)
|
|
1281
|
+
return '';
|
|
1282
|
+
return dq === -1 ? "'" : sq === -1 ? '"' : dq < sq ? '"' : "'";
|
|
1283
|
+
}
|
|
1284
|
+
function labelEndsAt(text, end, atLeast = false) {
|
|
1285
|
+
let k = end;
|
|
1286
|
+
while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
|
|
1287
|
+
k--;
|
|
1288
|
+
if (k === 0)
|
|
1289
|
+
return false;
|
|
1290
|
+
const sep = text.charAt(k - 1);
|
|
1291
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1292
|
+
return false;
|
|
1293
|
+
k--;
|
|
1294
|
+
while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
|
|
1295
|
+
k--;
|
|
1296
|
+
if (k > 0 && (text[k - 1] === '"' || text[k - 1] === "'"))
|
|
1297
|
+
k--;
|
|
1298
|
+
while (k > 0 && text.charCodeAt(k - 1) === 0x5c)
|
|
1299
|
+
k--;
|
|
1300
|
+
let w = k;
|
|
1301
|
+
while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
|
|
1302
|
+
w--;
|
|
1303
|
+
const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1304
|
+
for (let s = w; s < k; s++) {
|
|
1305
|
+
re.lastIndex = s;
|
|
1306
|
+
const m = re.exec(text);
|
|
1307
|
+
if (m !== null && (atLeast ? s + m[0].length >= end : s + m[0].length === end) && !isTokenCountLabel(text, s, m[1] ?? ''))
|
|
1308
|
+
return true;
|
|
1309
|
+
}
|
|
1310
|
+
return false;
|
|
1311
|
+
}
|
|
1312
|
+
function schemeWordEndsAt(text, end) {
|
|
1313
|
+
const sep = text.charAt(end - 1);
|
|
1314
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1315
|
+
return false;
|
|
1316
|
+
let w = end - 1;
|
|
1317
|
+
while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
|
|
1318
|
+
w--;
|
|
1319
|
+
const re = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
1320
|
+
for (let s = w; s < end - 1; s++) {
|
|
1321
|
+
re.lastIndex = s;
|
|
1322
|
+
const m = re.exec(text);
|
|
1323
|
+
if (m !== null && s + m[0].length === end)
|
|
1324
|
+
return true;
|
|
1325
|
+
}
|
|
1326
|
+
return false;
|
|
1327
|
+
}
|
|
1328
|
+
function quotedExtensionEnd(text, close, cache) {
|
|
1329
|
+
const escaped = text.charCodeAt(close) === 0x5c;
|
|
1330
|
+
const q = escaped ? text.charAt(close + 1) : text.charAt(close);
|
|
1331
|
+
if (q !== '"' && q !== "'")
|
|
1332
|
+
return close;
|
|
1333
|
+
const token = escaped ? `\\${q}` : q;
|
|
1334
|
+
const after = (at) => at + token.length;
|
|
1335
|
+
let memo = cache.get(token);
|
|
1336
|
+
if (memo === undefined) {
|
|
1337
|
+
memo = new Map();
|
|
1338
|
+
cache.set(token, memo);
|
|
1339
|
+
}
|
|
1340
|
+
const done = memo;
|
|
1341
|
+
const step = (cur) => {
|
|
1342
|
+
let n = after(cur);
|
|
1343
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1344
|
+
n++;
|
|
1345
|
+
const sepNext = text[n] === ':' || text[n] === '=' || text[n] === '\uFF1A' || text[n] === '\uFF1D';
|
|
1346
|
+
if (sepNext && labelEndsAt(text, n + 1, true) && !labelEndsAt(text, cur)) {
|
|
1347
|
+
const next = text.indexOf(token, after(cur));
|
|
1348
|
+
return next === -1 ? { kind: 'open', inner: false } : { kind: 'next', at: next };
|
|
1349
|
+
}
|
|
1350
|
+
if (labelEndsAt(text, cur)) {
|
|
1351
|
+
const innerClose = text.indexOf(token, after(cur));
|
|
1352
|
+
if (innerClose === -1)
|
|
1353
|
+
return { kind: 'open', inner: true };
|
|
1354
|
+
const next = text.indexOf(token, after(innerClose));
|
|
1355
|
+
return next === -1 ? { kind: 'open', inner: true } : { kind: 'next', at: next };
|
|
1356
|
+
}
|
|
1357
|
+
return { kind: 'end' };
|
|
1358
|
+
};
|
|
1359
|
+
const first = step(close);
|
|
1360
|
+
if (first.kind === 'end')
|
|
1361
|
+
return close;
|
|
1362
|
+
if (first.kind === 'open')
|
|
1363
|
+
return first.inner ? close : text.length;
|
|
1364
|
+
const path = [];
|
|
1365
|
+
let cur = first.at;
|
|
1366
|
+
let result;
|
|
1367
|
+
for (;;) {
|
|
1368
|
+
const hit = done.get(cur);
|
|
1369
|
+
if (hit !== undefined) {
|
|
1370
|
+
result = hit;
|
|
1371
|
+
break;
|
|
1372
|
+
}
|
|
1373
|
+
path.push(cur);
|
|
1374
|
+
const s = step(cur);
|
|
1375
|
+
if (s.kind === 'end') {
|
|
1376
|
+
result = cur;
|
|
1377
|
+
break;
|
|
1378
|
+
}
|
|
1379
|
+
if (s.kind === 'open') {
|
|
1380
|
+
result = text.length;
|
|
1381
|
+
break;
|
|
1382
|
+
}
|
|
1383
|
+
cur = s.at;
|
|
1384
|
+
}
|
|
1385
|
+
for (const p of path)
|
|
1386
|
+
done.set(p, result);
|
|
1387
|
+
return result;
|
|
1388
|
+
}
|
|
1389
|
+
function extensionMaskStart(text, close) {
|
|
1390
|
+
let n = close + (text.charCodeAt(close) === 0x5c ? 2 : 1);
|
|
1391
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1392
|
+
n++;
|
|
1393
|
+
const sep = text[n];
|
|
1394
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1395
|
+
return close;
|
|
1396
|
+
if (!labelEndsAt(text, n + 1, true) || labelEndsAt(text, close))
|
|
1397
|
+
return close;
|
|
1398
|
+
n++;
|
|
1399
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1400
|
+
n++;
|
|
1401
|
+
return n;
|
|
1402
|
+
}
|
|
1403
|
+
function quotedAddressEnd(S, from, to) {
|
|
1404
|
+
const text = S.text;
|
|
1405
|
+
for (let k = from; k < to; k++) {
|
|
1406
|
+
const c = text.charCodeAt(k);
|
|
1407
|
+
if ((isRegexSpace(c) && c !== 0xfeff) || S.unitAt(k) !== UNIT_NONE)
|
|
1408
|
+
return k;
|
|
1409
|
+
if (k > from && S.spaceGapBefore(k))
|
|
1410
|
+
return k;
|
|
1411
|
+
}
|
|
1412
|
+
return to;
|
|
1413
|
+
}
|
|
1414
|
+
function labelCandidates(S, re, regions, out) {
|
|
1415
|
+
const text = S.text;
|
|
1416
|
+
const unitAt = (k) => S.unitAt(k);
|
|
886
1417
|
const schemePass = re === SECRET_SCHEME_WORD;
|
|
887
1418
|
const idx = new InnerWordIndex(text);
|
|
1419
|
+
const extensionCache = new Map();
|
|
1420
|
+
let stops;
|
|
1421
|
+
const push = (start, end, kindText, te = -1, cls) => {
|
|
1422
|
+
let a = start;
|
|
1423
|
+
let b = end;
|
|
1424
|
+
while (a < b && text[a] === VIEW_SEPARATOR)
|
|
1425
|
+
a++;
|
|
1426
|
+
while (b > a && text[b - 1] === VIEW_SEPARATOR)
|
|
1427
|
+
b--;
|
|
1428
|
+
if (a >= b)
|
|
1429
|
+
return;
|
|
1430
|
+
if (text.slice(a, b) === kindText)
|
|
1431
|
+
return;
|
|
1432
|
+
const kind = KIND_BY_TEXT.get(kindText);
|
|
1433
|
+
if (kind === undefined)
|
|
1434
|
+
return;
|
|
1435
|
+
out.push({ a, b, kind, cls: cls ?? (kindText === REDACTED_SECRET ? 'word' : 'structural'), te });
|
|
1436
|
+
};
|
|
1437
|
+
const pushEdits = (edits, te) => {
|
|
1438
|
+
for (const e of edits) {
|
|
1439
|
+
const part = text.slice(e.start, e.end);
|
|
1440
|
+
if (e.text === REDACTED_USERINFO ? userinfoIsWholeMarker(part) : e.text !== REDACTED_SECRET && slotIsWholeMarker(part))
|
|
1441
|
+
continue;
|
|
1442
|
+
push(e.start, e.end, e.text, e.text === REDACTED_SECRET ? -1 : te);
|
|
1443
|
+
}
|
|
1444
|
+
};
|
|
1445
|
+
let markerCache = -2;
|
|
1446
|
+
const nextMarkerFrom = (p) => {
|
|
1447
|
+
if (markerCache === -1 || markerCache >= p)
|
|
1448
|
+
return markerCache;
|
|
1449
|
+
markerCache = text.indexOf(REDACTION_MARKER_PREFIX, p);
|
|
1450
|
+
return markerCache;
|
|
1451
|
+
};
|
|
1452
|
+
const maskGaps = (r0, end) => {
|
|
1453
|
+
let p = r0;
|
|
1454
|
+
while (p < end) {
|
|
1455
|
+
while (p < end && (isRegexSpace(text.charCodeAt(p)) || unitAt(p) !== UNIT_NONE))
|
|
1456
|
+
p++;
|
|
1457
|
+
if (p >= end)
|
|
1458
|
+
return;
|
|
1459
|
+
let inner = idx.next(p);
|
|
1460
|
+
while (inner !== null && inner.start < end && isTokenCountAt(text, inner.start))
|
|
1461
|
+
inner = idx.next(inner.start + 1);
|
|
1462
|
+
const hasInner = inner !== null && inner.start < end;
|
|
1463
|
+
let stop = hasInner && inner !== null ? inner.start : end;
|
|
1464
|
+
const head = withoutSeparators(text.slice(p, Math.min(stop, p + 80)));
|
|
1465
|
+
const addressEnd = ADDRESS_SCHEME_RE.test(head) || head.startsWith(`${REDACTED_USERINFO}@`) ? Math.min(stop, quotedAddressEnd(S, p, end)) : -1;
|
|
1466
|
+
if (addressEnd !== -1 && addressEnd < stop) {
|
|
1467
|
+
p = addressEnd;
|
|
1468
|
+
continue;
|
|
1469
|
+
}
|
|
1470
|
+
if (addressEnd === -1) {
|
|
1471
|
+
let marker = null;
|
|
1472
|
+
for (let at = nextMarkerFrom(p); at !== -1 && at < stop && marker === null; at = nextMarkerFrom(at + 1)) {
|
|
1473
|
+
const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
1474
|
+
mk.lastIndex = at;
|
|
1475
|
+
marker = mk.exec(text);
|
|
1476
|
+
}
|
|
1477
|
+
const markerHit = marker !== null && marker.index < stop;
|
|
1478
|
+
if (markerHit && marker !== null)
|
|
1479
|
+
stop = marker.index;
|
|
1480
|
+
let cut = stop;
|
|
1481
|
+
while (cut > p && (isCutDelimiter(text.charAt(cut - 1)) || isRegexSpace(text.charCodeAt(cut - 1))))
|
|
1482
|
+
cut--;
|
|
1483
|
+
if (cut > p) {
|
|
1484
|
+
const rest = withoutSeparators(text.slice(p, cut));
|
|
1485
|
+
if (rest.trim() !== '' && !valueIsWholeMarker(S, p, cut) && !isDiagnosticValue(rest))
|
|
1486
|
+
push(p, cut, REDACTED_SECRET);
|
|
1487
|
+
}
|
|
1488
|
+
if (markerHit && marker !== null) {
|
|
1489
|
+
p = marker.index + marker[0].length;
|
|
1490
|
+
continue;
|
|
1491
|
+
}
|
|
1492
|
+
}
|
|
1493
|
+
if (!hasInner || inner === null)
|
|
1494
|
+
return;
|
|
1495
|
+
const lab = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1496
|
+
lab.lastIndex = inner.start;
|
|
1497
|
+
let m = lab.exec(text);
|
|
1498
|
+
if (m === null) {
|
|
1499
|
+
const sch = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
1500
|
+
sch.lastIndex = inner.start;
|
|
1501
|
+
m = sch.exec(text);
|
|
1502
|
+
}
|
|
1503
|
+
if (m === null)
|
|
1504
|
+
return;
|
|
1505
|
+
const after = inner.start + m[0].length;
|
|
1506
|
+
const iv = scanSecretValue(text, after, idx);
|
|
1507
|
+
let next = iv === null ? after : iv.end;
|
|
1508
|
+
if (next < end && (text[next] === '"' || text[next] === "'"))
|
|
1509
|
+
next++;
|
|
1510
|
+
if (next <= p)
|
|
1511
|
+
return;
|
|
1512
|
+
p = next;
|
|
1513
|
+
}
|
|
1514
|
+
};
|
|
1515
|
+
const pushInnerRemainder = (from) => {
|
|
1516
|
+
let open = from;
|
|
1517
|
+
while (text.charCodeAt(open) === 0x5c)
|
|
1518
|
+
open++;
|
|
1519
|
+
let q = text.charAt(open);
|
|
1520
|
+
let at = open + 1;
|
|
1521
|
+
let outerEnd;
|
|
1522
|
+
if (q === '"' || q === "'") {
|
|
1523
|
+
const outer = scanSecretValue(text, from);
|
|
1524
|
+
if (outer === null)
|
|
1525
|
+
return;
|
|
1526
|
+
outerEnd = outer.end;
|
|
1527
|
+
}
|
|
1528
|
+
else {
|
|
1529
|
+
q = quoteSwallowedBefore(S, from);
|
|
1530
|
+
if (q === '')
|
|
1531
|
+
return;
|
|
1532
|
+
at = from;
|
|
1533
|
+
const close = text.indexOf(q, from);
|
|
1534
|
+
outerEnd = close === -1 ? text.length : close;
|
|
1535
|
+
}
|
|
1536
|
+
const lab = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1537
|
+
lab.lastIndex = at;
|
|
1538
|
+
let m = lab.exec(text);
|
|
1539
|
+
if (m === null) {
|
|
1540
|
+
const sch = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
1541
|
+
sch.lastIndex = at;
|
|
1542
|
+
m = sch.exec(text);
|
|
1543
|
+
}
|
|
1544
|
+
if (m === null)
|
|
1545
|
+
return;
|
|
1546
|
+
const inner = { start: at };
|
|
1547
|
+
const iv = scanSecretValue(text, inner.start + m[0].length, idx);
|
|
1548
|
+
if (iv === null)
|
|
1549
|
+
return;
|
|
1550
|
+
let r0 = iv.end;
|
|
1551
|
+
if (r0 < text.length && (text[r0] === '"' || text[r0] === "'") && text[r0] !== q)
|
|
1552
|
+
r0++;
|
|
1553
|
+
maskGaps(r0, outerEnd);
|
|
1554
|
+
};
|
|
1555
|
+
const regionKind = regions.length === 0 ? null : new Uint8Array(text.length);
|
|
1556
|
+
if (regionKind !== null)
|
|
1557
|
+
for (const r of regions)
|
|
1558
|
+
for (let k = r[0]; k < r[1] && k < text.length; k++)
|
|
1559
|
+
regionKind[k] = r[2] === 'query' ? 1 : 2;
|
|
1560
|
+
const inQuery = (k) => regionKind !== null && k < text.length && regionKind[k] === 1;
|
|
1561
|
+
const forbiddenTailAt = (p) => {
|
|
1562
|
+
if (!PASSWORD_FORBIDDEN.has(text.charAt(p)))
|
|
1563
|
+
return -1;
|
|
1564
|
+
const st = (stops ??= userinfoStops(text));
|
|
1565
|
+
const a = st.brk[p] ?? text.length;
|
|
1566
|
+
if (a >= text.length || text[a] !== '@' || S.gapAfter(a))
|
|
1567
|
+
return -1;
|
|
1568
|
+
const inner = idx.next(p);
|
|
1569
|
+
if (inner !== null && inner.start < a)
|
|
1570
|
+
return -1;
|
|
1571
|
+
const hostEnd = schemelessHostEnd(text, a + 1);
|
|
1572
|
+
if (hostEnd === -1)
|
|
1573
|
+
return -1;
|
|
1574
|
+
let q = -1;
|
|
1575
|
+
for (let k = a - 1; k >= p && q === -1; k--)
|
|
1576
|
+
if (PASSWORD_FORBIDDEN.has(text.charAt(k)))
|
|
1577
|
+
q = k;
|
|
1578
|
+
const closer = forbiddenCloser(text.charAt(q));
|
|
1579
|
+
if (closer !== '' && q < a - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
|
|
1580
|
+
return -1;
|
|
1581
|
+
return a;
|
|
1582
|
+
};
|
|
888
1583
|
re.lastIndex = 0;
|
|
889
1584
|
for (let hit = re.exec(text); hit !== null; hit = re.exec(text)) {
|
|
890
1585
|
const label = hit[1] ?? '';
|
|
@@ -892,25 +1587,110 @@ function labelEdits(text, re, literalBlanks = false, unitAt = NO_UNITS) {
|
|
|
892
1587
|
continue;
|
|
893
1588
|
const from = hit.index + hit[0].length;
|
|
894
1589
|
if (credentialWordStartsAt(text, from)) {
|
|
1590
|
+
pushInnerRemainder(from);
|
|
895
1591
|
re.lastIndex = from;
|
|
896
1592
|
continue;
|
|
897
1593
|
}
|
|
898
|
-
|
|
899
|
-
if (
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1594
|
+
let v0s = scanSecretValue(text, from, idx);
|
|
1595
|
+
if (v0s === null) {
|
|
1596
|
+
let k = from;
|
|
1597
|
+
while (k < text.length && text[k] === VIEW_SEPARATOR)
|
|
1598
|
+
k++;
|
|
1599
|
+
const a = inQuery(k) ? -1 : forbiddenTailAt(k);
|
|
1600
|
+
if (a === -1)
|
|
1601
|
+
continue;
|
|
1602
|
+
v0s = { start: k, end: a };
|
|
1603
|
+
}
|
|
1604
|
+
const v0 = { start: v0s.start, end: v0s.end };
|
|
1605
|
+
let open = from;
|
|
1606
|
+
while (text.charCodeAt(open) === 0x5c)
|
|
1607
|
+
open++;
|
|
1608
|
+
let quoted = text[open] === '"' || text[open] === "'";
|
|
1609
|
+
if (!quoted) {
|
|
1610
|
+
const q = quoteSwallowedBefore(S, v0.start);
|
|
1611
|
+
if (q !== '') {
|
|
1612
|
+
quoted = true;
|
|
1613
|
+
let j = v0.start;
|
|
1614
|
+
while (j < text.length && text[j] !== q)
|
|
1615
|
+
j += text[j] === '\\' ? 2 : 1;
|
|
1616
|
+
v0.end = Math.min(j, text.length);
|
|
1617
|
+
}
|
|
1618
|
+
}
|
|
1619
|
+
let keptSepEnd = -1;
|
|
1620
|
+
if (quoted) {
|
|
1621
|
+
let close = v0.end;
|
|
1622
|
+
while (close < text.length && text[close] === VIEW_SEPARATOR)
|
|
1623
|
+
close++;
|
|
1624
|
+
const ext = close < text.length ? quotedExtensionEnd(text, close, extensionCache) : close;
|
|
1625
|
+
if (ext > close) {
|
|
1626
|
+
const from = extensionMaskStart(text, close);
|
|
1627
|
+
if (from > close)
|
|
1628
|
+
keptSepEnd = from;
|
|
1629
|
+
push(from, ext, REDACTED_SECRET);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
const v = { start: v0.start, end: v0.end };
|
|
1633
|
+
if (quoted) {
|
|
1634
|
+
while (v.start < v.end && unitAt(v.start) !== UNIT_NONE)
|
|
1635
|
+
v.start++;
|
|
1636
|
+
while (v.end > v.start && unitAt(v.end - 1) !== UNIT_NONE)
|
|
1637
|
+
v.end--;
|
|
1638
|
+
if (v.start >= v.end) {
|
|
1639
|
+
re.lastIndex = from;
|
|
1640
|
+
continue;
|
|
1641
|
+
}
|
|
1642
|
+
}
|
|
1643
|
+
if (valueIsWholeMarker(S, v.start, v.end)) {
|
|
1644
|
+
re.lastIndex = quoted ? from : v.end;
|
|
903
1645
|
continue;
|
|
904
1646
|
}
|
|
905
|
-
if (
|
|
906
|
-
|
|
907
|
-
|
|
1647
|
+
if (!quoted && inQuery(v.start)) {
|
|
1648
|
+
for (let k = v.start; k < v.end; k++) {
|
|
1649
|
+
const ch = text[k];
|
|
1650
|
+
if (ch === '&' || ch === '#') {
|
|
1651
|
+
v.end = k;
|
|
1652
|
+
break;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
if (v.end <= v.start) {
|
|
1656
|
+
re.lastIndex = v0.end;
|
|
908
1657
|
continue;
|
|
909
1658
|
}
|
|
910
1659
|
}
|
|
1660
|
+
let forbiddenAt = -1;
|
|
1661
|
+
if (!quoted && !inQuery(v.start)) {
|
|
1662
|
+
forbiddenAt = forbiddenUserinfoAt(text, v.start, text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
|
|
1663
|
+
if (forbiddenAt !== -1)
|
|
1664
|
+
v.end = Math.max(v.end, addressExtentEnd(text, forbiddenAt));
|
|
1665
|
+
}
|
|
1666
|
+
else if (quoted) {
|
|
1667
|
+
forbiddenAt = forbiddenUserinfoAt(text, v.start, v.end, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
|
|
1668
|
+
}
|
|
1669
|
+
if (!quoted && forbiddenAt === -1 && !inQuery(v.start)) {
|
|
1670
|
+
const a = forbiddenTailAt(v.end);
|
|
1671
|
+
if (a !== -1)
|
|
1672
|
+
v.end = a;
|
|
1673
|
+
}
|
|
1674
|
+
if (!quoted) {
|
|
1675
|
+
const trailing = trailingLabelStart(text, v.start, v.end);
|
|
1676
|
+
if (trailing > 0 && innerWordHasValue(text, v.start + trailing, idx)) {
|
|
1677
|
+
let e = v.start + trailing;
|
|
1678
|
+
while (e > v.start && (isCutDelimiter(text.charAt(e - 1)) || text[e - 1] === VIEW_SEPARATOR))
|
|
1679
|
+
e--;
|
|
1680
|
+
if (e > v.start)
|
|
1681
|
+
v.end = e;
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1684
|
+
if (pemBeginAt(S, v.start)) {
|
|
1685
|
+
re.lastIndex = v.end;
|
|
1686
|
+
continue;
|
|
1687
|
+
}
|
|
911
1688
|
const rawValue = text.slice(v.start, v.end);
|
|
912
1689
|
const value = withoutSeparators(rawValue);
|
|
913
|
-
|
|
1690
|
+
if (MARKER_WITH_DOTS.test(value)) {
|
|
1691
|
+
re.lastIndex = quoted ? from : v.end;
|
|
1692
|
+
continue;
|
|
1693
|
+
}
|
|
914
1694
|
if (schemePass && !quoted && SCHEME_FOLLOWER_WORDS.has(value.replace(/[.,;:!?)]+$/, '').toLowerCase())) {
|
|
915
1695
|
re.lastIndex = v.end;
|
|
916
1696
|
continue;
|
|
@@ -927,75 +1707,69 @@ function labelEdits(text, re, literalBlanks = false, unitAt = NO_UNITS) {
|
|
|
927
1707
|
re.lastIndex = v.end;
|
|
928
1708
|
continue;
|
|
929
1709
|
}
|
|
930
|
-
|
|
1710
|
+
let cred = j > v.end && !labelStartsAt(text, j) ? scanSecretValue(text, j, idx) : null;
|
|
1711
|
+
if (j > v.end && !inQuery(j)) {
|
|
1712
|
+
const a = forbiddenTailAt(cred === null ? j : cred.end);
|
|
1713
|
+
if (a !== -1)
|
|
1714
|
+
cred = { start: cred === null ? j : cred.start, end: a };
|
|
1715
|
+
}
|
|
931
1716
|
if (cred !== null) {
|
|
932
1717
|
const credValue = withoutSeparators(text.slice(cred.start, cred.end));
|
|
933
1718
|
const keep = isMarker(credValue) || DIAGNOSTIC_VALUE_WORDS.has(credValue.replace(/[.,;:!?)\]}\x27\x22]+$/, '').toLowerCase());
|
|
934
1719
|
if (!keep)
|
|
935
|
-
|
|
1720
|
+
push(cred.start, cred.end, REDACTED_SECRET);
|
|
936
1721
|
re.lastIndex = cred.end;
|
|
937
1722
|
continue;
|
|
938
1723
|
}
|
|
939
1724
|
re.lastIndex = v.end;
|
|
940
1725
|
continue;
|
|
941
1726
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
1727
|
+
const forbiddenAddress = forbiddenAt !== -1 && text[v.start] !== ':';
|
|
1728
|
+
const rawAddress = forbiddenAddress || rawUserinfoAddressAt(text, v.start, quoted ? v.end : text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text));
|
|
1729
|
+
if (isDiagnosticValue(value, rawValue) || rawAddress) {
|
|
1730
|
+
if (isAddressShaped(value, rawValue) || rawAddress) {
|
|
1731
|
+
const extentFrom = forbiddenAddress ? forbiddenAt : v.start;
|
|
1732
|
+
let end = quoted ? quotedAddressEnd(S, extentFrom, v.end) : addressExtentEnd(text, extentFrom);
|
|
945
1733
|
const trailing = trailingLabelStart(text, v.start, end);
|
|
946
1734
|
if (trailing !== -1)
|
|
947
1735
|
end = v.start + trailing;
|
|
948
1736
|
if (gluedToSingleUnit(v.start, hit.index, unitAt) || userinfoHostHidden(text, v.start, unitAt)) {
|
|
949
1737
|
const cut = trailing !== -1 ? end : Math.max(end, v.end);
|
|
950
|
-
const
|
|
1738
|
+
const schemeHit = ADDRESS_SCHEME_RE.exec(text.slice(v.start, v.start + 20))?.[0] ?? '';
|
|
1739
|
+
const keep = schemeHit !== '' && schemeHit.length - 3 <= 12 ? schemeHit.length : 0;
|
|
951
1740
|
const q = text.slice(v.start, cut).search(/[?#]/);
|
|
952
1741
|
const bodyEnd = q === -1 ? cut : v.start + q;
|
|
953
1742
|
if (withoutSeparators(text.slice(v.start + keep, bodyEnd)) !== '')
|
|
954
|
-
|
|
1743
|
+
push(v.start + keep, bodyEnd, REDACTED_SECRET, -1, 'glued');
|
|
955
1744
|
if (q !== -1 && v.start + q + 1 < cut)
|
|
956
|
-
|
|
957
|
-
re.lastIndex = cut;
|
|
1745
|
+
push(v.start + q + 1, cut, text.charAt(v.start + q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY, cut, 'glued');
|
|
1746
|
+
re.lastIndex = quoted ? from : cut;
|
|
958
1747
|
continue;
|
|
959
1748
|
}
|
|
960
|
-
|
|
961
|
-
|
|
1749
|
+
const edits = [];
|
|
1750
|
+
exemptAddressEdits(text, v.start, end, edits, (a, b) => (b > a ? S.raw.slice(S.rawStart(a), S.rawEnd(b - 1)) : ''));
|
|
1751
|
+
pushEdits(edits, end);
|
|
1752
|
+
if (rawAddress) {
|
|
1753
|
+
const limit = quoted ? end : Math.min(end, v.end);
|
|
1754
|
+
const junk = addressJunkStart(text, v.start, limit);
|
|
1755
|
+
if (junk !== -1) {
|
|
1756
|
+
const rest = withoutSeparators(text.slice(junk, limit)).replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '');
|
|
1757
|
+
if (/[A-Za-z0-9]/.test(rest))
|
|
1758
|
+
push(junk, limit, REDACTED_SECRET);
|
|
1759
|
+
}
|
|
1760
|
+
}
|
|
1761
|
+
if (quoted && end < v.end) {
|
|
1762
|
+
maskGaps(end, v.end);
|
|
1763
|
+
}
|
|
1764
|
+
re.lastIndex = quoted ? from : end;
|
|
962
1765
|
continue;
|
|
963
1766
|
}
|
|
964
1767
|
}
|
|
965
1768
|
else {
|
|
966
|
-
|
|
1769
|
+
push(v.start, keptSepEnd !== -1 && withoutSeparators(text.slice(v.end, keptSepEnd)).search(/[^"'\\:=\uFF1A\uFF1D \t]/) === -1 ? keptSepEnd : v.end, REDACTED_SECRET);
|
|
967
1770
|
}
|
|
968
|
-
re.lastIndex = v.end;
|
|
1771
|
+
re.lastIndex = quoted ? from : v.end;
|
|
969
1772
|
}
|
|
970
|
-
return edits;
|
|
971
|
-
}
|
|
972
|
-
function literalBlankLeadJudgedAsBlank(text, v, label, schemePass) {
|
|
973
|
-
let restStart;
|
|
974
|
-
if (v.start > 0 && text.charCodeAt(v.start - 1) === 0x5c && nameBlankTokenAt(text, v.start - 1) !== -1)
|
|
975
|
-
restStart = v.start + 5;
|
|
976
|
-
else if (nameBlankTokenAt(text, v.start) !== -1)
|
|
977
|
-
restStart = v.start + 6;
|
|
978
|
-
else if (eatenBlankTokenTailAt(text, v.start))
|
|
979
|
-
restStart = v.start + 5;
|
|
980
|
-
else
|
|
981
|
-
return false;
|
|
982
|
-
const rest = withoutSeparators(text.slice(restStart, v.end));
|
|
983
|
-
const bare = rest.replace(/[.,;:!?)\]}'"]+$/, '');
|
|
984
|
-
if (bare === '')
|
|
985
|
-
return true;
|
|
986
|
-
if (credentialWordStartsAt(' ' + text.slice(restStart, Math.min(text.length, v.end + 64)), 1))
|
|
987
|
-
return true;
|
|
988
|
-
if (schemePass && SCHEME_FOLLOWER_WORDS.has(bare.toLowerCase()))
|
|
989
|
-
return true;
|
|
990
|
-
if (!schemePass && isNamedNotValue(label, rest))
|
|
991
|
-
return true;
|
|
992
|
-
if (!schemePass && /authorizations?$/i.test(label) && isAuthScheme(bare))
|
|
993
|
-
return true;
|
|
994
|
-
return !isAddressShaped(rest) && isDiagnosticValue(rest);
|
|
995
|
-
}
|
|
996
|
-
function eatenBlankTokenTailAt(text, at) {
|
|
997
|
-
const hit = /^u([0-9A-Fa-f]{4})/.exec(text.slice(at, at + 5));
|
|
998
|
-
return hit !== null && isNameBlank(Number.parseInt(hit[1] ?? '', 16));
|
|
999
1773
|
}
|
|
1000
1774
|
function isAuthParamAt(text, j, scheme) {
|
|
1001
1775
|
const p = /^[A-Za-z][A-Za-z0-9_-]*([ \t\ufeff]*)=([ \t\ufeff]*)("|[^\s=,"]+)([ \t\ufeff]*,)?/.exec(text.slice(j, j + 512));
|
|
@@ -1087,13 +1861,15 @@ function pemBodyEndWithoutTrailer(text, from) {
|
|
|
1087
1861
|
}
|
|
1088
1862
|
return end;
|
|
1089
1863
|
}
|
|
1090
|
-
function
|
|
1091
|
-
const
|
|
1092
|
-
if (!text.includes('-----BEGIN
|
|
1093
|
-
return
|
|
1864
|
+
function pemBlocks(text, soft = () => false) {
|
|
1865
|
+
const out = [];
|
|
1866
|
+
if (!text.includes('-----BEGIN'))
|
|
1867
|
+
return out;
|
|
1868
|
+
const head = pemHeadView(text, soft);
|
|
1094
1869
|
const begin = new RegExp(PEM_BEGIN_SOURCE, 'g');
|
|
1870
|
+
const trailer = /-----END[A-Z0-9 ]*-----/y;
|
|
1095
1871
|
let nextEnd = -2;
|
|
1096
|
-
for (let b = begin.exec(
|
|
1872
|
+
for (let b = begin.exec(head); b !== null; b = begin.exec(head)) {
|
|
1097
1873
|
let bodyStart = b.index + b[0].length;
|
|
1098
1874
|
const lead = lineBreakLengthAt(text, bodyStart);
|
|
1099
1875
|
if (lead > 0)
|
|
@@ -1104,18 +1880,56 @@ function pemEdits(text) {
|
|
|
1104
1880
|
if (nextEnd !== -1 && nextEnd < bodyStart)
|
|
1105
1881
|
nextEnd = text.indexOf(PEM_END_MARK, bodyStart);
|
|
1106
1882
|
let bodyEnd;
|
|
1107
|
-
|
|
1883
|
+
let regionEnd;
|
|
1884
|
+
if (nextEnd === -1) {
|
|
1108
1885
|
bodyEnd = pemBodyEndWithoutTrailer(text, bodyStart);
|
|
1886
|
+
regionEnd = bodyEnd + lineBreakLengthAt(text, bodyEnd);
|
|
1887
|
+
}
|
|
1109
1888
|
else {
|
|
1110
1889
|
bodyEnd = nextEnd;
|
|
1111
1890
|
const back = lineBreakLengthBefore(text, bodyEnd);
|
|
1112
1891
|
if (back > 0 && bodyEnd - back >= bodyStart)
|
|
1113
1892
|
bodyEnd -= back;
|
|
1893
|
+
trailer.lastIndex = nextEnd;
|
|
1894
|
+
regionEnd = trailer.test(head) ? trailer.lastIndex : nextEnd + PEM_END_MARK.length;
|
|
1114
1895
|
}
|
|
1115
|
-
|
|
1116
|
-
pushEdit(edits, text, bodyStart, bodyEnd, REDACTED_SECRET);
|
|
1896
|
+
out.push({ begin: b.index, bodyStart, bodyEnd, regionEnd });
|
|
1117
1897
|
begin.lastIndex = Math.max(begin.lastIndex, bodyEnd);
|
|
1118
1898
|
}
|
|
1899
|
+
return out;
|
|
1900
|
+
}
|
|
1901
|
+
function pemHeadView(text, soft) {
|
|
1902
|
+
let chars = null;
|
|
1903
|
+
for (let k = 0; k < text.length; k++) {
|
|
1904
|
+
if (!soft(k) || text[k] === ' ')
|
|
1905
|
+
continue;
|
|
1906
|
+
chars ??= text.split('');
|
|
1907
|
+
chars[k] = ' ';
|
|
1908
|
+
}
|
|
1909
|
+
return chars === null ? text : chars.join('');
|
|
1910
|
+
}
|
|
1911
|
+
function pemBeginAt(S, at) {
|
|
1912
|
+
const k = at + '-----BEGIN'.length;
|
|
1913
|
+
return S.text.startsWith('-----BEGIN', at) && (S.text[k] === ' ' || (k < S.text.length && pemSoft(S, k)));
|
|
1914
|
+
}
|
|
1915
|
+
const pemSoft = (S, k) => S.dropped(k) || S.painted(k);
|
|
1916
|
+
function pemBlockToRaw(S, blk) {
|
|
1917
|
+
const n = S.text.length;
|
|
1918
|
+
const at = (k) => (k < n ? S.rawStart(k) : S.raw.length);
|
|
1919
|
+
const begin = at(blk.begin);
|
|
1920
|
+
const bodyStart = at(blk.bodyStart);
|
|
1921
|
+
const bodyEnd = blk.bodyEnd > blk.bodyStart ? Math.max(bodyStart, S.rawEnd(blk.bodyEnd - 1)) : bodyStart;
|
|
1922
|
+
const regionEnd = blk.regionEnd > blk.begin ? Math.max(begin, S.rawEnd(blk.regionEnd - 1)) : begin;
|
|
1923
|
+
return { begin, bodyStart, bodyEnd, regionEnd };
|
|
1924
|
+
}
|
|
1925
|
+
function pemEdits(S, onBlock) {
|
|
1926
|
+
const text = S.text;
|
|
1927
|
+
const edits = [];
|
|
1928
|
+
for (const blk of pemBlocks(text, (k) => pemSoft(S, k))) {
|
|
1929
|
+
onBlock(blk);
|
|
1930
|
+
if (blk.bodyEnd > blk.bodyStart)
|
|
1931
|
+
pushEdit(edits, text, blk.bodyStart, blk.bodyEnd, REDACTED_SECRET);
|
|
1932
|
+
}
|
|
1119
1933
|
return edits;
|
|
1120
1934
|
}
|
|
1121
1935
|
function authParamEdits(text) {
|
|
@@ -1188,21 +2002,6 @@ function proseLiteralEdits(text) {
|
|
|
1188
2002
|
}
|
|
1189
2003
|
return edits;
|
|
1190
2004
|
}
|
|
1191
|
-
function washWordNets(target, literalBlanks = false) {
|
|
1192
|
-
const unitAt = (k) => target.unitAt(k);
|
|
1193
|
-
target.apply(pemEdits(target.text));
|
|
1194
|
-
target.apply(labelEdits(target.text, SECRET_SCHEME_WORD, literalBlanks, unitAt));
|
|
1195
|
-
target.apply(labelEdits(target.text, SECRET_LABELLED_WORD, literalBlanks, unitAt));
|
|
1196
|
-
target.apply(authParamEdits(target.text));
|
|
1197
|
-
target.apply(proseLiteralEdits(target.text));
|
|
1198
|
-
}
|
|
1199
|
-
function runCredentialNets(target, urls, words, literalBlanks = false) {
|
|
1200
|
-
if (urls)
|
|
1201
|
-
washUrlNets(target);
|
|
1202
|
-
if (words)
|
|
1203
|
-
washWordNets(target, literalBlanks);
|
|
1204
|
-
}
|
|
1205
|
-
const NEEDS_VIEW = /[\u0000-\u0008\u000E-\u001F\u007F-\u009F\uD800-\uDFFF]|\p{Cf}|\\u/u;
|
|
1206
2005
|
const FORMAT_CHAR = /^\p{Cf}$/u;
|
|
1207
2006
|
const isStrippableControl = (c) => (c <= 0x1f && (c < 0x09 || c > 0x0d)) || (c >= 0x7f && c <= 0x9f);
|
|
1208
2007
|
class SequenceReader {
|
|
@@ -1215,10 +2014,12 @@ class SequenceReader {
|
|
|
1215
2014
|
termHit = null;
|
|
1216
2015
|
breakFrom = -1;
|
|
1217
2016
|
breakAt = -1;
|
|
1218
|
-
|
|
2017
|
+
foldSpaces;
|
|
2018
|
+
constructor(text, atomicBlanks, foldBlanks = false, foldSpaces = false) {
|
|
1219
2019
|
this.text = text;
|
|
1220
2020
|
this.atomicBlanks = atomicBlanks;
|
|
1221
2021
|
this.foldBlanks = foldBlanks;
|
|
2022
|
+
this.foldSpaces = foldSpaces;
|
|
1222
2023
|
this.terminator = /[\u0007\u009c]|\u001b\\|\\u(?:0007|009[cC]|001[bB]\\)/g;
|
|
1223
2024
|
this.lineBreak = /[\n\r]/g;
|
|
1224
2025
|
}
|
|
@@ -1236,6 +2037,8 @@ class SequenceReader {
|
|
|
1236
2037
|
const c = this.text.charCodeAt(at);
|
|
1237
2038
|
if (c >= 0x20 && c <= 0x2f)
|
|
1238
2039
|
return 1;
|
|
2040
|
+
if (this.foldSpaces && ((c >= 0x09 && c <= 0x0d) || c === 0x2028 || c === 0x2029))
|
|
2041
|
+
return 1;
|
|
1239
2042
|
if (!this.foldBlanks)
|
|
1240
2043
|
return 0;
|
|
1241
2044
|
if (isFoldedBlank(c))
|
|
@@ -1351,41 +2154,38 @@ function nameBlankTokenAt(text, at) {
|
|
|
1351
2154
|
const cp = Number.parseInt(hit[1] ?? '', 16);
|
|
1352
2155
|
return isNameBlank(cp) ? cp : -1;
|
|
1353
2156
|
}
|
|
1354
|
-
function
|
|
1355
|
-
const folding = blankTokens === 'fold' || blankTokens === 'foldlit';
|
|
1356
|
-
if (!NEEDS_VIEW.test(text) && !(folding && hasRawFoldedBlank(text)))
|
|
1357
|
-
return null;
|
|
2157
|
+
function buildScanView(text, o) {
|
|
1358
2158
|
const chars = [];
|
|
1359
2159
|
const starts = [];
|
|
1360
2160
|
const ends = [];
|
|
1361
2161
|
const units = [];
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
const
|
|
2162
|
+
const dropped = [];
|
|
2163
|
+
const spaceUnits = [];
|
|
2164
|
+
const contents = [];
|
|
2165
|
+
const seq = new SequenceReader(text, o.blanks ? 'all' : 'none', o.blanks || o.fold, o.fold);
|
|
1365
2166
|
const keep = (ch, s, e, unit = UNIT_NONE) => {
|
|
1366
2167
|
chars.push(ch);
|
|
1367
2168
|
starts.push(s);
|
|
1368
2169
|
ends.push(e);
|
|
1369
2170
|
units.push(unit);
|
|
2171
|
+
dropped.push(0);
|
|
2172
|
+
spaceUnits.push(0);
|
|
1370
2173
|
};
|
|
1371
|
-
const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE) => {
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
2174
|
+
const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE, space = 0) => {
|
|
2175
|
+
chars.push(placeholder);
|
|
2176
|
+
starts.push(s);
|
|
2177
|
+
ends.push(e);
|
|
2178
|
+
units.push(unit);
|
|
2179
|
+
dropped.push(1);
|
|
2180
|
+
spaceUnits.push(space);
|
|
1375
2181
|
};
|
|
1376
2182
|
const dropUnit = (s, unit) => {
|
|
1377
2183
|
const last = unit.end - 1;
|
|
1378
|
-
if (
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
for (let u = 2; u < 6; u++)
|
|
1382
|
-
keep(text.charAt(last + u), last + 6, last + 6);
|
|
1383
|
-
return last + 6;
|
|
1384
|
-
}
|
|
1385
|
-
if (wordLetters && unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
|
|
2184
|
+
if (unit.content !== undefined)
|
|
2185
|
+
contents.push([unit.content[0], unit.content[1]]);
|
|
2186
|
+
if (unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
|
|
1386
2187
|
drop(s, last, VIEW_SEPARATOR, UNIT_SEQUENCE);
|
|
1387
2188
|
keep(text.charAt(last), last, unit.end);
|
|
1388
|
-
splitAny = true;
|
|
1389
2189
|
return unit.end;
|
|
1390
2190
|
}
|
|
1391
2191
|
drop(s, unit.end, VIEW_SEPARATOR, UNIT_SEQUENCE);
|
|
@@ -1394,12 +2194,6 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1394
2194
|
let i = 0;
|
|
1395
2195
|
while (i < text.length) {
|
|
1396
2196
|
const c = text.charCodeAt(i);
|
|
1397
|
-
if (folding && isFoldedBlank(c)) {
|
|
1398
|
-
changed = true;
|
|
1399
|
-
keep(' ', i, i + 1);
|
|
1400
|
-
i++;
|
|
1401
|
-
continue;
|
|
1402
|
-
}
|
|
1403
2197
|
if (c === 0x1b || c === 0x9b || c === 0x9d || c === 0x90 || c === 0x98 || c === 0x9e || c === 0x9f) {
|
|
1404
2198
|
const unit = c === 0x1b ? escapeSequenceAfterEsc(text, i + 1, seq) : c1SequenceAfter(text, c, i + 1, seq);
|
|
1405
2199
|
if (unit !== null) {
|
|
@@ -1422,30 +2216,33 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1422
2216
|
i = dropUnit(i, unit);
|
|
1423
2217
|
continue;
|
|
1424
2218
|
}
|
|
1425
|
-
|
|
2219
|
+
if (isRegexSpace(tok.cp) && tok.cp !== 0xfeff) {
|
|
2220
|
+
if (o.escapeSpaces !== null && o.escapeSpaces.has(tok.cp))
|
|
2221
|
+
drop(i, tok.end, VIEW_SEPARATOR, UNIT_SINGLE, 1);
|
|
2222
|
+
else
|
|
2223
|
+
drop(i, tok.end, ' ', UNIT_NONE);
|
|
2224
|
+
}
|
|
2225
|
+
else
|
|
2226
|
+
drop(i, tok.end);
|
|
1426
2227
|
i = tok.end;
|
|
1427
2228
|
continue;
|
|
1428
2229
|
}
|
|
1429
|
-
if (
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
if (blankTokens === 'blank')
|
|
1434
|
-
keep(String.fromCharCode(cp), i, i + 6);
|
|
1435
|
-
else if (blankTokens === 'fold')
|
|
1436
|
-
keep(isFoldedBlank(cp) ? ' ' : String.fromCharCode(cp), i, i + 6);
|
|
1437
|
-
else if (blankTokens === 'foldlit' && isFoldedBlank(cp))
|
|
1438
|
-
keep(' ', i, i + 6);
|
|
1439
|
-
else {
|
|
1440
|
-
keep(text.charAt(i), i, i + 6);
|
|
1441
|
-
for (let u = 1; u < 6; u++)
|
|
1442
|
-
keep(text.charAt(i + u), i + 6, i + 6);
|
|
1443
|
-
}
|
|
1444
|
-
i += 6;
|
|
1445
|
-
continue;
|
|
1446
|
-
}
|
|
2230
|
+
if (nameBlankTokenAt(text, i) !== -1) {
|
|
2231
|
+
drop(i, i + 6);
|
|
2232
|
+
i += 6;
|
|
2233
|
+
continue;
|
|
1447
2234
|
}
|
|
1448
2235
|
}
|
|
2236
|
+
if (o.blanks && isNameBlank(c)) {
|
|
2237
|
+
drop(i, i + 1);
|
|
2238
|
+
i++;
|
|
2239
|
+
continue;
|
|
2240
|
+
}
|
|
2241
|
+
if (o.escapeSpaces !== null && o.escapeSpaces.has(c)) {
|
|
2242
|
+
drop(i, i + 1, VIEW_SEPARATOR, UNIT_SINGLE, 1);
|
|
2243
|
+
i++;
|
|
2244
|
+
continue;
|
|
2245
|
+
}
|
|
1449
2246
|
if (isStrippableControl(c)) {
|
|
1450
2247
|
drop(i, i + 1);
|
|
1451
2248
|
i++;
|
|
@@ -1467,9 +2264,194 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1467
2264
|
keep(text.charAt(i), i, i + 1);
|
|
1468
2265
|
i++;
|
|
1469
2266
|
}
|
|
1470
|
-
|
|
2267
|
+
const rawUnit = new Uint8Array(text.length);
|
|
2268
|
+
for (let k = 0; k < chars.length; k++) {
|
|
2269
|
+
if (dropped[k] !== 1)
|
|
2270
|
+
continue;
|
|
2271
|
+
for (let x = starts[k] ?? 0; x < (ends[k] ?? 0); x++)
|
|
2272
|
+
rawUnit[x] = 1;
|
|
2273
|
+
}
|
|
2274
|
+
return {
|
|
2275
|
+
view: chars.join(''),
|
|
2276
|
+
starts: Int32Array.from(starts),
|
|
2277
|
+
ends: Int32Array.from(ends),
|
|
2278
|
+
units: Uint8Array.from(units),
|
|
2279
|
+
dropped: Uint8Array.from(dropped),
|
|
2280
|
+
spaceUnits: Uint8Array.from(spaceUnits),
|
|
2281
|
+
contents,
|
|
2282
|
+
rawUnit,
|
|
2283
|
+
};
|
|
2284
|
+
}
|
|
2285
|
+
const viewRawStart = (V, M, k) => {
|
|
2286
|
+
const s = V.starts[k] ?? 0;
|
|
2287
|
+
return M === null ? s : (M.ps[s] ?? 0);
|
|
2288
|
+
};
|
|
2289
|
+
const viewRawEnd = (V, M, k) => {
|
|
2290
|
+
const e = V.ends[k] ?? 0;
|
|
2291
|
+
return M === null ? e : (M.pe[Math.max(0, e - 1)] ?? 0);
|
|
2292
|
+
};
|
|
2293
|
+
const viewPainted = (V, M, k) => {
|
|
2294
|
+
if (M === null)
|
|
2295
|
+
return false;
|
|
2296
|
+
for (let x = V.starts[k] ?? 0; x < (V.ends[k] ?? 0); x++)
|
|
2297
|
+
if (M.pdot[x] === 1)
|
|
2298
|
+
return true;
|
|
2299
|
+
return false;
|
|
2300
|
+
};
|
|
2301
|
+
function separatedSurface(raw, V, M = null) {
|
|
2302
|
+
return {
|
|
2303
|
+
text: V.view,
|
|
2304
|
+
raw,
|
|
2305
|
+
unitAt: (k) => V.units[k] ?? UNIT_NONE,
|
|
2306
|
+
rawStart: (k) => viewRawStart(V, M, k),
|
|
2307
|
+
rawEnd: (k) => viewRawEnd(V, M, k),
|
|
2308
|
+
dropped: (k) => V.dropped[k] === 1,
|
|
2309
|
+
painted: (k) => viewPainted(V, M, k),
|
|
2310
|
+
gapAfter: (k) => V.dropped[k + 1] === 1,
|
|
2311
|
+
spaceGapBefore: () => false,
|
|
2312
|
+
rawDot: M === null ? null : M.rawDot,
|
|
2313
|
+
};
|
|
2314
|
+
}
|
|
2315
|
+
function joinedSurface(raw, V, spacesKept = false, M = null) {
|
|
2316
|
+
const map = [];
|
|
2317
|
+
const chars = [];
|
|
2318
|
+
const spaceBefore = [];
|
|
2319
|
+
let pendingSpace = 0;
|
|
2320
|
+
let keptAny = false;
|
|
2321
|
+
for (let k = 0; k < V.view.length; k++) {
|
|
2322
|
+
if (V.dropped[k] === 1) {
|
|
2323
|
+
if (V.spaceUnits[k] === 1) {
|
|
2324
|
+
if (spacesKept) {
|
|
2325
|
+
map.push(k);
|
|
2326
|
+
chars.push(' ');
|
|
2327
|
+
spaceBefore.push(0);
|
|
2328
|
+
keptAny = true;
|
|
2329
|
+
continue;
|
|
2330
|
+
}
|
|
2331
|
+
pendingSpace = 1;
|
|
2332
|
+
}
|
|
2333
|
+
continue;
|
|
2334
|
+
}
|
|
2335
|
+
map.push(k);
|
|
2336
|
+
chars.push(V.view.charAt(k));
|
|
2337
|
+
spaceBefore.push(pendingSpace);
|
|
2338
|
+
pendingSpace = 0;
|
|
2339
|
+
}
|
|
2340
|
+
if (map.length === V.view.length || (spacesKept && !keptAny))
|
|
1471
2341
|
return null;
|
|
1472
|
-
|
|
2342
|
+
const arr = Int32Array.from(map);
|
|
2343
|
+
const gapSpace = Uint8Array.from(spaceBefore);
|
|
2344
|
+
return {
|
|
2345
|
+
text: chars.join(''),
|
|
2346
|
+
raw,
|
|
2347
|
+
unitAt: () => UNIT_NONE,
|
|
2348
|
+
rawStart: (k) => viewRawStart(V, M, arr[k] ?? 0),
|
|
2349
|
+
rawEnd: (k) => viewRawEnd(V, M, arr[k] ?? 0),
|
|
2350
|
+
dropped: () => false,
|
|
2351
|
+
painted: (k) => viewPainted(V, M, arr[k] ?? 0),
|
|
2352
|
+
gapAfter: (k) => k + 1 < arr.length && arr[k + 1] !== (arr[k] ?? 0) + 1,
|
|
2353
|
+
spaceGapBefore: (k) => gapSpace[k] === 1,
|
|
2354
|
+
rawDot: M === null ? null : M.rawDot,
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
function screenSurfaces(text, o) {
|
|
2358
|
+
const face = o.face;
|
|
2359
|
+
if (face === null)
|
|
2360
|
+
return { surfaces: [], rawSoft: null, rawPlain: null };
|
|
2361
|
+
const unsafe = face.unsafe === null ? null : new RegExp(face.unsafe.source, 'u');
|
|
2362
|
+
const keepBlanks = face.keepBlanks === true;
|
|
2363
|
+
const chars = [];
|
|
2364
|
+
const ps = [];
|
|
2365
|
+
const pe = [];
|
|
2366
|
+
const pdot = [];
|
|
2367
|
+
const rawDot = new Uint8Array(text.length);
|
|
2368
|
+
const rawPlain = new Uint8Array(text.length);
|
|
2369
|
+
let changed = false;
|
|
2370
|
+
const foldable = (at) => {
|
|
2371
|
+
const c = text.codePointAt(at) ?? 0;
|
|
2372
|
+
return isRegexSpace(c) && !(keepBlanks && isNameBlank(c));
|
|
2373
|
+
};
|
|
2374
|
+
let lo = 0;
|
|
2375
|
+
let hi = text.length;
|
|
2376
|
+
if (face.fold) {
|
|
2377
|
+
while (lo < hi && foldable(lo))
|
|
2378
|
+
lo++;
|
|
2379
|
+
while (hi > lo && foldable(hi - 1))
|
|
2380
|
+
hi--;
|
|
2381
|
+
if (lo > 0 || hi < text.length)
|
|
2382
|
+
changed = true;
|
|
2383
|
+
}
|
|
2384
|
+
for (let i = lo; i < hi;) {
|
|
2385
|
+
const cp = text.codePointAt(i) ?? 0;
|
|
2386
|
+
const w = cp > 0xffff ? 2 : 1;
|
|
2387
|
+
const ch = text.slice(i, i + w);
|
|
2388
|
+
let form = null;
|
|
2389
|
+
let dotted = false;
|
|
2390
|
+
if (face.fold && foldable(i)) {
|
|
2391
|
+
let j = i + 1;
|
|
2392
|
+
while (j < hi && foldable(j))
|
|
2393
|
+
j++;
|
|
2394
|
+
if (j - i > 1 || cp !== 0x20)
|
|
2395
|
+
changed = true;
|
|
2396
|
+
chars.push(' ');
|
|
2397
|
+
ps.push(i);
|
|
2398
|
+
pe.push(j);
|
|
2399
|
+
pdot.push(cp === 0xfeff || j - i > 1 ? 1 : 0);
|
|
2400
|
+
for (let x = i; x < j; x++)
|
|
2401
|
+
if (text.charCodeAt(x) === 0xfeff)
|
|
2402
|
+
rawDot[x] = 1;
|
|
2403
|
+
i = j;
|
|
2404
|
+
continue;
|
|
2405
|
+
}
|
|
2406
|
+
else if (unsafe !== null && unsafe.test(ch) && !faceHitKept(ch, i, text)) {
|
|
2407
|
+
form = markOf(ch, face.mark);
|
|
2408
|
+
dotted = face.mark !== 'escape';
|
|
2409
|
+
}
|
|
2410
|
+
if (form === null || form === ch) {
|
|
2411
|
+
for (let u = 0; u < w; u++) {
|
|
2412
|
+
chars.push(text.charAt(i + u));
|
|
2413
|
+
ps.push(i + u);
|
|
2414
|
+
pe.push(i + u + 1);
|
|
2415
|
+
pdot.push(0);
|
|
2416
|
+
rawPlain[i + u] = 1;
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
else {
|
|
2420
|
+
changed = true;
|
|
2421
|
+
for (let u = 0; u < form.length; u++) {
|
|
2422
|
+
chars.push(form.charAt(u));
|
|
2423
|
+
ps.push(i);
|
|
2424
|
+
pe.push(i + w);
|
|
2425
|
+
pdot.push(dotted ? 1 : 0);
|
|
2426
|
+
}
|
|
2427
|
+
if (dotted)
|
|
2428
|
+
for (let x = i; x < i + w; x++)
|
|
2429
|
+
rawDot[x] = 1;
|
|
2430
|
+
}
|
|
2431
|
+
i += w;
|
|
2432
|
+
}
|
|
2433
|
+
if (!changed)
|
|
2434
|
+
return { surfaces: [], rawSoft: null, rawPlain: null };
|
|
2435
|
+
const screen = chars.join('');
|
|
2436
|
+
const M = { ps: Int32Array.from(ps), pe: Int32Array.from(pe), pdot: Uint8Array.from(pdot), rawDot };
|
|
2437
|
+
const V = buildScanView(screen, o);
|
|
2438
|
+
const rawSoft = Uint8Array.from(rawDot);
|
|
2439
|
+
for (let k = 0; k < V.view.length; k++) {
|
|
2440
|
+
if (V.dropped[k] !== 1)
|
|
2441
|
+
continue;
|
|
2442
|
+
for (let x = viewRawStart(V, M, k); x < viewRawEnd(V, M, k); x++) {
|
|
2443
|
+
rawSoft[x] = 1;
|
|
2444
|
+
rawPlain[x] = 0;
|
|
2445
|
+
}
|
|
2446
|
+
}
|
|
2447
|
+
const out = [separatedSurface(text, V, M)];
|
|
2448
|
+
const joined = joinedSurface(text, V, false, M);
|
|
2449
|
+
if (joined !== null)
|
|
2450
|
+
out.push(joined);
|
|
2451
|
+
const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true, M) : null;
|
|
2452
|
+
if (joinedSpaced !== null)
|
|
2453
|
+
out.push(joinedSpaced);
|
|
2454
|
+
return { surfaces: out, rawSoft, rawPlain };
|
|
1473
2455
|
}
|
|
1474
2456
|
function finalIsStructural(text, at) {
|
|
1475
2457
|
if (text.charCodeAt(at) !== 0x5c)
|
|
@@ -1497,260 +2479,340 @@ function credentialWordBeginsWith(text, at) {
|
|
|
1497
2479
|
let end = Math.min(text.length, at + 64);
|
|
1498
2480
|
while (end < text.length && isProbeFiller(text.charCodeAt(end - 1)))
|
|
1499
2481
|
end++;
|
|
1500
|
-
const probe = ' ' + text.slice(at, Math.min(text.length, end + 2));
|
|
2482
|
+
const probe = ' ' + text.slice(at, Math.min(text.length, end + 2)) + (end + 2 >= text.length ? ' ' : '');
|
|
1501
2483
|
if (letter)
|
|
1502
2484
|
return credentialWordStartsAt(probe, 1);
|
|
1503
2485
|
return credentialWordStartsAt(probe, 2) || credentialWordStartsAt(probe, 3);
|
|
1504
2486
|
}
|
|
1505
|
-
function
|
|
2487
|
+
function editsToCandidates(edits, cls, out, te = -1) {
|
|
2488
|
+
for (const e of edits) {
|
|
2489
|
+
const kind = KIND_BY_TEXT.get(e.text);
|
|
2490
|
+
if (kind === undefined || e.end <= e.start)
|
|
2491
|
+
continue;
|
|
2492
|
+
out.push({ a: e.start, b: e.end, kind, cls, te });
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
function pemRegions(recognized) {
|
|
2496
|
+
const seen = new Set();
|
|
1506
2497
|
const out = [];
|
|
1507
|
-
const
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
origin.push(x);
|
|
1513
|
-
};
|
|
1514
|
-
const inserted = (s) => {
|
|
1515
|
-
out.push(s);
|
|
1516
|
-
if (origin !== null)
|
|
1517
|
-
for (let x = 0; x < s.length; x++)
|
|
1518
|
-
origin.push(-1);
|
|
1519
|
-
};
|
|
1520
|
-
const viewLength = view.view.length;
|
|
1521
|
-
let cursor = 0;
|
|
1522
|
-
let k = 0;
|
|
1523
|
-
const text = target.text;
|
|
1524
|
-
while (k < text.length) {
|
|
1525
|
-
const s = target.src[k] ?? 0;
|
|
1526
|
-
if (s >= 0) {
|
|
1527
|
-
const end = view.ends[s] ?? cursor;
|
|
1528
|
-
if ((view.starts[s] ?? 0) >= cursor) {
|
|
1529
|
-
fromOriginal(cursor, end);
|
|
1530
|
-
cursor = end;
|
|
1531
|
-
}
|
|
1532
|
-
else
|
|
1533
|
-
inserted(text.charAt(k));
|
|
1534
|
-
k++;
|
|
2498
|
+
for (const blk of recognized) {
|
|
2499
|
+
if (blk.regionEnd <= blk.begin)
|
|
2500
|
+
continue;
|
|
2501
|
+
const key = `${blk.begin}:${blk.bodyStart}:${blk.bodyEnd}:${blk.regionEnd}`;
|
|
2502
|
+
if (seen.has(key))
|
|
1535
2503
|
continue;
|
|
2504
|
+
seen.add(key);
|
|
2505
|
+
out.push(blk);
|
|
2506
|
+
}
|
|
2507
|
+
return out;
|
|
2508
|
+
}
|
|
2509
|
+
function resolveCandidates(text, cands, rawUnit, rawDot, pems, rawSoft = rawDot, rawPlain = null) {
|
|
2510
|
+
const items = [];
|
|
2511
|
+
for (const c of cands) {
|
|
2512
|
+
let a = c.start;
|
|
2513
|
+
let b = c.end;
|
|
2514
|
+
if (c.cls === 'pem') {
|
|
1536
2515
|
}
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
2516
|
+
else if (c.dotted === true) {
|
|
2517
|
+
while (a < b && rawDot !== null && rawDot[a] === 1)
|
|
2518
|
+
a++;
|
|
2519
|
+
if (a >= b)
|
|
2520
|
+
a = c.start;
|
|
2521
|
+
else if (rawDot !== null && b < text.length && isKeyWordChar(text.charCodeAt(b))) {
|
|
2522
|
+
while (b > a && rawDot[b - 1] === 1)
|
|
2523
|
+
b--;
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
else if (c.pretrimmed !== true) {
|
|
2527
|
+
while (a < b && rawUnit[a] === 1)
|
|
2528
|
+
a++;
|
|
2529
|
+
while (b > a && rawUnit[b - 1] === 1)
|
|
2530
|
+
b--;
|
|
2531
|
+
if (rawPlain !== null && rawDot !== null) {
|
|
2532
|
+
let k = a;
|
|
2533
|
+
while (k > 0 && rawUnit[k - 1] === 1 && rawPlain[k - 1] === 1 && isKeyWordChar(text.charCodeAt(k - 1)))
|
|
2534
|
+
k--;
|
|
2535
|
+
if (k < a && k > 0 && (rawDot[k - 1] === 1 || isRegexSpace(text.charCodeAt(k - 1))))
|
|
2536
|
+
a = k;
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
if (a >= b)
|
|
2540
|
+
continue;
|
|
2541
|
+
const body = text.slice(a, b);
|
|
2542
|
+
if (isMarker(body) || MARKER_WITH_DOTS.test(body))
|
|
2543
|
+
continue;
|
|
2544
|
+
items.push({ start: a, end: b, kind: c.kind, cls: c.cls, tokenEnd: c.tokenEnd, origin: c.dotted === true ? c.start : a });
|
|
1553
2545
|
}
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
function washThroughView(original, view, urls, words, trace = false, literalBlanks = false) {
|
|
1559
|
-
const target = new TrackedTarget(view.view, view.view.length, view.units);
|
|
1560
|
-
runCredentialNets(target, urls, words, literalBlanks);
|
|
1561
|
-
return target.text === view.view ? { text: original } : rebuildFromView(original, view, target, trace);
|
|
1562
|
-
}
|
|
1563
|
-
function washCredentialNetsOnce(text, urls, words, nested, blankTokens, trace) {
|
|
1564
|
-
const contents = nested ? { text } : washControlStringContents(text, urls, words, blankTokens, trace);
|
|
1565
|
-
if (blankTokens === 'off')
|
|
1566
|
-
return washThroughViews(contents, urls, words, 'off', trace);
|
|
1567
|
-
const tokens = hasNameBlankToken(contents.text);
|
|
1568
|
-
const readings = [];
|
|
1569
|
-
if (tokens)
|
|
1570
|
-
readings.push('literal');
|
|
1571
|
-
const rawFolded = blankTokens === 'fold' && hasRawFoldedBlank(contents.text);
|
|
1572
|
-
if (blankTokens === 'fold' && (tokens || rawFolded))
|
|
1573
|
-
readings.push('fold');
|
|
1574
|
-
if (blankTokens === 'fold' && tokens && hasNameBlankToken(contents.text, false) && (rawFolded || hasNameBlankToken(contents.text, true)))
|
|
1575
|
-
readings.push('foldlit');
|
|
1576
|
-
readings.push('blank');
|
|
1577
|
-
if (readings.length === 1)
|
|
1578
|
-
return washThroughViews(contents, urls, words, 'blank', trace);
|
|
1579
|
-
const base = { text: contents.text };
|
|
1580
|
-
const union = unionMaskedRanges(base.text, readings.map((r) => washThroughViews(base, urls, words, r, true)));
|
|
1581
|
-
return traced(union.text, composeOrigin(contents.origin, union.origin));
|
|
1582
|
-
}
|
|
1583
|
-
function unionMaskedRanges(text, results) {
|
|
1584
|
-
const ranges = [];
|
|
1585
|
-
for (const r of results) {
|
|
1586
|
-
if (r.text === text || r.origin === undefined)
|
|
2546
|
+
items.sort((x, y) => (x.start !== y.start ? x.start - y.start : y.end - x.end));
|
|
2547
|
+
let maxTokenEnd = null;
|
|
2548
|
+
for (const x of items) {
|
|
2549
|
+
if (!isStructuralKind(x.kind) || x.cls !== 'url')
|
|
1587
2550
|
continue;
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
let k =
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
2551
|
+
if (maxTokenEnd === null)
|
|
2552
|
+
maxTokenEnd = new Int32Array(text.length).fill(-1);
|
|
2553
|
+
for (let k = x.start; k < x.end && k < text.length; k++)
|
|
2554
|
+
if (x.tokenEnd > (maxTokenEnd[k] ?? -1))
|
|
2555
|
+
maxTokenEnd[k] = x.tokenEnd;
|
|
2556
|
+
}
|
|
2557
|
+
const p2Drops = (x) => !isStructuralKind(x.kind) && x.cls !== 'pem' && x.cls !== 'glued' && maxTokenEnd !== null && x.end <= (maxTokenEnd[x.start] ?? -1);
|
|
2558
|
+
let pemCover = null;
|
|
2559
|
+
if (pems.length > 0) {
|
|
2560
|
+
const reachBefore = new Int32Array(text.length + 1).fill(-1);
|
|
2561
|
+
for (const x of items)
|
|
2562
|
+
if (x.cls !== 'pem' && !p2Drops(x) && x.origin < text.length && x.end > (reachBefore[x.origin + 1] ?? -1))
|
|
2563
|
+
reachBefore[x.origin + 1] = x.end;
|
|
2564
|
+
for (let k = 1; k <= text.length; k++)
|
|
2565
|
+
if ((reachBefore[k - 1] ?? -1) > (reachBefore[k] ?? -1))
|
|
2566
|
+
reachBefore[k] = reachBefore[k - 1] ?? -1;
|
|
2567
|
+
let pemMask = null;
|
|
2568
|
+
for (const x of items) {
|
|
2569
|
+
if (x.cls !== 'pem')
|
|
2570
|
+
continue;
|
|
2571
|
+
pemMask ??= new Uint8Array(text.length);
|
|
2572
|
+
for (let k = x.start; k < x.end && k < text.length; k++)
|
|
2573
|
+
pemMask[k] = 1;
|
|
2574
|
+
}
|
|
2575
|
+
for (const blk of pems) {
|
|
2576
|
+
const a = blk.begin;
|
|
2577
|
+
const b = blk.regionEnd;
|
|
2578
|
+
if ((reachBefore[a] ?? -1) > a)
|
|
1595
2579
|
continue;
|
|
2580
|
+
pemCover ??= new Uint8Array(text.length);
|
|
2581
|
+
for (let k = a; k < b && k < text.length; k++) {
|
|
2582
|
+
if (k >= blk.bodyStart && k < blk.bodyEnd && rawUnit[k] === 1 && pemMask?.[k] !== 1)
|
|
2583
|
+
continue;
|
|
2584
|
+
pemCover[k] = 1;
|
|
1596
2585
|
}
|
|
1597
|
-
let j = k;
|
|
1598
|
-
while (j < origin.length && (origin[j] ?? -1) < 0)
|
|
1599
|
-
j++;
|
|
1600
|
-
const next = j < origin.length ? (origin[j] ?? text.length) : text.length;
|
|
1601
|
-
ranges.push({ start: lastKept + 1, end: next, text: r.text.slice(k, j) });
|
|
1602
|
-
k = j;
|
|
1603
2586
|
}
|
|
1604
2587
|
}
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2588
|
+
const kept = items.filter((x) => {
|
|
2589
|
+
if (x.cls !== 'pem' && pemCover !== null && pemCover[x.origin] === 1)
|
|
2590
|
+
return false;
|
|
2591
|
+
if (p2Drops(x))
|
|
2592
|
+
return false;
|
|
2593
|
+
return true;
|
|
2594
|
+
});
|
|
2595
|
+
const groups = [];
|
|
2596
|
+
for (const it of kept) {
|
|
2597
|
+
const g = groups[groups.length - 1];
|
|
2598
|
+
if (g !== undefined && it.start < g.end) {
|
|
2599
|
+
if (it.end > g.end)
|
|
2600
|
+
g.end = it.end;
|
|
2601
|
+
g.members.push(it);
|
|
1617
2602
|
continue;
|
|
1618
2603
|
}
|
|
1619
|
-
|
|
1620
|
-
continue;
|
|
1621
|
-
merged.push({ ...m });
|
|
1622
|
-
}
|
|
1623
|
-
const parts = [];
|
|
1624
|
-
const origin = [];
|
|
1625
|
-
let cursor = 0;
|
|
1626
|
-
for (const m of merged) {
|
|
1627
|
-
parts.push(text.slice(cursor, m.start), m.text);
|
|
1628
|
-
for (let x = cursor; x < m.start; x++)
|
|
1629
|
-
origin.push(x);
|
|
1630
|
-
for (let x = 0; x < m.text.length; x++)
|
|
1631
|
-
origin.push(-1);
|
|
1632
|
-
cursor = m.end;
|
|
2604
|
+
groups.push({ start: it.start, end: it.end, members: [it] });
|
|
1633
2605
|
}
|
|
1634
|
-
|
|
1635
|
-
for (
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
2606
|
+
const out = [];
|
|
2607
|
+
for (const g of groups) {
|
|
2608
|
+
let structStart = Infinity;
|
|
2609
|
+
let structEnd = -Infinity;
|
|
2610
|
+
let firstStructKind = null;
|
|
2611
|
+
let firstStructStart = Infinity;
|
|
2612
|
+
let anySecret = false;
|
|
2613
|
+
const secretLike = (m) => !isStructuralKind(m.kind) || m.cls === 'userinfo-hidden';
|
|
2614
|
+
for (const m of g.members) {
|
|
2615
|
+
if (!secretLike(m)) {
|
|
2616
|
+
if (m.start < structStart)
|
|
2617
|
+
structStart = m.start;
|
|
2618
|
+
if (m.end > structEnd)
|
|
2619
|
+
structEnd = m.end;
|
|
2620
|
+
if (m.start < firstStructStart) {
|
|
2621
|
+
firstStructStart = m.start;
|
|
2622
|
+
firstStructKind = m.kind;
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
else
|
|
2626
|
+
anySecret = true;
|
|
2627
|
+
}
|
|
2628
|
+
let secretWider = false;
|
|
2629
|
+
if (anySecret)
|
|
2630
|
+
for (const m of g.members)
|
|
2631
|
+
if (secretLike(m) && (m.start < structStart || m.end > structEnd))
|
|
2632
|
+
secretWider = true;
|
|
2633
|
+
const onlyHidden = g.members.every((m) => m.cls === 'userinfo-hidden');
|
|
2634
|
+
const kind = onlyHidden ? 'userinfo' : firstStructKind === null || secretWider ? 'secret' : firstStructKind;
|
|
2635
|
+
const value = g.members.some((m) => m.cls === 'word' || m.cls === 'glued' || m.cls === 'param' || (m.kind !== 'userinfo' && m.kind !== 'secret' && (m.cls === 'url' || m.cls === 'structural')));
|
|
2636
|
+
out.push({ start: g.start, end: g.end, kind, value });
|
|
2637
|
+
}
|
|
2638
|
+
return continueValues(text, out, rawSoft ?? rawUnit);
|
|
2639
|
+
}
|
|
2640
|
+
function continueValues(text, drafts, rawSoft) {
|
|
2641
|
+
const out = [];
|
|
2642
|
+
const soft = (k) => rawSoft[k] === 1;
|
|
2643
|
+
let i = 0;
|
|
2644
|
+
while (i < drafts.length) {
|
|
2645
|
+
const cur = drafts[i];
|
|
2646
|
+
i++;
|
|
2647
|
+
if (cur === undefined)
|
|
2648
|
+
continue;
|
|
2649
|
+
const d = { start: cur.start, end: cur.end, kind: cur.kind, value: cur.value };
|
|
2650
|
+
for (;;) {
|
|
2651
|
+
let k = d.end;
|
|
2652
|
+
if (!d.value || k >= text.length || !soft(k))
|
|
2653
|
+
break;
|
|
2654
|
+
while (k < text.length && soft(k))
|
|
2655
|
+
k++;
|
|
2656
|
+
if (k >= text.length || !isKeyWordChar(text.charCodeAt(k)) || credentialWordStartsAt(text, k))
|
|
2657
|
+
break;
|
|
2658
|
+
let e = k;
|
|
2659
|
+
while (e < text.length && (soft(e) || !isBareValueStop(text.charAt(e))))
|
|
2660
|
+
e++;
|
|
2661
|
+
d.end = e;
|
|
2662
|
+
while (i < drafts.length && (drafts[i]?.start ?? Infinity) < d.end) {
|
|
2663
|
+
const next = drafts[i];
|
|
2664
|
+
i++;
|
|
2665
|
+
if (next === undefined)
|
|
2666
|
+
continue;
|
|
2667
|
+
if (next.end > d.end)
|
|
2668
|
+
d.end = next.end;
|
|
2669
|
+
if (next.kind !== d.kind)
|
|
2670
|
+
d.kind = 'secret';
|
|
2671
|
+
}
|
|
2672
|
+
}
|
|
2673
|
+
out.push(d);
|
|
1652
2674
|
}
|
|
1653
|
-
return
|
|
2675
|
+
return out;
|
|
1654
2676
|
}
|
|
1655
|
-
function
|
|
1656
|
-
|
|
1657
|
-
const literal = blankRead === 'literal' || blankRead === 'foldlit';
|
|
1658
|
-
const lettered = buildCredentialView(start, true, true, blankRead);
|
|
1659
|
-
const letteredWashed = lettered === null ? { text: start } : washThroughView(start, lettered, urls, words, trace, literal);
|
|
1660
|
-
const pre = letteredWashed.text;
|
|
1661
|
-
let origin = composeOrigin(input.origin, letteredWashed.origin);
|
|
1662
|
-
const separated = buildCredentialView(pre, true, false, blankRead);
|
|
1663
|
-
if (separated === null) {
|
|
1664
|
-
const target = new PlainTarget(pre, trace);
|
|
1665
|
-
runCredentialNets(target, urls, words, literal);
|
|
1666
|
-
return traced(target.text, composeOrigin(origin, target.origin));
|
|
1667
|
-
}
|
|
1668
|
-
const afterSeparated = washThroughView(pre, separated, urls, words, trace, literal);
|
|
1669
|
-
origin = composeOrigin(origin, afterSeparated.origin);
|
|
1670
|
-
const joined = buildCredentialView(afterSeparated.text, false, false, blankRead);
|
|
1671
|
-
if (joined === null)
|
|
1672
|
-
return traced(afterSeparated.text, origin);
|
|
1673
|
-
const afterJoined = washThroughView(afterSeparated.text, joined, urls, words, trace, literal);
|
|
1674
|
-
return traced(afterJoined.text, composeOrigin(origin, afterJoined.origin));
|
|
1675
|
-
}
|
|
1676
|
-
function washControlStringContents(text, urls, words, blankTokens, trace) {
|
|
1677
|
-
if (!/[\u001b\u0090\u0098\u009d-\u009f]|\\u/.test(text))
|
|
2677
|
+
function materialize(text, intervals, trace) {
|
|
2678
|
+
if (intervals.length === 0)
|
|
1678
2679
|
return { text };
|
|
1679
|
-
const ends = new SequenceReader(text, 'none');
|
|
1680
2680
|
const parts = [];
|
|
1681
2681
|
const origin = trace ? [] : null;
|
|
1682
|
-
const keptUpTo = (a, b) => {
|
|
1683
|
-
if (origin !== null)
|
|
1684
|
-
for (let x = a; x < b; x++)
|
|
1685
|
-
origin.push(x);
|
|
1686
|
-
};
|
|
1687
2682
|
let cursor = 0;
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
unit = c1SequenceAfter(text, c, i + 1, ends);
|
|
1697
|
-
else if (c === 0x5c && text.charCodeAt(i + 1) === 0x75) {
|
|
1698
|
-
const tok = escapeTokenAt(text, i);
|
|
1699
|
-
if (tok !== null) {
|
|
1700
|
-
next = tok.end;
|
|
1701
|
-
unit = tok.cp === 0x1b ? escapeSequenceAfterEsc(text, tok.end, ends) : tok.cp >= 0x90 && tok.cp <= 0x9f ? c1SequenceAfter(text, tok.cp, tok.end, ends) : null;
|
|
1702
|
-
}
|
|
2683
|
+
for (const iv of intervals) {
|
|
2684
|
+
const token = TEXT_BY_KIND[iv.kind];
|
|
2685
|
+
parts.push(text.slice(cursor, iv.start), token);
|
|
2686
|
+
if (origin !== null) {
|
|
2687
|
+
for (let x = cursor; x < iv.start; x++)
|
|
2688
|
+
origin.push(x);
|
|
2689
|
+
for (let x = 0; x < token.length; x++)
|
|
2690
|
+
origin.push(-1);
|
|
1703
2691
|
}
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
2692
|
+
cursor = iv.end;
|
|
2693
|
+
}
|
|
2694
|
+
parts.push(text.slice(cursor));
|
|
2695
|
+
if (origin !== null)
|
|
2696
|
+
for (let x = cursor; x < text.length; x++)
|
|
2697
|
+
origin.push(x);
|
|
2698
|
+
return origin === null ? { text: parts.join('') } : { text: parts.join(''), origin: Int32Array.from(origin) };
|
|
2699
|
+
}
|
|
2700
|
+
function toRaw(S, c) {
|
|
2701
|
+
let a = c.a;
|
|
2702
|
+
let b = c.b;
|
|
2703
|
+
while (a < b && S.dropped(a))
|
|
2704
|
+
a++;
|
|
2705
|
+
while (b > a && S.dropped(b - 1))
|
|
2706
|
+
b--;
|
|
2707
|
+
if (a >= b)
|
|
2708
|
+
return null;
|
|
2709
|
+
const te = c.te > b ? S.rawEnd(c.te - 1) : S.rawEnd(b - 1);
|
|
2710
|
+
return { start: S.rawStart(a), end: S.rawEnd(b - 1), kind: c.kind, cls: c.cls, tokenEnd: te };
|
|
2711
|
+
}
|
|
2712
|
+
function collectCandidates(text, o, nested) {
|
|
2713
|
+
const V = buildScanView(text, o);
|
|
2714
|
+
const separated = separatedSurface(text, V);
|
|
2715
|
+
const surfaces = [separated];
|
|
2716
|
+
const joined = joinedSurface(text, V);
|
|
2717
|
+
if (joined !== null)
|
|
2718
|
+
surfaces.push(joined);
|
|
2719
|
+
const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true) : null;
|
|
2720
|
+
if (joinedSpaced !== null)
|
|
2721
|
+
surfaces.push(joinedSpaced);
|
|
2722
|
+
const screen = screenSurfaces(text, o);
|
|
2723
|
+
surfaces.push(...screen.surfaces);
|
|
2724
|
+
const raw = [];
|
|
2725
|
+
const regionSurface = screen.surfaces[0] ?? separated;
|
|
2726
|
+
const pemBlocksSeen = [];
|
|
2727
|
+
for (const S of surfaces) {
|
|
2728
|
+
const cands = [];
|
|
2729
|
+
const regions = [];
|
|
2730
|
+
if (o.urls) {
|
|
2731
|
+
const unitPos = (k) => S.dropped(k) || V.rawUnit[S.rawStart(k)] === 1;
|
|
2732
|
+
urlCandidates(S.text, cands, regions, unitPos, (k) => unitPos(k) || S.painted(k));
|
|
2733
|
+
userinfoCandidates(S, cands, o.words);
|
|
1707
2734
|
}
|
|
1708
|
-
if (
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
2735
|
+
if (o.words) {
|
|
2736
|
+
editsToCandidates(pemEdits(S, (blk) => { if (S === regionSurface)
|
|
2737
|
+
pemBlocksSeen.push(pemBlockToRaw(S, blk)); }), 'pem', cands);
|
|
2738
|
+
labelCandidates(S, SECRET_SCHEME_WORD, regions, cands);
|
|
2739
|
+
labelCandidates(S, SECRET_LABELLED_WORD, regions, cands);
|
|
2740
|
+
editsToCandidates(authParamEdits(S.text), 'param', cands);
|
|
2741
|
+
editsToCandidates(proseLiteralEdits(S.text), 'prose', cands);
|
|
2742
|
+
}
|
|
2743
|
+
for (const c of cands) {
|
|
2744
|
+
const r = toRaw(S, c);
|
|
2745
|
+
if (r !== null)
|
|
2746
|
+
raw.push(S.rawDot !== null ? { ...r, dotted: true } : r);
|
|
2747
|
+
}
|
|
2748
|
+
}
|
|
2749
|
+
if (!nested) {
|
|
2750
|
+
for (const [a, b] of V.contents) {
|
|
2751
|
+
const sub = text.slice(a, b);
|
|
2752
|
+
const inner = collectCandidates(sub, o, true);
|
|
2753
|
+
for (const iv of resolveCandidates(sub, inner.cands, inner.rawUnit, inner.rawDot, o.words ? pemRegions(inner.pemBlocksSeen) : [], inner.rawSoft, inner.rawPlain)) {
|
|
2754
|
+
raw.push({ start: a + iv.start, end: a + iv.end, kind: iv.kind, cls: 'nested', tokenEnd: a + iv.end, pretrimmed: true });
|
|
1723
2755
|
}
|
|
1724
2756
|
}
|
|
1725
|
-
i = Math.max(unit.end, i + 1);
|
|
1726
2757
|
}
|
|
1727
|
-
|
|
1728
|
-
return { text };
|
|
1729
|
-
parts.push(text.slice(cursor));
|
|
1730
|
-
keptUpTo(cursor, text.length);
|
|
1731
|
-
const joined = parts.join('');
|
|
1732
|
-
return origin === null ? { text: joined } : { text: joined, origin: Int32Array.from(origin) };
|
|
2758
|
+
return { cands: raw, rawUnit: V.rawUnit, rawDot: screen.surfaces[0]?.rawDot ?? null, rawSoft: screen.rawSoft, rawPlain: screen.rawPlain, pemBlocksSeen };
|
|
1733
2759
|
}
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
2760
|
+
function scanOnce(text, o, trace) {
|
|
2761
|
+
const { cands, rawUnit, rawDot, rawSoft, rawPlain, pemBlocksSeen } = collectCandidates(text, o, false);
|
|
2762
|
+
const intervals = resolveCandidates(text, cands, rawUnit, rawDot, o.words ? pemRegions(pemBlocksSeen) : [], rawSoft, rawPlain);
|
|
2763
|
+
return materialize(text, intervals, trace);
|
|
2764
|
+
}
|
|
2765
|
+
const MAX_WASH_REWRITES = 8;
|
|
2766
|
+
function composeOrigin(first, second) {
|
|
2767
|
+
if (second === undefined)
|
|
2768
|
+
return first;
|
|
2769
|
+
if (first === undefined)
|
|
2770
|
+
return second;
|
|
2771
|
+
const out = new Int32Array(second.length);
|
|
2772
|
+
for (let k = 0; k < second.length; k++) {
|
|
2773
|
+
const s = second[k] ?? -1;
|
|
2774
|
+
out[k] = s < 0 ? -1 : (first[s] ?? -1);
|
|
1744
2775
|
}
|
|
1745
|
-
return
|
|
2776
|
+
return out;
|
|
1746
2777
|
}
|
|
1747
|
-
function
|
|
1748
|
-
|
|
2778
|
+
function scanAndMask(text, o, trace, stats) {
|
|
2779
|
+
if (!o.urls && !o.words) {
|
|
2780
|
+
if (stats !== undefined) {
|
|
2781
|
+
stats.passes = 0;
|
|
2782
|
+
stats.capped = false;
|
|
2783
|
+
}
|
|
2784
|
+
return { text };
|
|
2785
|
+
}
|
|
2786
|
+
let cur = scanOnce(text, o, trace);
|
|
2787
|
+
let passes = 1;
|
|
2788
|
+
let rewrites = cur.text !== text ? 1 : 0;
|
|
2789
|
+
let changed = rewrites > 0;
|
|
2790
|
+
while (changed && rewrites < MAX_WASH_REWRITES) {
|
|
2791
|
+
const next = scanOnce(cur.text, o, trace);
|
|
2792
|
+
passes++;
|
|
2793
|
+
changed = next.text !== cur.text;
|
|
2794
|
+
if (!changed)
|
|
2795
|
+
break;
|
|
2796
|
+
rewrites++;
|
|
2797
|
+
const origin = trace ? composeOrigin(cur.origin, next.origin) : undefined;
|
|
2798
|
+
cur = origin === undefined ? { text: next.text } : { text: next.text, origin };
|
|
2799
|
+
}
|
|
2800
|
+
if (stats !== undefined) {
|
|
2801
|
+
let capped = false;
|
|
2802
|
+
if (changed) {
|
|
2803
|
+
passes++;
|
|
2804
|
+
capped = scanOnce(cur.text, o, false).text !== cur.text;
|
|
2805
|
+
}
|
|
2806
|
+
stats.passes = passes;
|
|
2807
|
+
stats.capped = capped;
|
|
2808
|
+
}
|
|
2809
|
+
return cur;
|
|
1749
2810
|
}
|
|
2811
|
+
const WASH_OPTIONS = Object.freeze({ urls: true, words: true, blanks: false, fold: false, face: null, escapeSpaces: null });
|
|
1750
2812
|
export function washCredentials(text) {
|
|
1751
2813
|
if (typeof text !== 'string')
|
|
1752
2814
|
return text;
|
|
1753
|
-
return
|
|
2815
|
+
return scanAndMask(text, WASH_OPTIONS, false).text;
|
|
1754
2816
|
}
|
|
1755
2817
|
const C0_ALL = '\\u0000-\\u001F';
|
|
1756
2818
|
const C0_KEEP_LAYOUT = '\\u0000-\\u0008\\u000B-\\u001F';
|
|
@@ -2089,42 +3151,64 @@ function capDisplay(text, max, mark) {
|
|
|
2089
3151
|
}
|
|
2090
3152
|
const MAX_CAP_SETTLE_ROUNDS = 4;
|
|
2091
3153
|
function runDisplaySteps(start, steps) {
|
|
2092
|
-
const
|
|
2093
|
-
|
|
2094
|
-
if (steps.text(current) === steps.text(s))
|
|
2095
|
-
return current;
|
|
2096
|
-
for (let round = 1; round < MAX_SETTLE_ROUNDS; round++) {
|
|
2097
|
-
const next = steps.paint(steps.wash(current));
|
|
2098
|
-
if (steps.text(next) === steps.text(current))
|
|
2099
|
-
return current;
|
|
2100
|
-
current = next;
|
|
2101
|
-
}
|
|
2102
|
-
return current;
|
|
2103
|
-
};
|
|
2104
|
-
const settled = settle(start);
|
|
3154
|
+
const once = (s) => steps.paint(steps.wash(s));
|
|
3155
|
+
const settled = once(start);
|
|
2105
3156
|
let out = steps.cap(settled);
|
|
2106
3157
|
if (steps.text(out) === steps.text(settled))
|
|
2107
3158
|
return out;
|
|
2108
3159
|
for (let round = 0; round < MAX_CAP_SETTLE_ROUNDS; round++) {
|
|
2109
|
-
const again = steps.cap(
|
|
3160
|
+
const again = steps.cap(once(out));
|
|
2110
3161
|
if (steps.text(again) === steps.text(out))
|
|
2111
3162
|
break;
|
|
2112
3163
|
out = again;
|
|
2113
3164
|
}
|
|
2114
3165
|
return out;
|
|
2115
3166
|
}
|
|
3167
|
+
function escapeSpacesFor(o, face) {
|
|
3168
|
+
if (face.mark !== 'escape' || face.fold)
|
|
3169
|
+
return null;
|
|
3170
|
+
const set = new Set();
|
|
3171
|
+
if (o.controls !== false) {
|
|
3172
|
+
for (const c of [0x09, 0x0a, 0x0b, 0x0c, 0x0d])
|
|
3173
|
+
set.add(c);
|
|
3174
|
+
if (o.keepLayout === true) {
|
|
3175
|
+
set.delete(0x09);
|
|
3176
|
+
set.delete(0x0a);
|
|
3177
|
+
}
|
|
3178
|
+
}
|
|
3179
|
+
if (o.bidi !== false) {
|
|
3180
|
+
set.add(0x2028);
|
|
3181
|
+
set.add(0x2029);
|
|
3182
|
+
}
|
|
3183
|
+
return set.size === 0 ? null : set;
|
|
3184
|
+
}
|
|
3185
|
+
function scanOptionsOf(o, face) {
|
|
3186
|
+
return {
|
|
3187
|
+
urls: o.credentialUrls !== false,
|
|
3188
|
+
words: o.credentialWords !== false,
|
|
3189
|
+
blanks: face.keepBlanks === true,
|
|
3190
|
+
fold: face.fold,
|
|
3191
|
+
face,
|
|
3192
|
+
escapeSpaces: escapeSpacesFor(o, face),
|
|
3193
|
+
};
|
|
3194
|
+
}
|
|
3195
|
+
export function credentialWashPasses(text, opts = {}) {
|
|
3196
|
+
const input = typeof text === 'string' ? text : String(text ?? '');
|
|
3197
|
+
const stats = { passes: 0, capped: false };
|
|
3198
|
+
const scan = opts === null ? WASH_OPTIONS : scanOptionsOf(opts, publicFace(opts));
|
|
3199
|
+
scanAndMask(input, scan, false, stats);
|
|
3200
|
+
return { passes: stats.passes, capped: stats.capped };
|
|
3201
|
+
}
|
|
2116
3202
|
function renderUntrusted(text, opts, ledger) {
|
|
2117
3203
|
const input = typeof text === 'string' ? text : String(text ?? '');
|
|
2118
3204
|
const o = opts ?? {};
|
|
2119
|
-
const urls = o.credentialUrls !== false;
|
|
2120
|
-
const words = o.credentialWords !== false;
|
|
2121
3205
|
const face = publicFace(o);
|
|
2122
|
-
const
|
|
3206
|
+
const scan = scanOptionsOf(o, face);
|
|
2123
3207
|
const pairBackslashes = o.escapeBackslashes === true && face.mark !== 'dot' && face.mark !== 'space';
|
|
2124
3208
|
const book = ledger ?? (pairBackslashes ? { cps: [], lens: [] } : null);
|
|
2125
3209
|
if (book === null) {
|
|
2126
3210
|
return runDisplaySteps(input, {
|
|
2127
|
-
wash: (s) =>
|
|
3211
|
+
wash: (s) => scanAndMask(s, scan, false).text,
|
|
2128
3212
|
paint: (s) => applyDisplayFace(s, face),
|
|
2129
3213
|
cap: (s) => capDisplay(s, o.max, face.mark),
|
|
2130
3214
|
text: (s) => s,
|
|
@@ -2132,7 +3216,7 @@ function renderUntrusted(text, opts, ledger) {
|
|
|
2132
3216
|
}
|
|
2133
3217
|
const out = runDisplaySteps({ text: input, ids: new Int32Array(input.length).fill(-1) }, {
|
|
2134
3218
|
wash: (t) => {
|
|
2135
|
-
const r =
|
|
3219
|
+
const r = scanAndMask(t.text, scan, true);
|
|
2136
3220
|
return r.text === t.text ? t : { text: r.text, ids: carryIds(t.ids, r.origin) };
|
|
2137
3221
|
},
|
|
2138
3222
|
paint: (t) => faceTracked(t, face, book),
|