@sema-agent/client-core 0.87.0 → 0.89.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -0
- package/README.md +20 -15
- package/dist/adapter/activeRunSelfHeal.js +4 -2
- package/dist/adapter/downstream/eventToSdkMessage.js +17 -2
- package/dist/adapter/downstream/wiringManifestView.d.ts +11 -0
- package/dist/adapter/downstream/wiringManifestView.js +28 -0
- package/dist/controlRouter.d.ts +7 -1
- package/dist/controlRouter.js +43 -1
- package/dist/disableSkillsWire.d.ts +17 -0
- package/dist/disableSkillsWire.js +52 -0
- package/dist/displayUntrusted.d.ts +4 -0
- package/dist/displayUntrusted.js +1742 -591
- package/dist/engineErrorCodes.d.ts +2 -0
- package/dist/engineErrorCodes.js +1 -0
- package/dist/engineNoticeCodes.d.ts +44 -1
- package/dist/engineNoticeCodes.js +211 -39
- package/dist/engineWireFor.d.ts +1 -0
- package/dist/engineWireFor.js +1 -1
- package/dist/gateOutcome.d.ts +4 -0
- package/dist/gateOutcome.js +5 -0
- package/dist/generated/engineFactTables.d.ts +1 -0
- package/dist/generated/engineFactTables.js +3 -0
- package/dist/generated/engineNoticeTables.d.ts +3 -0
- package/dist/generated/engineNoticeTables.js +110 -0
- package/dist/hitl/crashConverged.d.ts +1 -1
- package/dist/hitl/crashConverged.js +6 -1
- package/dist/hitl/planReviewWire.js +30 -10
- package/dist/hitl/toolApprovalWire.d.ts +6 -2
- package/dist/hitl/toolApprovalWire.js +30 -0
- package/dist/hooksWireCaps.js +80 -12
- package/dist/index.d.ts +12 -3
- package/dist/index.js +8 -3
- package/dist/pluginHooksWire.d.ts +46 -4
- package/dist/pluginHooksWire.js +277 -24
- package/dist/promptDelivery.d.ts +20 -0
- package/dist/promptDelivery.js +153 -0
- package/dist/providerRejection.d.ts +6 -0
- package/dist/providerRejection.js +52 -0
- package/dist/registryConflict.d.ts +3 -0
- package/dist/registryConflict.js +44 -0
- package/dist/request/taskRequest.d.ts +3 -0
- package/dist/request/taskRequest.js +64 -8
- package/dist/resumeRefusalCopy.d.ts +7 -0
- package/dist/resumeRefusalCopy.js +45 -1
- package/dist/runTerminal.d.ts +6 -0
- package/dist/runTerminal.js +23 -0
- package/dist/seam.d.ts +4 -0
- package/dist/seatContract.d.ts +9 -1
- package/dist/seatContract.js +10 -0
- package/dist/sessionMemoryErase.d.ts +76 -0
- package/dist/sessionMemoryErase.js +432 -0
- package/dist/subagent/engineSubagentResume.js +2 -1
- package/dist/subagent/engineSubagentSteer.d.ts +8 -1
- package/dist/subagent/engineSubagentSteer.js +26 -3
- package/docs/INTEGRATION-CLIENTS.md +767 -14
- package/package.json +3 -3
package/dist/displayUntrusted.js
CHANGED
|
@@ -26,135 +26,6 @@ function pushEdit(edits, src, start, end, text) {
|
|
|
26
26
|
const UNIT_NONE = 0;
|
|
27
27
|
const UNIT_SEQUENCE = 1;
|
|
28
28
|
const UNIT_SINGLE = 2;
|
|
29
|
-
const NO_UNITS = () => UNIT_NONE;
|
|
30
|
-
function applyEdits(text, edits) {
|
|
31
|
-
if (edits.length === 0)
|
|
32
|
-
return text;
|
|
33
|
-
const parts = [];
|
|
34
|
-
let cursor = 0;
|
|
35
|
-
for (const e of edits) {
|
|
36
|
-
parts.push(text.slice(cursor, e.start), e.text);
|
|
37
|
-
cursor = e.end;
|
|
38
|
-
}
|
|
39
|
-
parts.push(text.slice(cursor));
|
|
40
|
-
return parts.join('');
|
|
41
|
-
}
|
|
42
|
-
class PlainTarget {
|
|
43
|
-
text;
|
|
44
|
-
origin;
|
|
45
|
-
trace;
|
|
46
|
-
constructor(text, trace = false) {
|
|
47
|
-
this.text = text;
|
|
48
|
-
this.trace = trace;
|
|
49
|
-
}
|
|
50
|
-
apply(edits) {
|
|
51
|
-
if (this.trace && edits.length > 0)
|
|
52
|
-
this.origin = editedOrigin(this.origin, this.text.length, edits);
|
|
53
|
-
this.text = applyEdits(this.text, edits);
|
|
54
|
-
}
|
|
55
|
-
unitAt() {
|
|
56
|
-
return UNIT_NONE;
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
const traced = (text, origin) => (origin === undefined ? { text } : { text, origin });
|
|
60
|
-
function editedOrigin(prev, length, edits) {
|
|
61
|
-
let n = length;
|
|
62
|
-
for (const e of edits)
|
|
63
|
-
n += e.text.length - (e.end - e.start);
|
|
64
|
-
const out = new Int32Array(n);
|
|
65
|
-
let w = 0;
|
|
66
|
-
let cursor = 0;
|
|
67
|
-
const copy = (a, b) => {
|
|
68
|
-
for (let k = a; k < b; k++)
|
|
69
|
-
out[w++] = prev === undefined ? k : (prev[k] ?? -1);
|
|
70
|
-
};
|
|
71
|
-
for (const e of edits) {
|
|
72
|
-
copy(cursor, e.start);
|
|
73
|
-
out.fill(-1, w, w + e.text.length);
|
|
74
|
-
w += e.text.length;
|
|
75
|
-
cursor = e.end;
|
|
76
|
-
}
|
|
77
|
-
copy(cursor, length);
|
|
78
|
-
return out;
|
|
79
|
-
}
|
|
80
|
-
function composeOrigin(outer, inner) {
|
|
81
|
-
if (inner === undefined)
|
|
82
|
-
return outer;
|
|
83
|
-
if (outer === undefined)
|
|
84
|
-
return inner;
|
|
85
|
-
const out = new Int32Array(inner.length);
|
|
86
|
-
for (let k = 0; k < inner.length; k++) {
|
|
87
|
-
const s = inner[k] ?? -1;
|
|
88
|
-
out[k] = s < 0 ? -1 : (outer[s] ?? -1);
|
|
89
|
-
}
|
|
90
|
-
return out;
|
|
91
|
-
}
|
|
92
|
-
class TrackedTarget {
|
|
93
|
-
text;
|
|
94
|
-
src;
|
|
95
|
-
spans = [];
|
|
96
|
-
viewLength;
|
|
97
|
-
units;
|
|
98
|
-
constructor(text, viewLength, units) {
|
|
99
|
-
this.text = text;
|
|
100
|
-
this.viewLength = viewLength;
|
|
101
|
-
this.units = units;
|
|
102
|
-
this.src = new Int32Array(text.length);
|
|
103
|
-
for (let k = 0; k < text.length; k++)
|
|
104
|
-
this.src[k] = k;
|
|
105
|
-
}
|
|
106
|
-
unitAt(k) {
|
|
107
|
-
const s = this.src[k] ?? -1;
|
|
108
|
-
return s >= 0 && this.units !== undefined ? (this.units[s] ?? UNIT_NONE) : UNIT_NONE;
|
|
109
|
-
}
|
|
110
|
-
rangeStart(k) {
|
|
111
|
-
const s = this.src[k] ?? 0;
|
|
112
|
-
return s >= 0 ? s : (this.spans[2 * (-1 - s)] ?? 0);
|
|
113
|
-
}
|
|
114
|
-
rangeEnd(k) {
|
|
115
|
-
const s = this.src[k] ?? 0;
|
|
116
|
-
return s >= 0 ? s + 1 : (this.spans[2 * (-1 - s) + 1] ?? 0);
|
|
117
|
-
}
|
|
118
|
-
apply(edits) {
|
|
119
|
-
if (edits.length === 0)
|
|
120
|
-
return;
|
|
121
|
-
let length = this.text.length;
|
|
122
|
-
for (const e of edits)
|
|
123
|
-
length += e.text.length - (e.end - e.start);
|
|
124
|
-
const src = new Int32Array(length);
|
|
125
|
-
const parts = [];
|
|
126
|
-
let cursor = 0;
|
|
127
|
-
let w = 0;
|
|
128
|
-
for (const e of edits) {
|
|
129
|
-
src.set(this.src.subarray(cursor, e.start), w);
|
|
130
|
-
w += e.start - cursor;
|
|
131
|
-
parts.push(this.text.slice(cursor, e.start), e.text);
|
|
132
|
-
let a;
|
|
133
|
-
let b;
|
|
134
|
-
if (e.start === e.end) {
|
|
135
|
-
a = e.start < this.text.length ? this.rangeStart(e.start) : this.viewLength;
|
|
136
|
-
b = a;
|
|
137
|
-
}
|
|
138
|
-
else {
|
|
139
|
-
a = this.rangeStart(e.start);
|
|
140
|
-
b = this.rangeEnd(e.start);
|
|
141
|
-
for (let k = e.start + 1; k < e.end; k++) {
|
|
142
|
-
a = Math.min(a, this.rangeStart(k));
|
|
143
|
-
b = Math.max(b, this.rangeEnd(k));
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
const id = this.spans.length / 2;
|
|
147
|
-
this.spans.push(a, b);
|
|
148
|
-
src.fill(-1 - id, w, w + e.text.length);
|
|
149
|
-
w += e.text.length;
|
|
150
|
-
cursor = e.end;
|
|
151
|
-
}
|
|
152
|
-
src.set(this.src.subarray(cursor), w);
|
|
153
|
-
parts.push(this.text.slice(cursor));
|
|
154
|
-
this.text = parts.join('');
|
|
155
|
-
this.src = src;
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
29
|
const SECRET_LITERAL_RE = /^(sk-|gh[opus]_|github_pat_|xox[baprse]-|xapp-|bearer\s|basic\s|eyJ[A-Za-z0-9_-]{10,}\.|AKIA[0-9A-Z]{12,}|AIza[0-9A-Za-z_-]{20,}|-----BEGIN[A-Z0-9 ]*PRIVATE KEY)/i;
|
|
159
30
|
const SCHEME_TOKEN_RE = /(?:^|[^A-Za-z0-9_-])(?:bearer|basic)[ \t]+\S{8,4096}/i;
|
|
160
31
|
const JWT_RE = /(?:^|[^A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{8,2048}\.[A-Za-z0-9_-]{4,2048}/i;
|
|
@@ -181,6 +52,147 @@ export function looksLikeKnownSecret(s) {
|
|
|
181
52
|
}
|
|
182
53
|
return false;
|
|
183
54
|
}
|
|
55
|
+
const KIND_BY_TEXT = new Map([
|
|
56
|
+
[REDACTED_USERINFO, 'userinfo'],
|
|
57
|
+
[REDACTED_QUERY, 'query'],
|
|
58
|
+
[REDACTED_FRAGMENT, 'fragment'],
|
|
59
|
+
[REDACTED_SECRET, 'secret'],
|
|
60
|
+
]);
|
|
61
|
+
const TEXT_BY_KIND = {
|
|
62
|
+
userinfo: REDACTED_USERINFO,
|
|
63
|
+
query: REDACTED_QUERY,
|
|
64
|
+
fragment: REDACTED_FRAGMENT,
|
|
65
|
+
secret: REDACTED_SECRET,
|
|
66
|
+
};
|
|
67
|
+
const isStructuralKind = (kind) => kind !== 'secret';
|
|
68
|
+
const MARKER_WITH_DOTS = /^[. ]*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ]*)+$/;
|
|
69
|
+
function valueIsWholeMarker(S, start, end, tokenInner = true) {
|
|
70
|
+
const text = S.text;
|
|
71
|
+
let k = start;
|
|
72
|
+
while (k < end && (S.dropped(k) || text[k] === '.' || text[k] === ' '))
|
|
73
|
+
k++;
|
|
74
|
+
let m;
|
|
75
|
+
const retired = text.startsWith(RETIRED_MARKER, k);
|
|
76
|
+
if (retired)
|
|
77
|
+
m = k + RETIRED_MARKER.length;
|
|
78
|
+
else {
|
|
79
|
+
if (!text.startsWith(REDACTION_MARKER_PREFIX, k))
|
|
80
|
+
return false;
|
|
81
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
82
|
+
re.lastIndex = k;
|
|
83
|
+
if (!re.test(text))
|
|
84
|
+
return false;
|
|
85
|
+
m = re.lastIndex;
|
|
86
|
+
}
|
|
87
|
+
let j = m;
|
|
88
|
+
while (j < end && S.dropped(j))
|
|
89
|
+
j++;
|
|
90
|
+
if (text[j] === '@' && text.startsWith(REDACTED_USERINFO, k))
|
|
91
|
+
return false;
|
|
92
|
+
if (retired) {
|
|
93
|
+
while (j < end && (S.dropped(j) || S.painted(j) || text[j] === '.' || text[j] === ' '))
|
|
94
|
+
j++;
|
|
95
|
+
return j >= end || credentialWordStartsExactlyAt(text, j, tokenInner) || credentialPairAt(S, j);
|
|
96
|
+
}
|
|
97
|
+
while (j < end && (S.dropped(j) || S.painted(j)))
|
|
98
|
+
j++;
|
|
99
|
+
return j >= end || !isKeyWordChar(text.charCodeAt(j)) || credentialWordStartsExactlyAt(text, j, tokenInner) || credentialPairAt(S, j);
|
|
100
|
+
}
|
|
101
|
+
const CREDENTIAL_NAME = /(?:(?:access|refresh|id|client|api|x[-_]api|session|auth)[-_]?)?(?:token|key|secret|password|authorization|credential)s?|bearer|basic/iy;
|
|
102
|
+
const LABEL_SEPARATOR = /(?:\\*["'])?[\s\u115F\u1160\u3164\uFFA0\u2800]*[:=\uFF1A\uFF1D]/y;
|
|
103
|
+
const SCHEME_SEPARATOR = /[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]/y;
|
|
104
|
+
function credentialPairAt(S, at) {
|
|
105
|
+
const text = S.text;
|
|
106
|
+
const re = new RegExp(CREDENTIAL_NAME.source, 'iy');
|
|
107
|
+
re.lastIndex = at;
|
|
108
|
+
const m = re.exec(text);
|
|
109
|
+
if (m === null)
|
|
110
|
+
return false;
|
|
111
|
+
let k = at + m[0].length;
|
|
112
|
+
if (isKeyWordChar(text.charCodeAt(k)))
|
|
113
|
+
return false;
|
|
114
|
+
const skippable = (p) => S.dropped(p) || S.painted(p) || text[p] === '.' || text[p] === ' ';
|
|
115
|
+
const skipped = k < text.length && skippable(k);
|
|
116
|
+
while (k < text.length && skippable(k))
|
|
117
|
+
k++;
|
|
118
|
+
if (k >= text.length)
|
|
119
|
+
return true;
|
|
120
|
+
const scheme = /^(?:bearer|basic)$/i.test(m[0]);
|
|
121
|
+
if (!(scheme && skipped)) {
|
|
122
|
+
const sep = new RegExp((scheme ? SCHEME_SEPARATOR : LABEL_SEPARATOR).source, 'y');
|
|
123
|
+
sep.lastIndex = k;
|
|
124
|
+
if (!sep.test(text))
|
|
125
|
+
return false;
|
|
126
|
+
k = sep.lastIndex;
|
|
127
|
+
}
|
|
128
|
+
while (k < text.length && (skippable(k) || isRegexSpace(text.charCodeAt(k)) || text[k] === '"' || text[k] === "'" || text[k] === '\\'))
|
|
129
|
+
k++;
|
|
130
|
+
if (k >= text.length || isBareValueStop(text.charAt(k)))
|
|
131
|
+
return true;
|
|
132
|
+
if (text.startsWith(RETIRED_MARKER, k))
|
|
133
|
+
return true;
|
|
134
|
+
const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
135
|
+
mk.lastIndex = k;
|
|
136
|
+
return mk.test(text);
|
|
137
|
+
}
|
|
138
|
+
const MARKER_LEAD = /^[. ,;:!?']*(?:(?:\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb|\*{3})[. ,;:!?']*)+/;
|
|
139
|
+
function nextPairOrNameAt(v, at) {
|
|
140
|
+
const name = new RegExp(CREDENTIAL_NAME.source, 'iy');
|
|
141
|
+
name.lastIndex = at;
|
|
142
|
+
if (name.test(v)) {
|
|
143
|
+
let k = name.lastIndex;
|
|
144
|
+
while (k < v.length && TRAILING_PUNCTUATION.has(v.charAt(k)))
|
|
145
|
+
k++;
|
|
146
|
+
if (k >= v.length)
|
|
147
|
+
return true;
|
|
148
|
+
}
|
|
149
|
+
return credentialWordStartsExactlyAt(v, at);
|
|
150
|
+
}
|
|
151
|
+
function userinfoIsWholeMarker(part) {
|
|
152
|
+
const v = withoutSeparators(part);
|
|
153
|
+
if (v.replace(/^[. ]+/, '').startsWith(`${REDACTED_USERINFO}@`))
|
|
154
|
+
return true;
|
|
155
|
+
const lead = MARKER_LEAD.exec(v);
|
|
156
|
+
if (lead === null)
|
|
157
|
+
return false;
|
|
158
|
+
return lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length);
|
|
159
|
+
}
|
|
160
|
+
function slotIsWholeMarker(part, soft) {
|
|
161
|
+
if (soft !== undefined && markerPunctSoftGlue(part, soft))
|
|
162
|
+
return false;
|
|
163
|
+
const v = withoutSeparators(part);
|
|
164
|
+
let k = 0;
|
|
165
|
+
while (k < v.length && (v[k] === '.' || v[k] === ' '))
|
|
166
|
+
k++;
|
|
167
|
+
if (v.startsWith(RETIRED_MARKER, k)) {
|
|
168
|
+
const lead = MARKER_LEAD.exec(v);
|
|
169
|
+
return lead !== null && (lead[0].length >= v.length || nextPairOrNameAt(v, lead[0].length));
|
|
170
|
+
}
|
|
171
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
172
|
+
re.lastIndex = k;
|
|
173
|
+
if (!re.test(v))
|
|
174
|
+
return false;
|
|
175
|
+
let j = re.lastIndex;
|
|
176
|
+
while (j < v.length && (v[j] === '.' || v[j] === ' '))
|
|
177
|
+
j++;
|
|
178
|
+
return j >= v.length || !isKeyWordChar(v.charCodeAt(j)) || nextPairOrNameAt(v, j);
|
|
179
|
+
}
|
|
180
|
+
function markerPunctSoftGlue(part, soft) {
|
|
181
|
+
let k = 0;
|
|
182
|
+
while (k < part.length && (part[k] === VIEW_SEPARATOR || part[k] === '.' || part[k] === ' '))
|
|
183
|
+
k++;
|
|
184
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
185
|
+
re.lastIndex = k;
|
|
186
|
+
if (!re.test(part))
|
|
187
|
+
return false;
|
|
188
|
+
let j = re.lastIndex;
|
|
189
|
+
while (j < part.length && part[j] !== VIEW_SEPARATOR && !soft(j) && TRAILING_PUNCTUATION.has(part.charAt(j)))
|
|
190
|
+
j++;
|
|
191
|
+
const s0 = j;
|
|
192
|
+
while (j < part.length && (part[j] === VIEW_SEPARATOR || soft(j)))
|
|
193
|
+
j++;
|
|
194
|
+
return j > s0 && j < part.length && isKeyWordChar(part.charCodeAt(j)) && !nextPairOrNameAt(part, j);
|
|
195
|
+
}
|
|
184
196
|
const TOKEN_STOP = new Set(['"', '<', '>', '`', '|', '\\', '^', '{', '}']);
|
|
185
197
|
const TRAILING_PUNCTUATION = new Set(['.', ',', ';', ':', '!', '?', "'"]);
|
|
186
198
|
const VALUE_BOUNDARY_TAIL = /[?#&=;]$/;
|
|
@@ -188,6 +200,10 @@ const isWhitespaceOrControl = (ch) => {
|
|
|
188
200
|
const c = ch.charCodeAt(0);
|
|
189
201
|
return c <= 0x20 || c === 0x7f;
|
|
190
202
|
};
|
|
203
|
+
const endsUrlFragmentAt = (s, at) => {
|
|
204
|
+
const ch = s.charAt(at);
|
|
205
|
+
return ch !== '' && (isWhitespaceOrControl(ch) || TOKEN_STOP.has(ch));
|
|
206
|
+
};
|
|
191
207
|
const isSchemeChar = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9') || ch === '+' || ch === '.' || ch === '-';
|
|
192
208
|
const isAlpha = (ch) => (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
|
|
193
209
|
const isHexDigit = (ch) => (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') || (ch >= 'A' && ch <= 'F');
|
|
@@ -226,7 +242,8 @@ function schemelessHostEnd(text, from) {
|
|
|
226
242
|
if (d > k + 1)
|
|
227
243
|
k = d;
|
|
228
244
|
}
|
|
229
|
-
|
|
245
|
+
const c = text.charCodeAt(k);
|
|
246
|
+
return endsSchemelessHost(c) || c === 0x27 ? k : -1;
|
|
230
247
|
}
|
|
231
248
|
function valueCoreForJudging(value) {
|
|
232
249
|
if (value.length === 0 || isMarker(value))
|
|
@@ -240,38 +257,49 @@ function isKnownSecretSegment(value) {
|
|
|
240
257
|
const core = valueCoreForJudging(value);
|
|
241
258
|
return core.length > 0 && looksLikeKnownSecret(core);
|
|
242
259
|
}
|
|
243
|
-
function
|
|
260
|
+
function queryCandidates(src, base, query, edits, soft) {
|
|
244
261
|
let pos = base;
|
|
245
262
|
for (const part of query.split('&')) {
|
|
246
263
|
if (part.length > 0) {
|
|
247
264
|
const eq = part.indexOf('=');
|
|
248
265
|
if (eq === -1) {
|
|
249
|
-
|
|
266
|
+
const at = pos;
|
|
267
|
+
if (!slotIsWholeMarker(part, soft && ((i) => soft(at + i))))
|
|
250
268
|
pushEdit(edits, src, pos, pos + part.length, REDACTED_QUERY);
|
|
251
269
|
}
|
|
252
270
|
else {
|
|
253
271
|
const value = part.slice(eq + 1);
|
|
254
|
-
|
|
272
|
+
const at = pos + eq + 1;
|
|
273
|
+
if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
|
|
255
274
|
pushEdit(edits, src, pos + eq + 1, pos + part.length, REDACTED_QUERY);
|
|
256
275
|
}
|
|
257
276
|
}
|
|
258
277
|
pos += part.length + 1;
|
|
259
278
|
}
|
|
260
279
|
}
|
|
261
|
-
function
|
|
280
|
+
function pathCandidates(src, base, path, opaqueSegments, edits, soft) {
|
|
262
281
|
let pos = base;
|
|
263
282
|
for (const segment of path.split('/')) {
|
|
264
283
|
const semi = segment.indexOf(';');
|
|
265
284
|
const head = semi === -1 ? segment : segment.slice(0, semi);
|
|
266
|
-
if (opaqueSegments && isKnownSecretSegment(head))
|
|
267
|
-
|
|
285
|
+
if (opaqueSegments && isKnownSecretSegment(head)) {
|
|
286
|
+
let a = 0;
|
|
287
|
+
let b = head.length;
|
|
288
|
+
while (a < b && !isKeyWordChar(head.charCodeAt(a)))
|
|
289
|
+
a++;
|
|
290
|
+
while (b > a && !isKeyWordChar(head.charCodeAt(b - 1)))
|
|
291
|
+
b--;
|
|
292
|
+
if (b > a)
|
|
293
|
+
pushEdit(edits, src, pos + a, pos + b, REDACTED_SECRET);
|
|
294
|
+
}
|
|
268
295
|
if (semi !== -1) {
|
|
269
296
|
let ppos = pos + semi + 1;
|
|
270
297
|
for (const part of segment.slice(semi + 1).split(';')) {
|
|
271
298
|
const eq = part.indexOf('=');
|
|
272
299
|
if (eq !== -1) {
|
|
273
300
|
const value = part.slice(eq + 1);
|
|
274
|
-
|
|
301
|
+
const at = ppos + eq + 1;
|
|
302
|
+
if (value.length > 0 && !slotIsWholeMarker(value, soft && ((i) => soft(at + i))))
|
|
275
303
|
pushEdit(edits, src, ppos + eq + 1, ppos + part.length, REDACTED_QUERY);
|
|
276
304
|
}
|
|
277
305
|
ppos += part.length + 1;
|
|
@@ -280,7 +308,7 @@ function pathEdits(src, base, path, opaqueSegments, edits) {
|
|
|
280
308
|
pos += segment.length + 1;
|
|
281
309
|
}
|
|
282
310
|
}
|
|
283
|
-
function
|
|
311
|
+
function urlTokenCandidatesAt(src, start, end, edits, soft) {
|
|
284
312
|
const token = src.slice(start, end);
|
|
285
313
|
const schemeEnd = token.indexOf('://') + '://'.length;
|
|
286
314
|
const scheme = token.slice(0, schemeEnd);
|
|
@@ -290,23 +318,23 @@ function urlTokenEditsAt(src, start, end, edits) {
|
|
|
290
318
|
const tailStart = schemeEnd + authority.length;
|
|
291
319
|
const tail = token.slice(tailStart);
|
|
292
320
|
const at = authority.lastIndexOf('@');
|
|
293
|
-
if (at !== -1)
|
|
321
|
+
if (at !== -1 && !userinfoIsWholeMarker(authority.slice(0, at)))
|
|
294
322
|
pushEdit(edits, src, start + schemeEnd, start + schemeEnd + at, REDACTED_USERINFO);
|
|
295
323
|
const hash = tail.indexOf('#');
|
|
296
324
|
const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
|
|
297
325
|
const question = beforeHash.indexOf('?');
|
|
298
326
|
const path = question === -1 ? beforeHash : beforeHash.slice(0, question);
|
|
299
|
-
|
|
327
|
+
pathCandidates(src, start + tailStart, path, !/^file:/i.test(scheme), edits, soft);
|
|
300
328
|
if (question !== -1)
|
|
301
|
-
|
|
329
|
+
queryCandidates(src, start + tailStart + question + 1, beforeHash.slice(question + 1), edits, soft);
|
|
302
330
|
if (hash !== -1) {
|
|
303
331
|
const rawFragment = tail.slice(hash + 1);
|
|
304
|
-
|
|
332
|
+
const at = start + tailStart + hash + 1;
|
|
333
|
+
if (rawFragment.length > 0 && !slotIsWholeMarker(rawFragment, soft && ((i) => soft(at + i))))
|
|
305
334
|
pushEdit(edits, src, start + tailStart + hash + 1, end, REDACTED_FRAGMENT);
|
|
306
335
|
}
|
|
307
336
|
}
|
|
308
|
-
function
|
|
309
|
-
const edits = [];
|
|
337
|
+
function urlCandidates(text, out, regions, unitPos = () => false, soft) {
|
|
310
338
|
let mark = text.indexOf('://');
|
|
311
339
|
while (mark !== -1) {
|
|
312
340
|
let start = mark;
|
|
@@ -319,20 +347,47 @@ function urlTokenEdits(text) {
|
|
|
319
347
|
continue;
|
|
320
348
|
}
|
|
321
349
|
let end = mark + 3;
|
|
350
|
+
let seenPathSep = false;
|
|
351
|
+
let atAhead = -2;
|
|
352
|
+
const colons = new AuthorityColons(text, mark + 3);
|
|
353
|
+
let firstVisible = -1;
|
|
354
|
+
const adjacent = (at) => {
|
|
355
|
+
if (firstVisible === -1) {
|
|
356
|
+
firstVisible = mark + 3;
|
|
357
|
+
while (firstVisible < text.length && unitPos(firstVisible))
|
|
358
|
+
firstVisible++;
|
|
359
|
+
}
|
|
360
|
+
return at <= firstVisible;
|
|
361
|
+
};
|
|
322
362
|
while (end < text.length) {
|
|
323
363
|
const ch = text.charAt(end);
|
|
324
|
-
if (isWhitespaceOrControl(ch)
|
|
364
|
+
if (isWhitespaceOrControl(ch))
|
|
365
|
+
break;
|
|
366
|
+
if (TOKEN_STOP.has(ch)) {
|
|
367
|
+
if (!seenPathSep) {
|
|
368
|
+
if (atAhead === -2)
|
|
369
|
+
atAhead = atInAuthority(text, end);
|
|
370
|
+
if (atAhead > end && (adjacent(end) || colons.colonBefore(end))) {
|
|
371
|
+
end++;
|
|
372
|
+
continue;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
325
375
|
break;
|
|
376
|
+
}
|
|
326
377
|
const c = ch.charCodeAt(0);
|
|
327
378
|
if (ch === VIEW_SEPARATOR || isUnicodeBlank(c)) {
|
|
328
379
|
let p = end;
|
|
329
380
|
while (p < text.length && (text[p] === VIEW_SEPARATOR || isUnicodeBlank(text.charCodeAt(p))))
|
|
330
381
|
p++;
|
|
331
|
-
if (
|
|
382
|
+
if (nextPairBeyondFragmentAt(text, p))
|
|
332
383
|
break;
|
|
333
384
|
end = p;
|
|
334
385
|
continue;
|
|
335
386
|
}
|
|
387
|
+
if ((ch === ',' || ch === ';') && nextPairBeyondFragmentAt(text, end + 1))
|
|
388
|
+
break;
|
|
389
|
+
if (ch === '/' || ch === '?' || ch === '#')
|
|
390
|
+
seenPathSep = true;
|
|
336
391
|
end++;
|
|
337
392
|
}
|
|
338
393
|
while (end > mark + 3 && text.charAt(end - 1) === VIEW_SEPARATOR)
|
|
@@ -342,10 +397,67 @@ function urlTokenEdits(text) {
|
|
|
342
397
|
runStart--;
|
|
343
398
|
if (runStart < end && !VALUE_BOUNDARY_TAIL.test(text.slice(start, runStart)))
|
|
344
399
|
end = runStart;
|
|
345
|
-
|
|
400
|
+
const tokenEdits = [];
|
|
401
|
+
urlTokenCandidatesAt(text, start, end, tokenEdits, soft);
|
|
402
|
+
editsToCandidates(tokenEdits, 'url', out, end);
|
|
403
|
+
const token = text.slice(start, end);
|
|
404
|
+
const schemeEnd = token.indexOf('://') + 3;
|
|
405
|
+
const rest = token.slice(schemeEnd);
|
|
406
|
+
const authorityEnd = rest.search(/[/?#]/);
|
|
407
|
+
const tailStart = schemeEnd + (authorityEnd === -1 ? rest.length : authorityEnd);
|
|
408
|
+
const tail = token.slice(tailStart);
|
|
409
|
+
const hash = tail.indexOf('#');
|
|
410
|
+
const beforeHash = hash === -1 ? tail : tail.slice(0, hash);
|
|
411
|
+
const question = beforeHash.indexOf('?');
|
|
412
|
+
if (question !== -1)
|
|
413
|
+
regions.push([start + tailStart + question + 1, start + tailStart + beforeHash.length, 'query']);
|
|
414
|
+
if (hash !== -1 && hash + 1 < tail.length)
|
|
415
|
+
regions.push([start + tailStart + hash + 1, end, 'fragment']);
|
|
346
416
|
mark = text.indexOf('://', end > mark + 3 ? end : mark + 3);
|
|
347
417
|
}
|
|
348
|
-
|
|
418
|
+
}
|
|
419
|
+
function atInAuthority(text, from) {
|
|
420
|
+
for (let p = from; p < text.length; p++) {
|
|
421
|
+
const ch = text.charAt(p);
|
|
422
|
+
if (isWhitespaceOrControl(ch) || ch === '/' || ch === '?' || ch === '#')
|
|
423
|
+
return -1;
|
|
424
|
+
if (ch === '@')
|
|
425
|
+
return p;
|
|
426
|
+
}
|
|
427
|
+
return -1;
|
|
428
|
+
}
|
|
429
|
+
class AuthorityColons {
|
|
430
|
+
text;
|
|
431
|
+
scanned;
|
|
432
|
+
colons = 0;
|
|
433
|
+
bracketEnd = -1;
|
|
434
|
+
constructor(text, from) {
|
|
435
|
+
this.text = text;
|
|
436
|
+
this.scanned = from;
|
|
437
|
+
if (text.charCodeAt(from) === 0x5b) {
|
|
438
|
+
let k = from + 1;
|
|
439
|
+
while (k < text.length && isIpv6Char(text.charCodeAt(k)))
|
|
440
|
+
k++;
|
|
441
|
+
if (text.charCodeAt(k) === 0x5d)
|
|
442
|
+
this.bracketEnd = k;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
colonBefore(to) {
|
|
446
|
+
for (; this.scanned < to; this.scanned++) {
|
|
447
|
+
const c = this.text.charCodeAt(this.scanned);
|
|
448
|
+
if (c === 0xab) {
|
|
449
|
+
const re = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
450
|
+
re.lastIndex = this.scanned;
|
|
451
|
+
if (re.test(this.text) && re.lastIndex <= to) {
|
|
452
|
+
this.scanned = re.lastIndex - 1;
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
if (c === 0x3a && this.scanned > this.bracketEnd)
|
|
457
|
+
this.colons++;
|
|
458
|
+
}
|
|
459
|
+
return this.colons > 0;
|
|
460
|
+
}
|
|
349
461
|
}
|
|
350
462
|
const MARKER_ANYWHERE = /\u00abredacted(?::[a-z-]{1,16}){0,2}\u00bb/g;
|
|
351
463
|
function markerSpans(text) {
|
|
@@ -357,18 +469,123 @@ function markerSpans(text) {
|
|
|
357
469
|
spans.push(hit.index, hit.index + hit[0].length);
|
|
358
470
|
return spans;
|
|
359
471
|
}
|
|
360
|
-
|
|
361
|
-
|
|
472
|
+
const USERNAME_DELIMITERS = new Set(['"', "'", '`', '(', ')', '[', ']', '{', '}', '<', '>', '|', '\\', '^', ',', ';', '=']);
|
|
473
|
+
const PASSWORD_FORBIDDEN = new Set(['"', '`', '<', '>']);
|
|
474
|
+
const forbiddenCloser = (ch) => (ch === '<' ? '>' : ch === '"' || ch === '`' ? ch : '');
|
|
475
|
+
const isBareUserChar = (c) => (c >= 0x30 && c <= 0x39) || (c >= 0x41 && c <= 0x5a) || (c >= 0x61 && c <= 0x7a) ||
|
|
476
|
+
c === 0x2e || c === 0x5f || c === 0x2d || c === 0x7e || c === 0x25 || c === 0x2b;
|
|
477
|
+
function userinfoCandidates(S, out, words = true) {
|
|
478
|
+
const text = S.text;
|
|
479
|
+
const unitAt = (k) => S.unitAt(k);
|
|
362
480
|
const markers = markerSpans(text);
|
|
363
|
-
|
|
481
|
+
const markerStartAt = new Map();
|
|
482
|
+
const markerStartByEnd = markers.length === 0 ? null : new Map();
|
|
483
|
+
const markerEndAt = markers.length === 0 ? null : new Int32Array(text.length).fill(-1);
|
|
484
|
+
for (let m = 0; m + 1 < markers.length; m += 2) {
|
|
485
|
+
const a = markers[m] ?? 0;
|
|
486
|
+
const b = markers[m + 1] ?? 0;
|
|
487
|
+
markerStartAt.set(a, b);
|
|
488
|
+
markerStartByEnd?.set(b, a);
|
|
489
|
+
if (markerEndAt !== null)
|
|
490
|
+
for (let k = a; k < b; k++)
|
|
491
|
+
markerEndAt[k] = b;
|
|
492
|
+
}
|
|
493
|
+
let markerFrom = 0;
|
|
494
|
+
const inMarker = (k) => (markerEndAt === null ? -1 : (markerEndAt[k] ?? -1));
|
|
495
|
+
let forbiddenNext = null;
|
|
496
|
+
const nextForbidden = (k) => {
|
|
497
|
+
if (forbiddenNext === null) {
|
|
498
|
+
const n = text.length;
|
|
499
|
+
const t = new Int32Array(n + 1).fill(n);
|
|
500
|
+
for (let x = n - 1; x >= 0; x--)
|
|
501
|
+
t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x)) ? x : (t[x + 1] ?? n);
|
|
502
|
+
forbiddenNext = t;
|
|
503
|
+
}
|
|
504
|
+
return forbiddenNext[k] ?? text.length;
|
|
505
|
+
};
|
|
506
|
+
let forbiddenPrev = null;
|
|
507
|
+
const prevForbidden = (k) => {
|
|
508
|
+
if (forbiddenPrev === null) {
|
|
509
|
+
const n = text.length;
|
|
510
|
+
const t = new Int32Array(n + 1).fill(-1);
|
|
511
|
+
for (let x = 1; x <= n; x++)
|
|
512
|
+
t[x] = PASSWORD_FORBIDDEN.has(text.charAt(x - 1)) ? x - 1 : (t[x - 1] ?? -1);
|
|
513
|
+
forbiddenPrev = t;
|
|
514
|
+
}
|
|
515
|
+
return forbiddenPrev[k] ?? -1;
|
|
516
|
+
};
|
|
517
|
+
let bareStopNext = null;
|
|
518
|
+
const nextBareStop = (k) => {
|
|
519
|
+
if (bareStopNext === null) {
|
|
520
|
+
const n = text.length;
|
|
521
|
+
const t = new Int32Array(n + 1).fill(n);
|
|
522
|
+
for (let x = n - 1; x >= 0; x--)
|
|
523
|
+
t[x] = isBareValueStop(text.charAt(x)) ? x : (t[x + 1] ?? n);
|
|
524
|
+
bareStopNext = t;
|
|
525
|
+
}
|
|
526
|
+
return bareStopNext[k] ?? text.length;
|
|
527
|
+
};
|
|
528
|
+
const labelValueReaches = (from, at) => {
|
|
529
|
+
let i = from;
|
|
530
|
+
while (i < at && text.charCodeAt(i) === 0x5c)
|
|
531
|
+
i++;
|
|
532
|
+
const q = text[i];
|
|
533
|
+
if (q === '"' || q === "'") {
|
|
534
|
+
if (i > from) {
|
|
535
|
+
const close = text.indexOf('\\' + q, i + 1);
|
|
536
|
+
return close === -1 || close >= at;
|
|
537
|
+
}
|
|
538
|
+
for (let j = i + 1; j < at; j++) {
|
|
539
|
+
if (text[j] === '\\') {
|
|
540
|
+
j++;
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
if (text[j] === q)
|
|
544
|
+
return false;
|
|
545
|
+
}
|
|
546
|
+
return true;
|
|
547
|
+
}
|
|
548
|
+
return nextBareStop(from) >= at;
|
|
549
|
+
};
|
|
364
550
|
let floor = 0;
|
|
365
|
-
for (let at = text.indexOf('@')
|
|
551
|
+
for (let at = text.indexOf('@'); at !== -1; at = text.indexOf('@', at + 1)) {
|
|
366
552
|
let hostEnd = schemelessHostEnd(text, at + 1);
|
|
367
553
|
const hidden = hostEnd === -1;
|
|
368
554
|
if (hidden)
|
|
369
555
|
hostEnd = hiddenHostEnd(text, at + 1, unitAt);
|
|
370
556
|
if (hostEnd === -1)
|
|
371
557
|
continue;
|
|
558
|
+
let shaped = null;
|
|
559
|
+
const hostShaped = () => {
|
|
560
|
+
if (shaped === null) {
|
|
561
|
+
const c = text.charCodeAt(hostEnd);
|
|
562
|
+
shaped = !(c === 0x3c || c === 0x3e || c === 0x7c || c === 0x5c || c === 0x5e || c === 0x7b || c === 0x7d);
|
|
563
|
+
if (!shaped)
|
|
564
|
+
for (let k = at + 1; k < hostEnd && !shaped; k++) {
|
|
565
|
+
const h = text.charCodeAt(k);
|
|
566
|
+
if (h === 0x2e || h === 0x3a || h === 0x5b)
|
|
567
|
+
shaped = true;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
return shaped;
|
|
571
|
+
};
|
|
572
|
+
const keptDespiteForbidden = (userStart, colon) => {
|
|
573
|
+
if (labelEndsBefore(text, userStart, floor, '\'"`'))
|
|
574
|
+
return 'label';
|
|
575
|
+
if (hidden || userStart >= colon)
|
|
576
|
+
return null;
|
|
577
|
+
if (labelEndsAt(text, colon + 1) || schemeWordEndsAt(text, colon + 1))
|
|
578
|
+
return words && labelValueReaches(colon + 1, at) ? null : 'label';
|
|
579
|
+
for (let k = userStart; k < colon; k++)
|
|
580
|
+
if (!isBareUserChar(text.charCodeAt(k)))
|
|
581
|
+
return null;
|
|
582
|
+
const q = prevForbidden(at);
|
|
583
|
+
const closer = forbiddenCloser(text.charAt(q));
|
|
584
|
+
if (closer !== '' && q < at - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
|
|
585
|
+
return null;
|
|
586
|
+
return hostShaped() ? 'bare' : null;
|
|
587
|
+
};
|
|
588
|
+
const refused = [];
|
|
372
589
|
let runStart = at;
|
|
373
590
|
while (runStart > floor && (!breaksUserinfo(text.charCodeAt(runStart - 1)) || (hidden && unitAt(runStart - 1) !== UNIT_NONE)))
|
|
374
591
|
runStart--;
|
|
@@ -377,67 +594,135 @@ function schemelessUserinfoEdits(text, unitAt = NO_UNITS) {
|
|
|
377
594
|
runStart--;
|
|
378
595
|
}
|
|
379
596
|
let found = -1;
|
|
380
|
-
let
|
|
381
|
-
let
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
597
|
+
let lo = runStart;
|
|
598
|
+
let keyClose = -2;
|
|
599
|
+
let wholeMarkerAt = -1;
|
|
600
|
+
for (let mk = markerFrom; mk + 1 < markers.length && (markers[mk] ?? 0) < at; mk += 2) {
|
|
601
|
+
const a = markers[mk] ?? 0;
|
|
602
|
+
const b = markers[mk + 1] ?? 0;
|
|
603
|
+
markerFrom = mk;
|
|
604
|
+
if (a < runStart)
|
|
605
|
+
continue;
|
|
606
|
+
let p = b;
|
|
607
|
+
while (p < at && (text[p] === VIEW_SEPARATOR || text[p] === '.'))
|
|
608
|
+
p++;
|
|
609
|
+
if ((p >= at || !isKeyWordChar(text.charCodeAt(p))) && labelWashedBefore(text, a, runStart))
|
|
610
|
+
wholeMarkerAt = a;
|
|
611
|
+
}
|
|
612
|
+
let markerTail = at;
|
|
613
|
+
for (;;) {
|
|
614
|
+
while (markerTail > runStart && (text[markerTail - 1] === VIEW_SEPARATOR || text[markerTail - 1] === '.' || text[markerTail - 1] === ' '))
|
|
615
|
+
markerTail--;
|
|
616
|
+
const ms = markerTail > runStart ? (markerStartByEnd?.get(markerTail) ?? -1) : -1;
|
|
617
|
+
if (ms === -1 || ms < runStart)
|
|
618
|
+
break;
|
|
619
|
+
markerTail = ms;
|
|
620
|
+
}
|
|
621
|
+
let sFrom = runStart;
|
|
622
|
+
for (let colon = runStart; colon < at && found === -1; colon++) {
|
|
623
|
+
const me = inMarker(colon);
|
|
624
|
+
if (me !== -1) {
|
|
625
|
+
colon = me - 1;
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
const ch = text.charAt(colon);
|
|
629
|
+
if (ch === '[') {
|
|
630
|
+
let k = colon + 1;
|
|
631
|
+
while (k < at && isIpv6Char(text.charCodeAt(k)))
|
|
632
|
+
k++;
|
|
633
|
+
if (k < at && text.charCodeAt(k) === 0x5d && k > colon + 1) {
|
|
634
|
+
colon = k;
|
|
635
|
+
lo = k + 1;
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
if (USERNAME_DELIMITERS.has(ch)) {
|
|
640
|
+
lo = colon + 1;
|
|
641
|
+
continue;
|
|
387
642
|
}
|
|
643
|
+
if (text.charCodeAt(colon) !== 0x3a)
|
|
644
|
+
continue;
|
|
388
645
|
if (colon + 1 >= at)
|
|
389
646
|
break;
|
|
390
|
-
|
|
391
|
-
|
|
647
|
+
const leftOfColon = colon > 0 ? text.charAt(colon - 1) : '';
|
|
648
|
+
if ((leftOfColon === "'" || leftOfColon === '"' || leftOfColon === '`') && colon > 1 && isKeyWordChar(text.charCodeAt(colon - 2))) {
|
|
649
|
+
lo = colon + 1;
|
|
650
|
+
keyClose = colon;
|
|
392
651
|
continue;
|
|
393
652
|
}
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
s = colon + 1;
|
|
653
|
+
const forbidden = nextForbidden(colon + 1) < at;
|
|
654
|
+
if (!forbidden && text[colon + 1] === "'" && text[hostEnd] === "'") {
|
|
655
|
+
lo = colon + 1;
|
|
398
656
|
continue;
|
|
399
657
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
658
|
+
if (wholeMarkerAt > colon && colon + 1 < markerTail) {
|
|
659
|
+
lo = colon + 1;
|
|
660
|
+
continue;
|
|
661
|
+
}
|
|
662
|
+
if (wholeMarkerAt === colon + 1 && markerTail === colon + 1 && text.startsWith(REDACTED_USERINFO, colon + 1)) {
|
|
663
|
+
lo = colon + 1;
|
|
664
|
+
continue;
|
|
665
|
+
}
|
|
666
|
+
let userStart = -1;
|
|
667
|
+
if (lo === colon)
|
|
668
|
+
userStart = colon;
|
|
669
|
+
else {
|
|
670
|
+
for (let s = Math.max(lo, sFrom); s < colon; s++) {
|
|
671
|
+
if (markerStartAt.has(s)) {
|
|
672
|
+
userStart = s;
|
|
673
|
+
break;
|
|
674
|
+
}
|
|
675
|
+
const prev = s === 0 ? -1 : text.charCodeAt(s - 1);
|
|
676
|
+
const midPair = prev >= 0xd800 && prev <= 0xdbff && (text.charCodeAt(s) & 0xfc00) === 0xdc00;
|
|
677
|
+
if ((prev === -1 || !blocksUserinfoStart(prev) || s === keyClose + 1) && !midPair) {
|
|
678
|
+
userStart = s;
|
|
679
|
+
break;
|
|
680
|
+
}
|
|
681
|
+
const inner = inMarker(s);
|
|
682
|
+
if (inner !== -1)
|
|
683
|
+
s = inner - 1;
|
|
684
|
+
}
|
|
685
|
+
if (userStart === -1)
|
|
686
|
+
sFrom = colon;
|
|
687
|
+
}
|
|
688
|
+
const kept = userStart === -1 || !forbidden ? null : keptDespiteForbidden(userStart, colon);
|
|
689
|
+
if (userStart === -1 || (forbidden && kept === null)) {
|
|
690
|
+
if (forbidden) {
|
|
691
|
+
lo = colon + 1;
|
|
692
|
+
if (userStart !== -1)
|
|
693
|
+
refused.push(colon, userStart);
|
|
694
|
+
}
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
found = userStart;
|
|
698
|
+
if (kept === 'bare') {
|
|
699
|
+
for (let r = 0; r + 1 < refused.length; r += 2) {
|
|
700
|
+
if (nextForbidden((refused[r] ?? 0) + 1) >= userStart) {
|
|
701
|
+
found = refused[r + 1] ?? userStart;
|
|
702
|
+
break;
|
|
703
|
+
}
|
|
704
|
+
}
|
|
405
705
|
}
|
|
406
|
-
s++;
|
|
407
706
|
}
|
|
408
707
|
if (found === -1)
|
|
409
708
|
continue;
|
|
410
|
-
|
|
411
|
-
continue;
|
|
412
|
-
pushEdit(edits, text, found, at, REDACTED_USERINFO);
|
|
709
|
+
out.push({ a: found, b: at, kind: 'userinfo', cls: hidden ? 'userinfo-hidden' : 'structural', te: hostEnd });
|
|
413
710
|
floor = hostEnd;
|
|
414
711
|
}
|
|
415
|
-
return edits;
|
|
416
712
|
}
|
|
417
|
-
function
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
let
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
}
|
|
430
|
-
function credentialWordEndsFrom(seg, from) {
|
|
431
|
-
for (const word of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
|
|
432
|
-
const re = new RegExp(word.source, 'gi');
|
|
433
|
-
for (let hit = re.exec(seg); hit !== null; hit = re.exec(seg)) {
|
|
434
|
-
if (hit.index + hit[0].length >= from)
|
|
435
|
-
return true;
|
|
436
|
-
if (hit[0].length === 0)
|
|
437
|
-
re.lastIndex++;
|
|
438
|
-
}
|
|
439
|
-
}
|
|
440
|
-
return false;
|
|
713
|
+
function labelWashedBefore(text, a, floor) {
|
|
714
|
+
return labelEndsBefore(text, a, floor, "'");
|
|
715
|
+
}
|
|
716
|
+
function labelEndsBefore(text, a, floor, quotes) {
|
|
717
|
+
let e = a;
|
|
718
|
+
while (e > floor && text[e - 1] === VIEW_SEPARATOR)
|
|
719
|
+
e--;
|
|
720
|
+
if (e > floor && quotes.includes(text[e - 1] ?? '\u0000'))
|
|
721
|
+
e--;
|
|
722
|
+
const w = text.slice(Math.max(floor, e - 96), e);
|
|
723
|
+
if (w === '')
|
|
724
|
+
return false;
|
|
725
|
+
return new RegExp(`(?:${SECRET_LABELLED_WORD.source})$`, 'i').test(w) || new RegExp(`(?:${SECRET_SCHEME_WORD.source})$`, 'i').test(w);
|
|
441
726
|
}
|
|
442
727
|
function unitRunEnd(text, from, unitAt) {
|
|
443
728
|
let k = from;
|
|
@@ -459,16 +744,6 @@ function hiddenHostEnd(text, from, unitAt) {
|
|
|
459
744
|
const c = text.charCodeAt(k);
|
|
460
745
|
return endsSchemelessHost(c) || c === 0x40 ? k : -1;
|
|
461
746
|
}
|
|
462
|
-
const MAX_URL_WASH_PASSES = 4;
|
|
463
|
-
function washUrlNets(target) {
|
|
464
|
-
for (let pass = 0; pass < MAX_URL_WASH_PASSES; pass++) {
|
|
465
|
-
target.apply(urlTokenEdits(target.text));
|
|
466
|
-
const handedOff = schemelessUserinfoEdits(target.text, (k) => target.unitAt(k));
|
|
467
|
-
target.apply(handedOff);
|
|
468
|
-
if (handedOff.length === 0)
|
|
469
|
-
return;
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
747
|
const NAME_BLANK_RANGES = [
|
|
473
748
|
[0x00a0, 0x00a0],
|
|
474
749
|
[0x1680, 0x1680],
|
|
@@ -484,7 +759,8 @@ const NAME_BLANK_RANGES = [
|
|
|
484
759
|
const isNameBlank = (cp) => NAME_BLANK_RANGES.some(([a, b]) => cp >= a && cp <= b);
|
|
485
760
|
const isFoldedBlank = (c) => isNameBlank(c) && isRegexSpace(c);
|
|
486
761
|
const isUnicodeBlank = (c) => (c > 0x7f && isRegexSpace(c) && c !== 0xfeff) || isNameBlank(c);
|
|
487
|
-
const SECRET_SCHEME_WORD = /(?:\b|(?<=\\[A-Za-z"']))(bearer|basic)[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]+/gi;
|
|
762
|
+
const SECRET_SCHEME_WORD = /(?:\b|(?<=\\[A-Za-z"']))(bearer|basic|token)[\s:=\uFF1A\uFF1D\u115F\u1160\u3164\uFFA0\u2800]+/gi;
|
|
763
|
+
const schemeWordIsInner = (word, tokenInner) => tokenInner || word.toLowerCase() !== 'token';
|
|
488
764
|
const SECRET_LABELLED_WORD = /(?:\b|(?<=_)|(?<=\\[A-Za-z"']))((?:access[-_ ]?|refresh[-_ ]?|id[-_ ]?|client[-_ ]?|api[-_ ]?|x[-_]api[-_ ]?|session[-_ ]?|auth[-_ ]?)?(?:token|key|secret|password|authorization|credential)s?)(?:\\*["'])?[\s\u115F\u1160\u3164\uFFA0\u2800]*[:=\uFF1A\uFF1D][\s\u115F\u1160\u3164\uFFA0\u2800]*/gi;
|
|
489
765
|
const DIAGNOSTIC_VALUE_WORDS = new Set([
|
|
490
766
|
'missing', 'required', 'invalid', 'expired', 'unauthorized', 'unauthenticated', 'authentication',
|
|
@@ -502,6 +778,37 @@ const SCHEME_FOLLOWER_WORDS = new Set([
|
|
|
502
778
|
'setup', 'plan', 'tier', 'info', 'information', 'support', 'level', 'string', 'config', 'configuration', 'settings',
|
|
503
779
|
'version', 'check', 'validation', 'error', 'errors', 'failed', 'failure', 'credentials', 'user', 'username',
|
|
504
780
|
]);
|
|
781
|
+
const TOKEN_FOLLOWER_WORDS = new Set([
|
|
782
|
+
...SCHEME_FOLLOWER_WORDS,
|
|
783
|
+
'count', 'counts', 'counting', 'counter', 'counters', 'limit', 'limits', 'budget', 'budgets', 'window', 'windows', 'estimate', 'estimates',
|
|
784
|
+
'estimation', 'estimator', 'length', 'size', 'cost', 'costs', 'price', 'pricing', 'rate', 'rates', 'quota', 'quotas', 'cap', 'caps', 'ceiling',
|
|
785
|
+
'threshold', 'total', 'totals', 'accounting', 'allocation', 'consumption', 'overflow', 'savings', 'reduction', 'spend', 'metrics', 'stats',
|
|
786
|
+
'boundary', 'boundaries', 'stream', 'streaming', 'position', 'positions', 'index', 'offset', 'probability', 'probabilities', 'logprob',
|
|
787
|
+
'logprobs', 'sequence', 'encoding', 'tokenizer', 'vocabulary', 'id', 'ids', 'maximum', 'minimum', 'max', 'min', 'ratio', 'table', 'line',
|
|
788
|
+
'lines', 'unit', 'units', 'indices',
|
|
789
|
+
'refresh', 'refreshed', 'refreshes', 'refreshing', 'exchange', 'exchanged', 'endpoint', 'endpoints', 'request', 'requests', 'response',
|
|
790
|
+
'responses', 'store', 'stored', 'storage', 'cache', 'cached', 'caching', 'file', 'files', 'path', 'dir', 'directory', 'source', 'provider',
|
|
791
|
+
'providers', 'supplier', 'helper', 'command', 'env', 'environment', 'variable', 'lifetime', 'ttl', 'expiry', 'expiration', 'expires', 'expiring',
|
|
792
|
+
'rotation', 'rotated', 'revocation', 'revoke', 'introspection', 'verification', 'verified', 'generation', 'generated', 'issuer', 'issued',
|
|
793
|
+
'audience', 'claims', 'scope', 'scopes', 'permission', 'permissions', 'owner', 'name', 'label', 'hash', 'signature', 'payload', 'body',
|
|
794
|
+
'cookie', 'kind', 'types', 'data', 'metadata', 'envelope', 'binding', 'bound', 'renewal', 'minted', 'acquired', 'fetch', 'fetched', 'parsing',
|
|
795
|
+
'parse', 'parsed', 'persisted', 'url', 'urls', 'client', 'argument', 'flag', 'option', 'field', 'governance', 'target',
|
|
796
|
+
'has', 'have', 'had', 'will', 'would', 'could', 'may', 'might', 'does', 'did', 'do', 'doesn', 'isn', 'wasn', 'hasn', 'didn', 'won', 'couldn',
|
|
797
|
+
'wouldn', 'shouldn', 'aren', 'weren', 'haven', 'been', 'being', 'needs', 'need', 'appears', 'seems', 'looks', 'lives', 'stays', 'remains',
|
|
798
|
+
'starts', 'stopped', 'closed', 'arrived', 'surfaced', 'vanished', 'belongs', 'carries', 'contains', 'lacks', 'matches', 'match', 'changed',
|
|
799
|
+
'created', 'deleted', 'removed', 'updated', 'exceeded', 'exceeds', 'exhausted', 'reached', 'returned', 'returns', 'received', 'provided',
|
|
800
|
+
'supplied', 'found', 'used', 'using', 'saved', 'loaded', 'accepted', 'validated', 'passed', 'sent', 'fails', 'works', 'already', 'still',
|
|
801
|
+
'also', 'only', 'never', 'now', 'here', 'there', 'then', 'than', 'so', 'but', 'as', 'if', 'when', 'while', 'until', 'before', 'after',
|
|
802
|
+
'since', 'because', 'via', 'into', 'over', 'under', 'within', 'without', 'per', 'each', 'every', 'any', 'all', 'its', 'which', 'whose',
|
|
803
|
+
'please', 'run', 'try', 'retry', 'see', 'use', 'visit', 'reauthenticate', 'reconnect',
|
|
804
|
+
]);
|
|
805
|
+
function schemeFollowerReleased(scheme, value, afterPathSlash) {
|
|
806
|
+
const word = value.replace(/[.,;:!?)]+$/, '').toLowerCase();
|
|
807
|
+
const isToken = scheme.toLowerCase() === 'token';
|
|
808
|
+
if (isToken && afterPathSlash && /^\d{3}$/.test(word))
|
|
809
|
+
return true;
|
|
810
|
+
return (isToken ? TOKEN_FOLLOWER_WORDS : SCHEME_FOLLOWER_WORDS).has(word);
|
|
811
|
+
}
|
|
505
812
|
const AUTH_SCHEME_WORDS = new Set([
|
|
506
813
|
'bearer', 'basic', 'digest', 'token', 'bot', 'ssws', 'api-key', 'apikey', 'key', 'hmac', 'oauth', 'oauth2', 'jwt',
|
|
507
814
|
'negotiate', 'ntlm', 'hawk', 'mac', 'dsn', 'splunk', 'sapisidhash', 'googlelogin', 'aws', 'client-id', 'pat',
|
|
@@ -511,9 +818,11 @@ const BARE_VALUE_STOP_CHARS = new Set([',', ';', '"', "'", '\\', '(', ')', '[',
|
|
|
511
818
|
const isBareValueStop = (ch) => ch <= ' ' || BARE_VALUE_STOP_CHARS.has(ch);
|
|
512
819
|
class InnerWordIndex {
|
|
513
820
|
text;
|
|
821
|
+
tokenInner;
|
|
514
822
|
probes;
|
|
515
|
-
constructor(text) {
|
|
823
|
+
constructor(text, tokenInner = true) {
|
|
516
824
|
this.text = text;
|
|
825
|
+
this.tokenInner = tokenInner;
|
|
517
826
|
this.probes = [
|
|
518
827
|
{ re: new RegExp(SECRET_LABELLED_WORD.source, SECRET_LABELLED_WORD.flags), scheme: false, from: -1, hit: null },
|
|
519
828
|
{ re: new RegExp(SECRET_SCHEME_WORD.source, SECRET_SCHEME_WORD.flags), scheme: true, from: -1, hit: null },
|
|
@@ -525,7 +834,11 @@ class InnerWordIndex {
|
|
|
525
834
|
const reusable = p.from !== -1 && q >= p.from && (p.hit === null || q <= p.hit.start);
|
|
526
835
|
if (!reusable) {
|
|
527
836
|
p.re.lastIndex = q;
|
|
528
|
-
|
|
837
|
+
let m = p.re.exec(this.text);
|
|
838
|
+
while (m !== null && p.scheme && !schemeWordIsInner(m[1] ?? '', this.tokenInner)) {
|
|
839
|
+
p.re.lastIndex = m.index + 1;
|
|
840
|
+
m = p.re.exec(this.text);
|
|
841
|
+
}
|
|
529
842
|
p.from = q;
|
|
530
843
|
if (m === null)
|
|
531
844
|
p.hit = null;
|
|
@@ -610,7 +923,7 @@ function scanSecretValue(text, from, idx) {
|
|
|
610
923
|
let p = j;
|
|
611
924
|
while (p < text.length && text[p] === VIEW_SEPARATOR)
|
|
612
925
|
p++;
|
|
613
|
-
if (credentialWordStartsNear(text, p))
|
|
926
|
+
if (credentialWordStartsNear(text, p, idx?.tokenInner ?? true))
|
|
614
927
|
break;
|
|
615
928
|
j = p;
|
|
616
929
|
continue;
|
|
@@ -660,6 +973,22 @@ function isTokenCountLabel(text, at, label) {
|
|
|
660
973
|
const lastSep = Math.max(head.lastIndexOf('_'), head.lastIndexOf('-'));
|
|
661
974
|
return TOKEN_COUNT_HEAD_WORDS.has(head.slice(lastSep + 1));
|
|
662
975
|
}
|
|
976
|
+
function isParserDiagnosticToken(text, at, label) {
|
|
977
|
+
if (label.toLowerCase() !== 'token' || at < 1)
|
|
978
|
+
return false;
|
|
979
|
+
if (text.charAt(at - 1) !== ' ')
|
|
980
|
+
return false;
|
|
981
|
+
const head = 'unexpected';
|
|
982
|
+
const start = at - 1 - head.length;
|
|
983
|
+
if (start < 0 || text.slice(start, at - 1).toLowerCase() !== head)
|
|
984
|
+
return false;
|
|
985
|
+
if (start === 0)
|
|
986
|
+
return true;
|
|
987
|
+
const before = text.charAt(start - 1);
|
|
988
|
+
if (!/[A-Za-z0-9_]/.test(before))
|
|
989
|
+
return true;
|
|
990
|
+
return start >= 2 && text.charAt(start - 2) === '\\' && /[A-Za-z"']/.test(before);
|
|
991
|
+
}
|
|
663
992
|
const BACKTICKED_CREDENTIAL_NAME = /^\x60[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+\x60[.,;:!?)]*$/;
|
|
664
993
|
const CREDENTIAL_NAME_SEGMENTS = new Set([
|
|
665
994
|
'KEY', 'KEYS', 'TOKEN', 'TOKENS', 'SECRET', 'SECRETS', 'PASSWORD', 'PASSWD', 'PWD', 'CREDENTIAL', 'CREDENTIALS',
|
|
@@ -686,29 +1015,58 @@ function labelStartsAt(text, at) {
|
|
|
686
1015
|
re.lastIndex = at;
|
|
687
1016
|
return re.test(text);
|
|
688
1017
|
}
|
|
689
|
-
function credentialWordStartsAt(text, at) {
|
|
1018
|
+
function credentialWordStartsAt(text, at, tokenInner = true) {
|
|
690
1019
|
if (at >= text.length)
|
|
691
1020
|
return false;
|
|
692
|
-
if (credentialWordStartsExactlyAt(text, at))
|
|
1021
|
+
if (credentialWordStartsExactlyAt(text, at, tokenInner))
|
|
693
1022
|
return true;
|
|
694
1023
|
let k = at;
|
|
695
1024
|
while (k < text.length && text.charCodeAt(k) === 0x5c)
|
|
696
1025
|
k++;
|
|
697
1026
|
if (text[k] === '"' || text[k] === "'")
|
|
698
1027
|
k++;
|
|
699
|
-
return k > at && credentialWordStartsExactlyAt(text, k);
|
|
1028
|
+
return k > at && credentialWordStartsExactlyAt(text, k, tokenInner);
|
|
700
1029
|
}
|
|
701
|
-
function credentialWordStartsExactlyAt(text, at) {
|
|
1030
|
+
function credentialWordStartsExactlyAt(text, at, tokenInner = true) {
|
|
702
1031
|
if (at >= text.length)
|
|
703
1032
|
return false;
|
|
704
1033
|
if (labelStartsAt(text, at))
|
|
705
1034
|
return true;
|
|
706
1035
|
const re = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
707
1036
|
re.lastIndex = at;
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
1037
|
+
const m = re.exec(text);
|
|
1038
|
+
return m !== null && schemeWordIsInner(m[1] ?? '', tokenInner);
|
|
1039
|
+
}
|
|
1040
|
+
function credentialWordStartsNear(text, at, tokenInner = true) {
|
|
1041
|
+
return credentialWordStartsAt(text, at, tokenInner) || credentialWordStartsAt(text, at + 1, tokenInner);
|
|
1042
|
+
}
|
|
1043
|
+
function innerWordMatchAt(text, at, tokenInner) {
|
|
1044
|
+
const lab = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1045
|
+
lab.lastIndex = at;
|
|
1046
|
+
const m = lab.exec(text);
|
|
1047
|
+
if (m !== null)
|
|
1048
|
+
return m;
|
|
1049
|
+
const sch = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
1050
|
+
sch.lastIndex = at;
|
|
1051
|
+
const s = sch.exec(text);
|
|
1052
|
+
return s !== null && schemeWordIsInner(s[1] ?? '', tokenInner) ? s : null;
|
|
1053
|
+
}
|
|
1054
|
+
function nextPairBeyondFragmentAt(text, at) {
|
|
1055
|
+
for (const p of [at, at + 1]) {
|
|
1056
|
+
let k = p;
|
|
1057
|
+
while (k < text.length && text.charCodeAt(k) === 0x5c)
|
|
1058
|
+
k++;
|
|
1059
|
+
if (text[k] === '"' || text[k] === "'")
|
|
1060
|
+
k++;
|
|
1061
|
+
for (const src of [SECRET_LABELLED_WORD, SECRET_SCHEME_WORD]) {
|
|
1062
|
+
const re = new RegExp(src.source, 'iy');
|
|
1063
|
+
re.lastIndex = k;
|
|
1064
|
+
const m = re.exec(text);
|
|
1065
|
+
if (m !== null && endsUrlFragmentAt(m[0], m[0].length - 1))
|
|
1066
|
+
return true;
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
return false;
|
|
712
1070
|
}
|
|
713
1071
|
function percentDecodeForMatching(text) {
|
|
714
1072
|
const chars = [];
|
|
@@ -773,7 +1131,7 @@ function hostTokenIsLabel(afterUserinfo, hostEnd) {
|
|
|
773
1131
|
}
|
|
774
1132
|
return false;
|
|
775
1133
|
}
|
|
776
|
-
function exemptAddressEdits(src, from, to, edits) {
|
|
1134
|
+
function exemptAddressEdits(src, from, to, edits, rawPart = (a, b) => src.slice(a, b)) {
|
|
777
1135
|
const value = src.slice(from, to);
|
|
778
1136
|
const q = value.search(/[?#]/);
|
|
779
1137
|
const prefix = q === -1 ? value : value.slice(0, q);
|
|
@@ -790,11 +1148,48 @@ function exemptAddressEdits(src, from, to, edits) {
|
|
|
790
1148
|
const pathStart = scheme.length + (at === -1 ? cut : at + 1 + (cut - userinfoLen));
|
|
791
1149
|
const pathEnd = q === -1 ? value.length : q;
|
|
792
1150
|
const pathCut = pathCutAfterFirstLabel(value.slice(pathStart, pathEnd));
|
|
793
|
-
if (pathCut !== -1)
|
|
1151
|
+
if (pathCut !== -1 && !onlyQuotesAndMarkers(rawPart(from + pathStart + pathCut, from + pathEnd)))
|
|
794
1152
|
pushEdit(edits, src, from + pathStart + pathCut, from + pathEnd, REDACTED_SECRET);
|
|
795
1153
|
if (q !== -1)
|
|
796
1154
|
pushEdit(edits, src, from + q + 1, to, value.charAt(q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY);
|
|
797
1155
|
}
|
|
1156
|
+
function addressJunkStart(text, from, to) {
|
|
1157
|
+
let authEnd = from;
|
|
1158
|
+
while (authEnd < to && text[authEnd] !== '/' && text[authEnd] !== '?' && text[authEnd] !== '#')
|
|
1159
|
+
authEnd++;
|
|
1160
|
+
const at = text.lastIndexOf('@', authEnd - 1);
|
|
1161
|
+
if (at < from)
|
|
1162
|
+
return -1;
|
|
1163
|
+
let k = at + 1;
|
|
1164
|
+
if (text[k] === '[') {
|
|
1165
|
+
const close = text.indexOf(']', k);
|
|
1166
|
+
if (close === -1 || close >= to)
|
|
1167
|
+
return -1;
|
|
1168
|
+
k = close + 1;
|
|
1169
|
+
}
|
|
1170
|
+
else {
|
|
1171
|
+
while (k < to) {
|
|
1172
|
+
const c = text.charCodeAt(k);
|
|
1173
|
+
if (!(isHostChar(c) || (c >= 0xa1 && c !== 0xab && c !== 0xbb && !isRegexSpace(c))))
|
|
1174
|
+
break;
|
|
1175
|
+
k++;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
if (text.charCodeAt(k) === 0x3a) {
|
|
1179
|
+
let d = k + 1;
|
|
1180
|
+
while (d < to && text.charCodeAt(d) >= 0x30 && text.charCodeAt(d) <= 0x39)
|
|
1181
|
+
d++;
|
|
1182
|
+
if (d > k + 1)
|
|
1183
|
+
k = d;
|
|
1184
|
+
}
|
|
1185
|
+
if (k >= to || text[k] === '/' || text[k] === '?' || text[k] === '#')
|
|
1186
|
+
return -1;
|
|
1187
|
+
return k;
|
|
1188
|
+
}
|
|
1189
|
+
function onlyQuotesAndMarkers(part) {
|
|
1190
|
+
const core = part.replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '').split(RETIRED_MARKER).join('');
|
|
1191
|
+
return /^["'\\]*$/.test(core);
|
|
1192
|
+
}
|
|
798
1193
|
function isAddressShaped(value, raw = value) {
|
|
799
1194
|
return value.startsWith(`${REDACTED_USERINFO}@`) || ADDRESS_SCHEME_RE.test(raw);
|
|
800
1195
|
}
|
|
@@ -866,13 +1261,15 @@ function addressExtentEnd(text, from) {
|
|
|
866
1261
|
}
|
|
867
1262
|
return end;
|
|
868
1263
|
}
|
|
869
|
-
function trailingLabelStart(text, from, end) {
|
|
1264
|
+
function trailingLabelStart(text, from, end, tokenInner = true) {
|
|
870
1265
|
const window = text.slice(from, Math.min(text.length, end + 12));
|
|
871
1266
|
const segLen = end - from;
|
|
872
1267
|
let best = -1;
|
|
873
1268
|
for (const src of [SECRET_SCHEME_WORD, SECRET_LABELLED_WORD]) {
|
|
874
1269
|
const re = new RegExp(src.source, src.flags);
|
|
875
1270
|
for (let hit = re.exec(window); hit !== null && hit.index < segLen; hit = re.exec(window)) {
|
|
1271
|
+
if (src === SECRET_SCHEME_WORD && !schemeWordIsInner(hit[1] ?? '', tokenInner))
|
|
1272
|
+
continue;
|
|
876
1273
|
if (hit.index + hit[0].length >= segLen && (best === -1 || hit.index < best))
|
|
877
1274
|
best = hit.index;
|
|
878
1275
|
}
|
|
@@ -881,37 +1278,476 @@ function trailingLabelStart(text, from, end) {
|
|
|
881
1278
|
}
|
|
882
1279
|
const withoutSeparators = (s) => (s.includes(VIEW_SEPARATOR) ? s.split(VIEW_SEPARATOR).join('') : s);
|
|
883
1280
|
const isHorizontalSpace = (ch) => ch === ' ' || ch === '\t' || ch === '\u00a0' || ch === VIEW_SEPARATOR;
|
|
884
|
-
function
|
|
885
|
-
const
|
|
886
|
-
|
|
887
|
-
|
|
1281
|
+
function rawUserinfoAddressAt(text, at, end, gapAfter, stops) {
|
|
1282
|
+
const k = Math.min(end, stops.user[at] ?? text.length);
|
|
1283
|
+
if (k === at || text[k] !== ':')
|
|
1284
|
+
return false;
|
|
1285
|
+
const p = Math.min(end, stops.password[k + 1] ?? text.length);
|
|
1286
|
+
if (p <= k + 1 || text[p] !== '@')
|
|
1287
|
+
return false;
|
|
1288
|
+
if (gapAfter(p))
|
|
1289
|
+
return false;
|
|
1290
|
+
return schemelessHostEnd(text, p + 1) !== -1;
|
|
1291
|
+
}
|
|
1292
|
+
function userinfoStops(text) {
|
|
1293
|
+
const n = text.length;
|
|
1294
|
+
const user = new Int32Array(n + 1).fill(n);
|
|
1295
|
+
const password = new Int32Array(n + 1).fill(n);
|
|
1296
|
+
const brk = new Int32Array(n + 1).fill(n);
|
|
1297
|
+
for (let k = n - 1; k >= 0; k--) {
|
|
1298
|
+
const c = text.charCodeAt(k);
|
|
1299
|
+
const br = breaksUserinfo(c);
|
|
1300
|
+
user[k] = br || c === 0x3a || USERNAME_DELIMITERS.has(text.charAt(k)) ? k : (user[k + 1] ?? n);
|
|
1301
|
+
brk[k] = br && text[k] !== VIEW_SEPARATOR ? k : (brk[k + 1] ?? n);
|
|
1302
|
+
password[k] = (br && text[k] !== VIEW_SEPARATOR) || PASSWORD_FORBIDDEN.has(text.charAt(k)) ? k : (password[k + 1] ?? n);
|
|
1303
|
+
}
|
|
1304
|
+
return { user, password, brk };
|
|
1305
|
+
}
|
|
1306
|
+
function forbiddenUserinfoAt(text, at, end, gapAfter, stops, unitAt) {
|
|
1307
|
+
const k = Math.min(end, stops.user[at] ?? text.length);
|
|
1308
|
+
if (k >= end || text[k] !== ':')
|
|
1309
|
+
return -1;
|
|
1310
|
+
for (let u = at; u < k; u++)
|
|
1311
|
+
if (unitAt(u) !== UNIT_NONE || text[u] === VIEW_SEPARATOR)
|
|
1312
|
+
return -1;
|
|
1313
|
+
const p = Math.min(end, stops.password[k + 1] ?? text.length);
|
|
1314
|
+
if (p >= end || !PASSWORD_FORBIDDEN.has(text.charAt(p)))
|
|
1315
|
+
return -1;
|
|
1316
|
+
const a = Math.min(end, stops.brk[k + 1] ?? text.length);
|
|
1317
|
+
if (a >= end || text[a] !== '@' || gapAfter(a))
|
|
1318
|
+
return -1;
|
|
1319
|
+
return schemelessHostEnd(text, a + 1) === -1 ? -1 : a;
|
|
1320
|
+
}
|
|
1321
|
+
function isTokenCountAt(text, at) {
|
|
1322
|
+
const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1323
|
+
re.lastIndex = at;
|
|
1324
|
+
const m = re.exec(text);
|
|
1325
|
+
return m !== null && isTokenCountLabel(text, at, m[1] ?? '');
|
|
1326
|
+
}
|
|
1327
|
+
function innerWordHasValue(text, at, idx) {
|
|
1328
|
+
const m = innerWordMatchAt(text, at, idx.tokenInner);
|
|
1329
|
+
return m !== null && scanSecretValue(text, at + m[0].length, idx) !== null;
|
|
1330
|
+
}
|
|
1331
|
+
function quoteSwallowedBefore(S, at) {
|
|
1332
|
+
let k = at - 1;
|
|
1333
|
+
while (k >= 0 && S.dropped(k))
|
|
1334
|
+
k--;
|
|
1335
|
+
const lo = k >= 0 ? S.rawEnd(k) : 0;
|
|
1336
|
+
const hi = at < S.text.length ? S.rawStart(at) : S.raw.length;
|
|
1337
|
+
if (hi <= lo)
|
|
1338
|
+
return '';
|
|
1339
|
+
const gap = S.raw.slice(lo, hi);
|
|
1340
|
+
const dq = gap.indexOf('"');
|
|
1341
|
+
const sq = gap.indexOf("'");
|
|
1342
|
+
if (dq === -1 && sq === -1)
|
|
1343
|
+
return '';
|
|
1344
|
+
return dq === -1 ? "'" : sq === -1 ? '"' : dq < sq ? '"' : "'";
|
|
1345
|
+
}
|
|
1346
|
+
function labelEndsAt(text, end, atLeast = false) {
|
|
1347
|
+
let k = end;
|
|
1348
|
+
while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
|
|
1349
|
+
k--;
|
|
1350
|
+
if (k === 0)
|
|
1351
|
+
return false;
|
|
1352
|
+
const sep = text.charAt(k - 1);
|
|
1353
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1354
|
+
return false;
|
|
1355
|
+
k--;
|
|
1356
|
+
while (k > 0 && isRegexSpace(text.charCodeAt(k - 1)))
|
|
1357
|
+
k--;
|
|
1358
|
+
if (k > 0 && (text[k - 1] === '"' || text[k - 1] === "'"))
|
|
1359
|
+
k--;
|
|
1360
|
+
while (k > 0 && text.charCodeAt(k - 1) === 0x5c)
|
|
1361
|
+
k--;
|
|
1362
|
+
let w = k;
|
|
1363
|
+
while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
|
|
1364
|
+
w--;
|
|
1365
|
+
const re = new RegExp(SECRET_LABELLED_WORD.source, 'iy');
|
|
1366
|
+
for (let s = w; s < k; s++) {
|
|
1367
|
+
re.lastIndex = s;
|
|
1368
|
+
const m = re.exec(text);
|
|
1369
|
+
if (m !== null && (atLeast ? s + m[0].length >= end : s + m[0].length === end) && !isTokenCountLabel(text, s, m[1] ?? ''))
|
|
1370
|
+
return true;
|
|
1371
|
+
}
|
|
1372
|
+
return false;
|
|
1373
|
+
}
|
|
1374
|
+
function schemeWordEndsAt(text, end) {
|
|
1375
|
+
const sep = text.charAt(end - 1);
|
|
1376
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1377
|
+
return false;
|
|
1378
|
+
let w = end - 1;
|
|
1379
|
+
while (w > 0 && isIdentOrDash(text.charAt(w - 1)))
|
|
1380
|
+
w--;
|
|
1381
|
+
const re = new RegExp(SECRET_SCHEME_WORD.source, 'iy');
|
|
1382
|
+
for (let s = w; s < end - 1; s++) {
|
|
1383
|
+
re.lastIndex = s;
|
|
1384
|
+
const m = re.exec(text);
|
|
1385
|
+
if (m !== null && s + m[0].length === end)
|
|
1386
|
+
return true;
|
|
1387
|
+
}
|
|
1388
|
+
return false;
|
|
1389
|
+
}
|
|
1390
|
+
function quotedExtensionEnd(text, close, cache) {
|
|
1391
|
+
const escaped = text.charCodeAt(close) === 0x5c;
|
|
1392
|
+
const q = escaped ? text.charAt(close + 1) : text.charAt(close);
|
|
1393
|
+
if (q !== '"' && q !== "'")
|
|
1394
|
+
return close;
|
|
1395
|
+
const token = escaped ? `\\${q}` : q;
|
|
1396
|
+
const after = (at) => at + token.length;
|
|
1397
|
+
let memo = cache.get(token);
|
|
1398
|
+
if (memo === undefined) {
|
|
1399
|
+
memo = new Map();
|
|
1400
|
+
cache.set(token, memo);
|
|
1401
|
+
}
|
|
1402
|
+
const done = memo;
|
|
1403
|
+
const step = (cur) => {
|
|
1404
|
+
let n = after(cur);
|
|
1405
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1406
|
+
n++;
|
|
1407
|
+
const sepNext = text[n] === ':' || text[n] === '=' || text[n] === '\uFF1A' || text[n] === '\uFF1D';
|
|
1408
|
+
if (sepNext && labelEndsAt(text, n + 1, true) && !labelEndsAt(text, cur)) {
|
|
1409
|
+
const next = text.indexOf(token, after(cur));
|
|
1410
|
+
return next === -1 ? { kind: 'open', inner: false } : { kind: 'next', at: next };
|
|
1411
|
+
}
|
|
1412
|
+
if (labelEndsAt(text, cur)) {
|
|
1413
|
+
const innerClose = text.indexOf(token, after(cur));
|
|
1414
|
+
if (innerClose === -1)
|
|
1415
|
+
return { kind: 'open', inner: true };
|
|
1416
|
+
const next = text.indexOf(token, after(innerClose));
|
|
1417
|
+
return next === -1 ? { kind: 'open', inner: true } : { kind: 'next', at: next };
|
|
1418
|
+
}
|
|
1419
|
+
return { kind: 'end' };
|
|
1420
|
+
};
|
|
1421
|
+
const first = step(close);
|
|
1422
|
+
if (first.kind === 'end')
|
|
1423
|
+
return close;
|
|
1424
|
+
if (first.kind === 'open')
|
|
1425
|
+
return first.inner ? close : text.length;
|
|
1426
|
+
const path = [];
|
|
1427
|
+
let cur = first.at;
|
|
1428
|
+
let result;
|
|
1429
|
+
for (;;) {
|
|
1430
|
+
const hit = done.get(cur);
|
|
1431
|
+
if (hit !== undefined) {
|
|
1432
|
+
result = hit;
|
|
1433
|
+
break;
|
|
1434
|
+
}
|
|
1435
|
+
path.push(cur);
|
|
1436
|
+
const s = step(cur);
|
|
1437
|
+
if (s.kind === 'end') {
|
|
1438
|
+
result = cur;
|
|
1439
|
+
break;
|
|
1440
|
+
}
|
|
1441
|
+
if (s.kind === 'open') {
|
|
1442
|
+
result = text.length;
|
|
1443
|
+
break;
|
|
1444
|
+
}
|
|
1445
|
+
cur = s.at;
|
|
1446
|
+
}
|
|
1447
|
+
for (const p of path)
|
|
1448
|
+
done.set(p, result);
|
|
1449
|
+
return result;
|
|
1450
|
+
}
|
|
1451
|
+
function extensionMaskStart(text, close) {
|
|
1452
|
+
let n = close + (text.charCodeAt(close) === 0x5c ? 2 : 1);
|
|
1453
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1454
|
+
n++;
|
|
1455
|
+
const sep = text[n];
|
|
1456
|
+
if (sep !== ':' && sep !== '=' && sep !== '\uFF1A' && sep !== '\uFF1D')
|
|
1457
|
+
return close;
|
|
1458
|
+
if (!labelEndsAt(text, n + 1, true) || labelEndsAt(text, close))
|
|
1459
|
+
return close;
|
|
1460
|
+
n++;
|
|
1461
|
+
while (n < text.length && (text[n] === ' ' || text[n] === '\t'))
|
|
1462
|
+
n++;
|
|
1463
|
+
return n;
|
|
1464
|
+
}
|
|
1465
|
+
function quotedAddressEnd(S, from, to) {
|
|
1466
|
+
const text = S.text;
|
|
1467
|
+
for (let k = from; k < to; k++) {
|
|
1468
|
+
const c = text.charCodeAt(k);
|
|
1469
|
+
if ((isRegexSpace(c) && c !== 0xfeff) || S.unitAt(k) !== UNIT_NONE)
|
|
1470
|
+
return k;
|
|
1471
|
+
if (k > from && S.spaceGapBefore(k))
|
|
1472
|
+
return k;
|
|
1473
|
+
}
|
|
1474
|
+
return to;
|
|
1475
|
+
}
|
|
1476
|
+
function labelCandidates(S, pass, regions, out) {
|
|
1477
|
+
const text = S.text;
|
|
1478
|
+
const unitAt = (k) => S.unitAt(k);
|
|
1479
|
+
const re = pass === 'label' ? SECRET_LABELLED_WORD : SECRET_SCHEME_WORD;
|
|
1480
|
+
const schemePass = pass !== 'label';
|
|
1481
|
+
const tokenPass = pass === 'token';
|
|
1482
|
+
const idx = new InnerWordIndex(text, tokenPass);
|
|
1483
|
+
const extensionCache = new Map();
|
|
1484
|
+
let stops;
|
|
1485
|
+
const push = (start, end, kindText, te = -1, cls) => {
|
|
1486
|
+
let a = start;
|
|
1487
|
+
let b = end;
|
|
1488
|
+
while (a < b && text[a] === VIEW_SEPARATOR)
|
|
1489
|
+
a++;
|
|
1490
|
+
while (b > a && text[b - 1] === VIEW_SEPARATOR)
|
|
1491
|
+
b--;
|
|
1492
|
+
if (a >= b)
|
|
1493
|
+
return;
|
|
1494
|
+
if (text.slice(a, b) === kindText)
|
|
1495
|
+
return;
|
|
1496
|
+
const kind = KIND_BY_TEXT.get(kindText);
|
|
1497
|
+
if (kind === undefined)
|
|
1498
|
+
return;
|
|
1499
|
+
out.push({ a, b, kind, cls: cls ?? (kindText === REDACTED_SECRET ? 'word' : 'structural'), te });
|
|
1500
|
+
};
|
|
1501
|
+
const pushEdits = (edits, te) => {
|
|
1502
|
+
for (const e of edits) {
|
|
1503
|
+
const part = text.slice(e.start, e.end);
|
|
1504
|
+
if (e.text === REDACTED_USERINFO ? userinfoIsWholeMarker(part) : e.text !== REDACTED_SECRET && slotIsWholeMarker(part))
|
|
1505
|
+
continue;
|
|
1506
|
+
push(e.start, e.end, e.text, e.text === REDACTED_SECRET ? -1 : te);
|
|
1507
|
+
}
|
|
1508
|
+
};
|
|
1509
|
+
let markerCache = -2;
|
|
1510
|
+
const nextMarkerFrom = (p) => {
|
|
1511
|
+
if (markerCache === -1 || markerCache >= p)
|
|
1512
|
+
return markerCache;
|
|
1513
|
+
markerCache = text.indexOf(REDACTION_MARKER_PREFIX, p);
|
|
1514
|
+
return markerCache;
|
|
1515
|
+
};
|
|
1516
|
+
const maskGaps = (r0, end) => {
|
|
1517
|
+
let p = r0;
|
|
1518
|
+
while (p < end) {
|
|
1519
|
+
while (p < end && (isRegexSpace(text.charCodeAt(p)) || unitAt(p) !== UNIT_NONE))
|
|
1520
|
+
p++;
|
|
1521
|
+
if (p >= end)
|
|
1522
|
+
return;
|
|
1523
|
+
let inner = idx.next(p);
|
|
1524
|
+
while (inner !== null && inner.start < end && isTokenCountAt(text, inner.start))
|
|
1525
|
+
inner = idx.next(inner.start + 1);
|
|
1526
|
+
const hasInner = inner !== null && inner.start < end;
|
|
1527
|
+
let stop = hasInner && inner !== null ? inner.start : end;
|
|
1528
|
+
const head = withoutSeparators(text.slice(p, Math.min(stop, p + 80)));
|
|
1529
|
+
const addressEnd = ADDRESS_SCHEME_RE.test(head) || head.startsWith(`${REDACTED_USERINFO}@`) ? Math.min(stop, quotedAddressEnd(S, p, end)) : -1;
|
|
1530
|
+
if (addressEnd !== -1 && addressEnd < stop) {
|
|
1531
|
+
p = addressEnd;
|
|
1532
|
+
continue;
|
|
1533
|
+
}
|
|
1534
|
+
if (addressEnd === -1) {
|
|
1535
|
+
let marker = null;
|
|
1536
|
+
for (let at = nextMarkerFrom(p); at !== -1 && at < stop && marker === null; at = nextMarkerFrom(at + 1)) {
|
|
1537
|
+
const mk = new RegExp(MARKER_ANYWHERE.source, 'y');
|
|
1538
|
+
mk.lastIndex = at;
|
|
1539
|
+
marker = mk.exec(text);
|
|
1540
|
+
}
|
|
1541
|
+
const markerHit = marker !== null && marker.index < stop;
|
|
1542
|
+
if (markerHit && marker !== null)
|
|
1543
|
+
stop = marker.index;
|
|
1544
|
+
let cut = stop;
|
|
1545
|
+
while (cut > p && (isCutDelimiter(text.charAt(cut - 1)) || isRegexSpace(text.charCodeAt(cut - 1))))
|
|
1546
|
+
cut--;
|
|
1547
|
+
if (cut > p) {
|
|
1548
|
+
const rest = withoutSeparators(text.slice(p, cut));
|
|
1549
|
+
if (rest.trim() !== '' && !valueIsWholeMarker(S, p, cut, idx.tokenInner) && !isDiagnosticValue(rest))
|
|
1550
|
+
push(p, cut, REDACTED_SECRET);
|
|
1551
|
+
}
|
|
1552
|
+
if (markerHit && marker !== null) {
|
|
1553
|
+
p = marker.index + marker[0].length;
|
|
1554
|
+
continue;
|
|
1555
|
+
}
|
|
1556
|
+
}
|
|
1557
|
+
if (!hasInner || inner === null)
|
|
1558
|
+
return;
|
|
1559
|
+
const m = innerWordMatchAt(text, inner.start, idx.tokenInner);
|
|
1560
|
+
if (m === null)
|
|
1561
|
+
return;
|
|
1562
|
+
const after = inner.start + m[0].length;
|
|
1563
|
+
const iv = scanSecretValue(text, after, idx);
|
|
1564
|
+
let next = iv === null ? after : iv.end;
|
|
1565
|
+
if (next < end && (text[next] === '"' || text[next] === "'"))
|
|
1566
|
+
next++;
|
|
1567
|
+
if (next <= p)
|
|
1568
|
+
return;
|
|
1569
|
+
p = next;
|
|
1570
|
+
}
|
|
1571
|
+
};
|
|
1572
|
+
const pushInnerRemainder = (from) => {
|
|
1573
|
+
let open = from;
|
|
1574
|
+
while (text.charCodeAt(open) === 0x5c)
|
|
1575
|
+
open++;
|
|
1576
|
+
let q = text.charAt(open);
|
|
1577
|
+
let at = open + 1;
|
|
1578
|
+
let outerEnd;
|
|
1579
|
+
if (q === '"' || q === "'") {
|
|
1580
|
+
const outer = scanSecretValue(text, from);
|
|
1581
|
+
if (outer === null)
|
|
1582
|
+
return;
|
|
1583
|
+
outerEnd = outer.end;
|
|
1584
|
+
}
|
|
1585
|
+
else {
|
|
1586
|
+
q = quoteSwallowedBefore(S, from);
|
|
1587
|
+
if (q === '')
|
|
1588
|
+
return;
|
|
1589
|
+
at = from;
|
|
1590
|
+
const close = text.indexOf(q, from);
|
|
1591
|
+
outerEnd = close === -1 ? text.length : close;
|
|
1592
|
+
}
|
|
1593
|
+
const m = innerWordMatchAt(text, at, idx.tokenInner);
|
|
1594
|
+
if (m === null)
|
|
1595
|
+
return;
|
|
1596
|
+
const inner = { start: at };
|
|
1597
|
+
const iv = scanSecretValue(text, inner.start + m[0].length, idx);
|
|
1598
|
+
if (iv === null)
|
|
1599
|
+
return;
|
|
1600
|
+
let r0 = iv.end;
|
|
1601
|
+
if (r0 < text.length && (text[r0] === '"' || text[r0] === "'") && text[r0] !== q)
|
|
1602
|
+
r0++;
|
|
1603
|
+
maskGaps(r0, outerEnd);
|
|
1604
|
+
};
|
|
1605
|
+
const regionKind = regions.length === 0 ? null : new Uint8Array(text.length);
|
|
1606
|
+
if (regionKind !== null)
|
|
1607
|
+
for (const r of regions)
|
|
1608
|
+
for (let k = r[0]; k < r[1] && k < text.length; k++)
|
|
1609
|
+
regionKind[k] = r[2] === 'query' ? 1 : 2;
|
|
1610
|
+
const inQuery = (k) => regionKind !== null && k < text.length && regionKind[k] === 1;
|
|
1611
|
+
const forbiddenTailAt = (p) => {
|
|
1612
|
+
if (!PASSWORD_FORBIDDEN.has(text.charAt(p)))
|
|
1613
|
+
return -1;
|
|
1614
|
+
const st = (stops ??= userinfoStops(text));
|
|
1615
|
+
const a = st.brk[p] ?? text.length;
|
|
1616
|
+
if (a >= text.length || text[a] !== '@' || S.gapAfter(a))
|
|
1617
|
+
return -1;
|
|
1618
|
+
const inner = idx.next(p);
|
|
1619
|
+
if (inner !== null && inner.start < a)
|
|
1620
|
+
return -1;
|
|
1621
|
+
const hostEnd = schemelessHostEnd(text, a + 1);
|
|
1622
|
+
if (hostEnd === -1)
|
|
1623
|
+
return -1;
|
|
1624
|
+
let q = -1;
|
|
1625
|
+
for (let k = a - 1; k >= p && q === -1; k--)
|
|
1626
|
+
if (PASSWORD_FORBIDDEN.has(text.charAt(k)))
|
|
1627
|
+
q = k;
|
|
1628
|
+
const closer = forbiddenCloser(text.charAt(q));
|
|
1629
|
+
if (closer !== '' && q < a - 1 && text.charCodeAt(q - 1) !== 0x5c && text.charAt(hostEnd) === closer)
|
|
1630
|
+
return -1;
|
|
1631
|
+
return a;
|
|
1632
|
+
};
|
|
888
1633
|
re.lastIndex = 0;
|
|
889
1634
|
for (let hit = re.exec(text); hit !== null; hit = re.exec(text)) {
|
|
890
1635
|
const label = hit[1] ?? '';
|
|
1636
|
+
if (schemePass && (label.toLowerCase() === 'token') !== tokenPass)
|
|
1637
|
+
continue;
|
|
891
1638
|
if (isTokenCountLabel(text, hit.index, label))
|
|
892
1639
|
continue;
|
|
1640
|
+
if (schemePass && isParserDiagnosticToken(text, hit.index, label))
|
|
1641
|
+
continue;
|
|
1642
|
+
if (schemePass && labelStartsAt(text, hit.index))
|
|
1643
|
+
continue;
|
|
893
1644
|
const from = hit.index + hit[0].length;
|
|
894
|
-
if (credentialWordStartsAt(text, from)) {
|
|
1645
|
+
if (credentialWordStartsAt(text, from, idx.tokenInner)) {
|
|
1646
|
+
pushInnerRemainder(from);
|
|
895
1647
|
re.lastIndex = from;
|
|
896
1648
|
continue;
|
|
897
1649
|
}
|
|
898
|
-
|
|
899
|
-
if (
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1650
|
+
let v0s = scanSecretValue(text, from, idx);
|
|
1651
|
+
if (v0s === null) {
|
|
1652
|
+
let k = from;
|
|
1653
|
+
while (k < text.length && text[k] === VIEW_SEPARATOR)
|
|
1654
|
+
k++;
|
|
1655
|
+
const a = inQuery(k) ? -1 : forbiddenTailAt(k);
|
|
1656
|
+
if (a === -1)
|
|
1657
|
+
continue;
|
|
1658
|
+
v0s = { start: k, end: a };
|
|
1659
|
+
}
|
|
1660
|
+
const v0 = { start: v0s.start, end: v0s.end };
|
|
1661
|
+
let open = from;
|
|
1662
|
+
while (text.charCodeAt(open) === 0x5c)
|
|
1663
|
+
open++;
|
|
1664
|
+
let quoted = text[open] === '"' || text[open] === "'";
|
|
1665
|
+
if (!quoted) {
|
|
1666
|
+
const q = quoteSwallowedBefore(S, v0.start);
|
|
1667
|
+
if (q !== '') {
|
|
1668
|
+
quoted = true;
|
|
1669
|
+
let j = v0.start;
|
|
1670
|
+
while (j < text.length && text[j] !== q)
|
|
1671
|
+
j += text[j] === '\\' ? 2 : 1;
|
|
1672
|
+
v0.end = Math.min(j, text.length);
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
let keptSepEnd = -1;
|
|
1676
|
+
if (quoted) {
|
|
1677
|
+
let close = v0.end;
|
|
1678
|
+
while (close < text.length && text[close] === VIEW_SEPARATOR)
|
|
1679
|
+
close++;
|
|
1680
|
+
const ext = close < text.length ? quotedExtensionEnd(text, close, extensionCache) : close;
|
|
1681
|
+
if (ext > close) {
|
|
1682
|
+
const from = extensionMaskStart(text, close);
|
|
1683
|
+
if (from > close)
|
|
1684
|
+
keptSepEnd = from;
|
|
1685
|
+
push(from, ext, REDACTED_SECRET);
|
|
1686
|
+
}
|
|
1687
|
+
}
|
|
1688
|
+
const v = { start: v0.start, end: v0.end };
|
|
1689
|
+
if (quoted) {
|
|
1690
|
+
while (v.start < v.end && unitAt(v.start) !== UNIT_NONE)
|
|
1691
|
+
v.start++;
|
|
1692
|
+
while (v.end > v.start && unitAt(v.end - 1) !== UNIT_NONE)
|
|
1693
|
+
v.end--;
|
|
1694
|
+
if (v.start >= v.end) {
|
|
1695
|
+
re.lastIndex = from;
|
|
1696
|
+
continue;
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
if (valueIsWholeMarker(S, v.start, v.end, idx.tokenInner)) {
|
|
1700
|
+
re.lastIndex = quoted ? from : v.end;
|
|
903
1701
|
continue;
|
|
904
1702
|
}
|
|
905
|
-
if (
|
|
906
|
-
|
|
907
|
-
|
|
1703
|
+
if (!quoted && inQuery(v.start)) {
|
|
1704
|
+
for (let k = v.start; k < v.end; k++) {
|
|
1705
|
+
const ch = text[k];
|
|
1706
|
+
if (ch === '&' || ch === '#') {
|
|
1707
|
+
v.end = k;
|
|
1708
|
+
break;
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
if (v.end <= v.start) {
|
|
1712
|
+
re.lastIndex = v0.end;
|
|
908
1713
|
continue;
|
|
909
1714
|
}
|
|
910
1715
|
}
|
|
1716
|
+
let forbiddenAt = -1;
|
|
1717
|
+
if (!quoted && !inQuery(v.start)) {
|
|
1718
|
+
forbiddenAt = forbiddenUserinfoAt(text, v.start, text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
|
|
1719
|
+
if (forbiddenAt !== -1)
|
|
1720
|
+
v.end = Math.max(v.end, addressExtentEnd(text, forbiddenAt));
|
|
1721
|
+
}
|
|
1722
|
+
else if (quoted) {
|
|
1723
|
+
forbiddenAt = forbiddenUserinfoAt(text, v.start, v.end, (k) => S.gapAfter(k), stops ??= userinfoStops(text), unitAt);
|
|
1724
|
+
}
|
|
1725
|
+
if (!quoted && forbiddenAt === -1 && !inQuery(v.start)) {
|
|
1726
|
+
const a = forbiddenTailAt(v.end);
|
|
1727
|
+
if (a !== -1)
|
|
1728
|
+
v.end = a;
|
|
1729
|
+
}
|
|
1730
|
+
if (!quoted) {
|
|
1731
|
+
const trailing = trailingLabelStart(text, v.start, v.end, idx.tokenInner);
|
|
1732
|
+
if (trailing > 0 && innerWordHasValue(text, v.start + trailing, idx)) {
|
|
1733
|
+
let e = v.start + trailing;
|
|
1734
|
+
while (e > v.start && (isCutDelimiter(text.charAt(e - 1)) || text[e - 1] === VIEW_SEPARATOR))
|
|
1735
|
+
e--;
|
|
1736
|
+
if (e > v.start)
|
|
1737
|
+
v.end = e;
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
if (pemBeginAt(S, v.start)) {
|
|
1741
|
+
re.lastIndex = v.end;
|
|
1742
|
+
continue;
|
|
1743
|
+
}
|
|
911
1744
|
const rawValue = text.slice(v.start, v.end);
|
|
912
1745
|
const value = withoutSeparators(rawValue);
|
|
913
|
-
|
|
914
|
-
|
|
1746
|
+
if (MARKER_WITH_DOTS.test(value)) {
|
|
1747
|
+
re.lastIndex = quoted ? from : v.end;
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
if (schemePass && !quoted && schemeFollowerReleased(label, value, text.charCodeAt(hit.index - 1) === 0x2f)) {
|
|
915
1751
|
re.lastIndex = v.end;
|
|
916
1752
|
continue;
|
|
917
1753
|
}
|
|
@@ -927,75 +1763,69 @@ function labelEdits(text, re, literalBlanks = false, unitAt = NO_UNITS) {
|
|
|
927
1763
|
re.lastIndex = v.end;
|
|
928
1764
|
continue;
|
|
929
1765
|
}
|
|
930
|
-
|
|
1766
|
+
let cred = j > v.end && !labelStartsAt(text, j) ? scanSecretValue(text, j, idx) : null;
|
|
1767
|
+
if (j > v.end && !inQuery(j)) {
|
|
1768
|
+
const a = forbiddenTailAt(cred === null ? j : cred.end);
|
|
1769
|
+
if (a !== -1)
|
|
1770
|
+
cred = { start: cred === null ? j : cred.start, end: a };
|
|
1771
|
+
}
|
|
931
1772
|
if (cred !== null) {
|
|
932
1773
|
const credValue = withoutSeparators(text.slice(cred.start, cred.end));
|
|
933
1774
|
const keep = isMarker(credValue) || DIAGNOSTIC_VALUE_WORDS.has(credValue.replace(/[.,;:!?)\]}\x27\x22]+$/, '').toLowerCase());
|
|
934
1775
|
if (!keep)
|
|
935
|
-
|
|
1776
|
+
push(cred.start, cred.end, REDACTED_SECRET);
|
|
936
1777
|
re.lastIndex = cred.end;
|
|
937
1778
|
continue;
|
|
938
1779
|
}
|
|
939
1780
|
re.lastIndex = v.end;
|
|
940
1781
|
continue;
|
|
941
1782
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1783
|
+
const forbiddenAddress = forbiddenAt !== -1 && text[v.start] !== ':';
|
|
1784
|
+
const rawAddress = forbiddenAddress || rawUserinfoAddressAt(text, v.start, quoted ? v.end : text.length, (k) => S.gapAfter(k), stops ??= userinfoStops(text));
|
|
1785
|
+
if (isDiagnosticValue(value, rawValue) || rawAddress) {
|
|
1786
|
+
if (isAddressShaped(value, rawValue) || rawAddress) {
|
|
1787
|
+
const extentFrom = forbiddenAddress ? forbiddenAt : v.start;
|
|
1788
|
+
let end = quoted ? quotedAddressEnd(S, extentFrom, v.end) : addressExtentEnd(text, extentFrom);
|
|
1789
|
+
const trailing = trailingLabelStart(text, v.start, end, idx.tokenInner);
|
|
946
1790
|
if (trailing !== -1)
|
|
947
1791
|
end = v.start + trailing;
|
|
948
1792
|
if (gluedToSingleUnit(v.start, hit.index, unitAt) || userinfoHostHidden(text, v.start, unitAt)) {
|
|
949
1793
|
const cut = trailing !== -1 ? end : Math.max(end, v.end);
|
|
950
|
-
const
|
|
1794
|
+
const schemeHit = ADDRESS_SCHEME_RE.exec(text.slice(v.start, v.start + 20))?.[0] ?? '';
|
|
1795
|
+
const keep = schemeHit !== '' && schemeHit.length - 3 <= 12 ? schemeHit.length : 0;
|
|
951
1796
|
const q = text.slice(v.start, cut).search(/[?#]/);
|
|
952
1797
|
const bodyEnd = q === -1 ? cut : v.start + q;
|
|
953
1798
|
if (withoutSeparators(text.slice(v.start + keep, bodyEnd)) !== '')
|
|
954
|
-
|
|
1799
|
+
push(v.start + keep, bodyEnd, REDACTED_SECRET, -1, 'glued');
|
|
955
1800
|
if (q !== -1 && v.start + q + 1 < cut)
|
|
956
|
-
|
|
957
|
-
re.lastIndex = cut;
|
|
1801
|
+
push(v.start + q + 1, cut, text.charAt(v.start + q) === '#' ? REDACTED_FRAGMENT : REDACTED_QUERY, cut, 'glued');
|
|
1802
|
+
re.lastIndex = quoted ? from : cut;
|
|
958
1803
|
continue;
|
|
959
1804
|
}
|
|
960
|
-
|
|
961
|
-
|
|
1805
|
+
const edits = [];
|
|
1806
|
+
exemptAddressEdits(text, v.start, end, edits, (a, b) => (b > a ? S.raw.slice(S.rawStart(a), S.rawEnd(b - 1)) : ''));
|
|
1807
|
+
pushEdits(edits, end);
|
|
1808
|
+
if (rawAddress) {
|
|
1809
|
+
const limit = quoted ? end : Math.min(end, v.end);
|
|
1810
|
+
const junk = addressJunkStart(text, v.start, limit);
|
|
1811
|
+
if (junk !== -1) {
|
|
1812
|
+
const rest = withoutSeparators(text.slice(junk, limit)).replace(new RegExp(MARKER_ANYWHERE.source, 'g'), '');
|
|
1813
|
+
if (/[A-Za-z0-9]/.test(rest))
|
|
1814
|
+
push(junk, limit, REDACTED_SECRET);
|
|
1815
|
+
}
|
|
1816
|
+
}
|
|
1817
|
+
if (quoted && end < v.end) {
|
|
1818
|
+
maskGaps(end, v.end);
|
|
1819
|
+
}
|
|
1820
|
+
re.lastIndex = quoted ? from : end;
|
|
962
1821
|
continue;
|
|
963
1822
|
}
|
|
964
1823
|
}
|
|
965
1824
|
else {
|
|
966
|
-
|
|
1825
|
+
push(v.start, keptSepEnd !== -1 && withoutSeparators(text.slice(v.end, keptSepEnd)).search(/[^"'\\:=\uFF1A\uFF1D \t]/) === -1 ? keptSepEnd : v.end, REDACTED_SECRET);
|
|
967
1826
|
}
|
|
968
|
-
re.lastIndex = v.end;
|
|
1827
|
+
re.lastIndex = quoted ? from : v.end;
|
|
969
1828
|
}
|
|
970
|
-
return edits;
|
|
971
|
-
}
|
|
972
|
-
function literalBlankLeadJudgedAsBlank(text, v, label, schemePass) {
|
|
973
|
-
let restStart;
|
|
974
|
-
if (v.start > 0 && text.charCodeAt(v.start - 1) === 0x5c && nameBlankTokenAt(text, v.start - 1) !== -1)
|
|
975
|
-
restStart = v.start + 5;
|
|
976
|
-
else if (nameBlankTokenAt(text, v.start) !== -1)
|
|
977
|
-
restStart = v.start + 6;
|
|
978
|
-
else if (eatenBlankTokenTailAt(text, v.start))
|
|
979
|
-
restStart = v.start + 5;
|
|
980
|
-
else
|
|
981
|
-
return false;
|
|
982
|
-
const rest = withoutSeparators(text.slice(restStart, v.end));
|
|
983
|
-
const bare = rest.replace(/[.,;:!?)\]}'"]+$/, '');
|
|
984
|
-
if (bare === '')
|
|
985
|
-
return true;
|
|
986
|
-
if (credentialWordStartsAt(' ' + text.slice(restStart, Math.min(text.length, v.end + 64)), 1))
|
|
987
|
-
return true;
|
|
988
|
-
if (schemePass && SCHEME_FOLLOWER_WORDS.has(bare.toLowerCase()))
|
|
989
|
-
return true;
|
|
990
|
-
if (!schemePass && isNamedNotValue(label, rest))
|
|
991
|
-
return true;
|
|
992
|
-
if (!schemePass && /authorizations?$/i.test(label) && isAuthScheme(bare))
|
|
993
|
-
return true;
|
|
994
|
-
return !isAddressShaped(rest) && isDiagnosticValue(rest);
|
|
995
|
-
}
|
|
996
|
-
function eatenBlankTokenTailAt(text, at) {
|
|
997
|
-
const hit = /^u([0-9A-Fa-f]{4})/.exec(text.slice(at, at + 5));
|
|
998
|
-
return hit !== null && isNameBlank(Number.parseInt(hit[1] ?? '', 16));
|
|
999
1829
|
}
|
|
1000
1830
|
function isAuthParamAt(text, j, scheme) {
|
|
1001
1831
|
const p = /^[A-Za-z][A-Za-z0-9_-]*([ \t\ufeff]*)=([ \t\ufeff]*)("|[^\s=,"]+)([ \t\ufeff]*,)?/.exec(text.slice(j, j + 512));
|
|
@@ -1087,13 +1917,15 @@ function pemBodyEndWithoutTrailer(text, from) {
|
|
|
1087
1917
|
}
|
|
1088
1918
|
return end;
|
|
1089
1919
|
}
|
|
1090
|
-
function
|
|
1091
|
-
const
|
|
1092
|
-
if (!text.includes('-----BEGIN
|
|
1093
|
-
return
|
|
1920
|
+
function pemBlocks(text, soft = () => false) {
|
|
1921
|
+
const out = [];
|
|
1922
|
+
if (!text.includes('-----BEGIN'))
|
|
1923
|
+
return out;
|
|
1924
|
+
const head = pemHeadView(text, soft);
|
|
1094
1925
|
const begin = new RegExp(PEM_BEGIN_SOURCE, 'g');
|
|
1926
|
+
const trailer = /-----END[A-Z0-9 ]*-----/y;
|
|
1095
1927
|
let nextEnd = -2;
|
|
1096
|
-
for (let b = begin.exec(
|
|
1928
|
+
for (let b = begin.exec(head); b !== null; b = begin.exec(head)) {
|
|
1097
1929
|
let bodyStart = b.index + b[0].length;
|
|
1098
1930
|
const lead = lineBreakLengthAt(text, bodyStart);
|
|
1099
1931
|
if (lead > 0)
|
|
@@ -1104,18 +1936,56 @@ function pemEdits(text) {
|
|
|
1104
1936
|
if (nextEnd !== -1 && nextEnd < bodyStart)
|
|
1105
1937
|
nextEnd = text.indexOf(PEM_END_MARK, bodyStart);
|
|
1106
1938
|
let bodyEnd;
|
|
1107
|
-
|
|
1939
|
+
let regionEnd;
|
|
1940
|
+
if (nextEnd === -1) {
|
|
1108
1941
|
bodyEnd = pemBodyEndWithoutTrailer(text, bodyStart);
|
|
1942
|
+
regionEnd = bodyEnd + lineBreakLengthAt(text, bodyEnd);
|
|
1943
|
+
}
|
|
1109
1944
|
else {
|
|
1110
1945
|
bodyEnd = nextEnd;
|
|
1111
1946
|
const back = lineBreakLengthBefore(text, bodyEnd);
|
|
1112
1947
|
if (back > 0 && bodyEnd - back >= bodyStart)
|
|
1113
1948
|
bodyEnd -= back;
|
|
1949
|
+
trailer.lastIndex = nextEnd;
|
|
1950
|
+
regionEnd = trailer.test(head) ? trailer.lastIndex : nextEnd + PEM_END_MARK.length;
|
|
1114
1951
|
}
|
|
1115
|
-
|
|
1116
|
-
pushEdit(edits, text, bodyStart, bodyEnd, REDACTED_SECRET);
|
|
1952
|
+
out.push({ begin: b.index, bodyStart, bodyEnd, regionEnd });
|
|
1117
1953
|
begin.lastIndex = Math.max(begin.lastIndex, bodyEnd);
|
|
1118
1954
|
}
|
|
1955
|
+
return out;
|
|
1956
|
+
}
|
|
1957
|
+
function pemHeadView(text, soft) {
|
|
1958
|
+
let chars = null;
|
|
1959
|
+
for (let k = 0; k < text.length; k++) {
|
|
1960
|
+
if (!soft(k) || text[k] === ' ')
|
|
1961
|
+
continue;
|
|
1962
|
+
chars ??= text.split('');
|
|
1963
|
+
chars[k] = ' ';
|
|
1964
|
+
}
|
|
1965
|
+
return chars === null ? text : chars.join('');
|
|
1966
|
+
}
|
|
1967
|
+
function pemBeginAt(S, at) {
|
|
1968
|
+
const k = at + '-----BEGIN'.length;
|
|
1969
|
+
return S.text.startsWith('-----BEGIN', at) && (S.text[k] === ' ' || (k < S.text.length && pemSoft(S, k)));
|
|
1970
|
+
}
|
|
1971
|
+
const pemSoft = (S, k) => S.dropped(k) || S.painted(k);
|
|
1972
|
+
function pemBlockToRaw(S, blk) {
|
|
1973
|
+
const n = S.text.length;
|
|
1974
|
+
const at = (k) => (k < n ? S.rawStart(k) : S.raw.length);
|
|
1975
|
+
const begin = at(blk.begin);
|
|
1976
|
+
const bodyStart = at(blk.bodyStart);
|
|
1977
|
+
const bodyEnd = blk.bodyEnd > blk.bodyStart ? Math.max(bodyStart, S.rawEnd(blk.bodyEnd - 1)) : bodyStart;
|
|
1978
|
+
const regionEnd = blk.regionEnd > blk.begin ? Math.max(begin, S.rawEnd(blk.regionEnd - 1)) : begin;
|
|
1979
|
+
return { begin, bodyStart, bodyEnd, regionEnd };
|
|
1980
|
+
}
|
|
1981
|
+
function pemEdits(S, onBlock) {
|
|
1982
|
+
const text = S.text;
|
|
1983
|
+
const edits = [];
|
|
1984
|
+
for (const blk of pemBlocks(text, (k) => pemSoft(S, k))) {
|
|
1985
|
+
onBlock(blk);
|
|
1986
|
+
if (blk.bodyEnd > blk.bodyStart)
|
|
1987
|
+
pushEdit(edits, text, blk.bodyStart, blk.bodyEnd, REDACTED_SECRET);
|
|
1988
|
+
}
|
|
1119
1989
|
return edits;
|
|
1120
1990
|
}
|
|
1121
1991
|
function authParamEdits(text) {
|
|
@@ -1188,21 +2058,6 @@ function proseLiteralEdits(text) {
|
|
|
1188
2058
|
}
|
|
1189
2059
|
return edits;
|
|
1190
2060
|
}
|
|
1191
|
-
function washWordNets(target, literalBlanks = false) {
|
|
1192
|
-
const unitAt = (k) => target.unitAt(k);
|
|
1193
|
-
target.apply(pemEdits(target.text));
|
|
1194
|
-
target.apply(labelEdits(target.text, SECRET_SCHEME_WORD, literalBlanks, unitAt));
|
|
1195
|
-
target.apply(labelEdits(target.text, SECRET_LABELLED_WORD, literalBlanks, unitAt));
|
|
1196
|
-
target.apply(authParamEdits(target.text));
|
|
1197
|
-
target.apply(proseLiteralEdits(target.text));
|
|
1198
|
-
}
|
|
1199
|
-
function runCredentialNets(target, urls, words, literalBlanks = false) {
|
|
1200
|
-
if (urls)
|
|
1201
|
-
washUrlNets(target);
|
|
1202
|
-
if (words)
|
|
1203
|
-
washWordNets(target, literalBlanks);
|
|
1204
|
-
}
|
|
1205
|
-
const NEEDS_VIEW = /[\u0000-\u0008\u000E-\u001F\u007F-\u009F\uD800-\uDFFF]|\p{Cf}|\\u/u;
|
|
1206
2061
|
const FORMAT_CHAR = /^\p{Cf}$/u;
|
|
1207
2062
|
const isStrippableControl = (c) => (c <= 0x1f && (c < 0x09 || c > 0x0d)) || (c >= 0x7f && c <= 0x9f);
|
|
1208
2063
|
class SequenceReader {
|
|
@@ -1215,10 +2070,12 @@ class SequenceReader {
|
|
|
1215
2070
|
termHit = null;
|
|
1216
2071
|
breakFrom = -1;
|
|
1217
2072
|
breakAt = -1;
|
|
1218
|
-
|
|
2073
|
+
foldSpaces;
|
|
2074
|
+
constructor(text, atomicBlanks, foldBlanks = false, foldSpaces = false) {
|
|
1219
2075
|
this.text = text;
|
|
1220
2076
|
this.atomicBlanks = atomicBlanks;
|
|
1221
2077
|
this.foldBlanks = foldBlanks;
|
|
2078
|
+
this.foldSpaces = foldSpaces;
|
|
1222
2079
|
this.terminator = /[\u0007\u009c]|\u001b\\|\\u(?:0007|009[cC]|001[bB]\\)/g;
|
|
1223
2080
|
this.lineBreak = /[\n\r]/g;
|
|
1224
2081
|
}
|
|
@@ -1236,6 +2093,8 @@ class SequenceReader {
|
|
|
1236
2093
|
const c = this.text.charCodeAt(at);
|
|
1237
2094
|
if (c >= 0x20 && c <= 0x2f)
|
|
1238
2095
|
return 1;
|
|
2096
|
+
if (this.foldSpaces && ((c >= 0x09 && c <= 0x0d) || c === 0x2028 || c === 0x2029))
|
|
2097
|
+
return 1;
|
|
1239
2098
|
if (!this.foldBlanks)
|
|
1240
2099
|
return 0;
|
|
1241
2100
|
if (isFoldedBlank(c))
|
|
@@ -1351,41 +2210,38 @@ function nameBlankTokenAt(text, at) {
|
|
|
1351
2210
|
const cp = Number.parseInt(hit[1] ?? '', 16);
|
|
1352
2211
|
return isNameBlank(cp) ? cp : -1;
|
|
1353
2212
|
}
|
|
1354
|
-
function
|
|
1355
|
-
const folding = blankTokens === 'fold' || blankTokens === 'foldlit';
|
|
1356
|
-
if (!NEEDS_VIEW.test(text) && !(folding && hasRawFoldedBlank(text)))
|
|
1357
|
-
return null;
|
|
2213
|
+
function buildScanView(text, o) {
|
|
1358
2214
|
const chars = [];
|
|
1359
2215
|
const starts = [];
|
|
1360
2216
|
const ends = [];
|
|
1361
2217
|
const units = [];
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
const
|
|
2218
|
+
const dropped = [];
|
|
2219
|
+
const spaceUnits = [];
|
|
2220
|
+
const contents = [];
|
|
2221
|
+
const seq = new SequenceReader(text, o.blanks ? 'all' : 'none', o.blanks || o.fold, o.fold);
|
|
1365
2222
|
const keep = (ch, s, e, unit = UNIT_NONE) => {
|
|
1366
2223
|
chars.push(ch);
|
|
1367
2224
|
starts.push(s);
|
|
1368
2225
|
ends.push(e);
|
|
1369
2226
|
units.push(unit);
|
|
2227
|
+
dropped.push(0);
|
|
2228
|
+
spaceUnits.push(0);
|
|
1370
2229
|
};
|
|
1371
|
-
const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE) => {
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
2230
|
+
const drop = (s, e, placeholder = VIEW_SEPARATOR, unit = UNIT_SINGLE, space = 0) => {
|
|
2231
|
+
chars.push(placeholder);
|
|
2232
|
+
starts.push(s);
|
|
2233
|
+
ends.push(e);
|
|
2234
|
+
units.push(unit);
|
|
2235
|
+
dropped.push(1);
|
|
2236
|
+
spaceUnits.push(space);
|
|
1375
2237
|
};
|
|
1376
2238
|
const dropUnit = (s, unit) => {
|
|
1377
2239
|
const last = unit.end - 1;
|
|
1378
|
-
if (
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
for (let u = 2; u < 6; u++)
|
|
1382
|
-
keep(text.charAt(last + u), last + 6, last + 6);
|
|
1383
|
-
return last + 6;
|
|
1384
|
-
}
|
|
1385
|
-
if (wordLetters && unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
|
|
2240
|
+
if (unit.content !== undefined)
|
|
2241
|
+
contents.push([unit.content[0], unit.content[1]]);
|
|
2242
|
+
if (unit.content === undefined && last > s && (credentialWordBeginsWith(text, last) || finalIsStructural(text, last))) {
|
|
1386
2243
|
drop(s, last, VIEW_SEPARATOR, UNIT_SEQUENCE);
|
|
1387
2244
|
keep(text.charAt(last), last, unit.end);
|
|
1388
|
-
splitAny = true;
|
|
1389
2245
|
return unit.end;
|
|
1390
2246
|
}
|
|
1391
2247
|
drop(s, unit.end, VIEW_SEPARATOR, UNIT_SEQUENCE);
|
|
@@ -1394,12 +2250,6 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1394
2250
|
let i = 0;
|
|
1395
2251
|
while (i < text.length) {
|
|
1396
2252
|
const c = text.charCodeAt(i);
|
|
1397
|
-
if (folding && isFoldedBlank(c)) {
|
|
1398
|
-
changed = true;
|
|
1399
|
-
keep(' ', i, i + 1);
|
|
1400
|
-
i++;
|
|
1401
|
-
continue;
|
|
1402
|
-
}
|
|
1403
2253
|
if (c === 0x1b || c === 0x9b || c === 0x9d || c === 0x90 || c === 0x98 || c === 0x9e || c === 0x9f) {
|
|
1404
2254
|
const unit = c === 0x1b ? escapeSequenceAfterEsc(text, i + 1, seq) : c1SequenceAfter(text, c, i + 1, seq);
|
|
1405
2255
|
if (unit !== null) {
|
|
@@ -1422,30 +2272,33 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1422
2272
|
i = dropUnit(i, unit);
|
|
1423
2273
|
continue;
|
|
1424
2274
|
}
|
|
1425
|
-
|
|
2275
|
+
if (isRegexSpace(tok.cp) && tok.cp !== 0xfeff) {
|
|
2276
|
+
if (o.escapeSpaces !== null && o.escapeSpaces.has(tok.cp))
|
|
2277
|
+
drop(i, tok.end, VIEW_SEPARATOR, UNIT_SINGLE, 1);
|
|
2278
|
+
else
|
|
2279
|
+
drop(i, tok.end, ' ', UNIT_NONE);
|
|
2280
|
+
}
|
|
2281
|
+
else
|
|
2282
|
+
drop(i, tok.end);
|
|
1426
2283
|
i = tok.end;
|
|
1427
2284
|
continue;
|
|
1428
2285
|
}
|
|
1429
|
-
if (
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
if (blankTokens === 'blank')
|
|
1434
|
-
keep(String.fromCharCode(cp), i, i + 6);
|
|
1435
|
-
else if (blankTokens === 'fold')
|
|
1436
|
-
keep(isFoldedBlank(cp) ? ' ' : String.fromCharCode(cp), i, i + 6);
|
|
1437
|
-
else if (blankTokens === 'foldlit' && isFoldedBlank(cp))
|
|
1438
|
-
keep(' ', i, i + 6);
|
|
1439
|
-
else {
|
|
1440
|
-
keep(text.charAt(i), i, i + 6);
|
|
1441
|
-
for (let u = 1; u < 6; u++)
|
|
1442
|
-
keep(text.charAt(i + u), i + 6, i + 6);
|
|
1443
|
-
}
|
|
1444
|
-
i += 6;
|
|
1445
|
-
continue;
|
|
1446
|
-
}
|
|
2286
|
+
if (nameBlankTokenAt(text, i) !== -1) {
|
|
2287
|
+
drop(i, i + 6);
|
|
2288
|
+
i += 6;
|
|
2289
|
+
continue;
|
|
1447
2290
|
}
|
|
1448
2291
|
}
|
|
2292
|
+
if (o.blanks && isNameBlank(c)) {
|
|
2293
|
+
drop(i, i + 1);
|
|
2294
|
+
i++;
|
|
2295
|
+
continue;
|
|
2296
|
+
}
|
|
2297
|
+
if (o.escapeSpaces !== null && o.escapeSpaces.has(c)) {
|
|
2298
|
+
drop(i, i + 1, VIEW_SEPARATOR, UNIT_SINGLE, 1);
|
|
2299
|
+
i++;
|
|
2300
|
+
continue;
|
|
2301
|
+
}
|
|
1449
2302
|
if (isStrippableControl(c)) {
|
|
1450
2303
|
drop(i, i + 1);
|
|
1451
2304
|
i++;
|
|
@@ -1467,9 +2320,195 @@ function buildCredentialView(text, separated, wordLetters = false, blankTokens =
|
|
|
1467
2320
|
keep(text.charAt(i), i, i + 1);
|
|
1468
2321
|
i++;
|
|
1469
2322
|
}
|
|
1470
|
-
|
|
2323
|
+
const rawUnit = new Uint8Array(text.length);
|
|
2324
|
+
for (let k = 0; k < chars.length; k++) {
|
|
2325
|
+
if (dropped[k] !== 1)
|
|
2326
|
+
continue;
|
|
2327
|
+
for (let x = starts[k] ?? 0; x < (ends[k] ?? 0); x++)
|
|
2328
|
+
rawUnit[x] = 1;
|
|
2329
|
+
}
|
|
2330
|
+
return {
|
|
2331
|
+
view: chars.join(''),
|
|
2332
|
+
starts: Int32Array.from(starts),
|
|
2333
|
+
ends: Int32Array.from(ends),
|
|
2334
|
+
units: Uint8Array.from(units),
|
|
2335
|
+
dropped: Uint8Array.from(dropped),
|
|
2336
|
+
spaceUnits: Uint8Array.from(spaceUnits),
|
|
2337
|
+
contents,
|
|
2338
|
+
rawUnit,
|
|
2339
|
+
};
|
|
2340
|
+
}
|
|
2341
|
+
const viewRawStart = (V, M, k) => {
|
|
2342
|
+
const s = V.starts[k] ?? 0;
|
|
2343
|
+
return M === null ? s : (M.ps[s] ?? 0);
|
|
2344
|
+
};
|
|
2345
|
+
const viewRawEnd = (V, M, k) => {
|
|
2346
|
+
const e = V.ends[k] ?? 0;
|
|
2347
|
+
return M === null ? e : (M.pe[Math.max(0, e - 1)] ?? 0);
|
|
2348
|
+
};
|
|
2349
|
+
const viewPainted = (V, M, k) => {
|
|
2350
|
+
if (M === null)
|
|
2351
|
+
return false;
|
|
2352
|
+
for (let x = V.starts[k] ?? 0; x < (V.ends[k] ?? 0); x++)
|
|
2353
|
+
if (M.pdot[x] === 1)
|
|
2354
|
+
return true;
|
|
2355
|
+
return false;
|
|
2356
|
+
};
|
|
2357
|
+
function separatedSurface(raw, V, M = null) {
|
|
2358
|
+
return {
|
|
2359
|
+
text: V.view,
|
|
2360
|
+
raw,
|
|
2361
|
+
unitAt: (k) => V.units[k] ?? UNIT_NONE,
|
|
2362
|
+
rawStart: (k) => viewRawStart(V, M, k),
|
|
2363
|
+
rawEnd: (k) => viewRawEnd(V, M, k),
|
|
2364
|
+
dropped: (k) => V.dropped[k] === 1,
|
|
2365
|
+
painted: (k) => viewPainted(V, M, k),
|
|
2366
|
+
gapAfter: (k) => V.dropped[k + 1] === 1,
|
|
2367
|
+
spaceGapBefore: () => false,
|
|
2368
|
+
rawDot: M === null ? null : M.rawDot,
|
|
2369
|
+
};
|
|
2370
|
+
}
|
|
2371
|
+
function joinedSurface(raw, V, spacesKept = false, M = null) {
|
|
2372
|
+
const map = [];
|
|
2373
|
+
const chars = [];
|
|
2374
|
+
const spaceBefore = [];
|
|
2375
|
+
let pendingSpace = 0;
|
|
2376
|
+
let keptAny = false;
|
|
2377
|
+
for (let k = 0; k < V.view.length; k++) {
|
|
2378
|
+
if (V.dropped[k] === 1) {
|
|
2379
|
+
if (V.spaceUnits[k] === 1) {
|
|
2380
|
+
if (spacesKept) {
|
|
2381
|
+
map.push(k);
|
|
2382
|
+
chars.push(' ');
|
|
2383
|
+
spaceBefore.push(0);
|
|
2384
|
+
keptAny = true;
|
|
2385
|
+
continue;
|
|
2386
|
+
}
|
|
2387
|
+
pendingSpace = 1;
|
|
2388
|
+
}
|
|
2389
|
+
continue;
|
|
2390
|
+
}
|
|
2391
|
+
map.push(k);
|
|
2392
|
+
chars.push(V.view.charAt(k));
|
|
2393
|
+
spaceBefore.push(pendingSpace);
|
|
2394
|
+
pendingSpace = 0;
|
|
2395
|
+
}
|
|
2396
|
+
if (map.length === V.view.length || (spacesKept && !keptAny))
|
|
1471
2397
|
return null;
|
|
1472
|
-
|
|
2398
|
+
const arr = Int32Array.from(map);
|
|
2399
|
+
const gapSpace = Uint8Array.from(spaceBefore);
|
|
2400
|
+
return {
|
|
2401
|
+
text: chars.join(''),
|
|
2402
|
+
raw,
|
|
2403
|
+
unitAt: () => UNIT_NONE,
|
|
2404
|
+
rawStart: (k) => viewRawStart(V, M, arr[k] ?? 0),
|
|
2405
|
+
rawEnd: (k) => viewRawEnd(V, M, arr[k] ?? 0),
|
|
2406
|
+
dropped: () => false,
|
|
2407
|
+
painted: (k) => viewPainted(V, M, arr[k] ?? 0),
|
|
2408
|
+
gapAfter: (k) => k + 1 < arr.length && arr[k + 1] !== (arr[k] ?? 0) + 1,
|
|
2409
|
+
spaceGapBefore: (k) => gapSpace[k] === 1,
|
|
2410
|
+
rawDot: M === null ? null : M.rawDot,
|
|
2411
|
+
};
|
|
2412
|
+
}
|
|
2413
|
+
function screenSurfaces(text, o) {
|
|
2414
|
+
const face = o.face;
|
|
2415
|
+
if (face === null)
|
|
2416
|
+
return { surfaces: [], rawSoft: null, rawPlain: null, contents: [] };
|
|
2417
|
+
const unsafe = face.unsafe === null ? null : new RegExp(face.unsafe.source, 'u');
|
|
2418
|
+
const keepBlanks = face.keepBlanks === true;
|
|
2419
|
+
const chars = [];
|
|
2420
|
+
const ps = [];
|
|
2421
|
+
const pe = [];
|
|
2422
|
+
const pdot = [];
|
|
2423
|
+
const rawDot = new Uint8Array(text.length);
|
|
2424
|
+
const rawPlain = new Uint8Array(text.length);
|
|
2425
|
+
let changed = false;
|
|
2426
|
+
const foldable = (at) => {
|
|
2427
|
+
const c = text.codePointAt(at) ?? 0;
|
|
2428
|
+
return isRegexSpace(c) && !(keepBlanks && isNameBlank(c));
|
|
2429
|
+
};
|
|
2430
|
+
let lo = 0;
|
|
2431
|
+
let hi = text.length;
|
|
2432
|
+
if (face.fold) {
|
|
2433
|
+
while (lo < hi && foldable(lo))
|
|
2434
|
+
lo++;
|
|
2435
|
+
while (hi > lo && foldable(hi - 1))
|
|
2436
|
+
hi--;
|
|
2437
|
+
if (lo > 0 || hi < text.length)
|
|
2438
|
+
changed = true;
|
|
2439
|
+
}
|
|
2440
|
+
for (let i = lo; i < hi;) {
|
|
2441
|
+
const cp = text.codePointAt(i) ?? 0;
|
|
2442
|
+
const w = cp > 0xffff ? 2 : 1;
|
|
2443
|
+
const ch = text.slice(i, i + w);
|
|
2444
|
+
let form = null;
|
|
2445
|
+
let dotted = false;
|
|
2446
|
+
if (face.fold && foldable(i)) {
|
|
2447
|
+
let j = i + 1;
|
|
2448
|
+
while (j < hi && foldable(j))
|
|
2449
|
+
j++;
|
|
2450
|
+
if (j - i > 1 || cp !== 0x20)
|
|
2451
|
+
changed = true;
|
|
2452
|
+
chars.push(' ');
|
|
2453
|
+
ps.push(i);
|
|
2454
|
+
pe.push(j);
|
|
2455
|
+
pdot.push(cp === 0xfeff || j - i > 1 ? 1 : 0);
|
|
2456
|
+
for (let x = i; x < j; x++)
|
|
2457
|
+
if (text.charCodeAt(x) === 0xfeff)
|
|
2458
|
+
rawDot[x] = 1;
|
|
2459
|
+
i = j;
|
|
2460
|
+
continue;
|
|
2461
|
+
}
|
|
2462
|
+
else if (unsafe !== null && unsafe.test(ch) && !faceHitKept(ch, i, text)) {
|
|
2463
|
+
form = markOf(ch, face.mark);
|
|
2464
|
+
dotted = face.mark !== 'escape';
|
|
2465
|
+
}
|
|
2466
|
+
if (form === null || form === ch) {
|
|
2467
|
+
for (let u = 0; u < w; u++) {
|
|
2468
|
+
chars.push(text.charAt(i + u));
|
|
2469
|
+
ps.push(i + u);
|
|
2470
|
+
pe.push(i + u + 1);
|
|
2471
|
+
pdot.push(0);
|
|
2472
|
+
rawPlain[i + u] = 1;
|
|
2473
|
+
}
|
|
2474
|
+
}
|
|
2475
|
+
else {
|
|
2476
|
+
changed = true;
|
|
2477
|
+
for (let u = 0; u < form.length; u++) {
|
|
2478
|
+
chars.push(form.charAt(u));
|
|
2479
|
+
ps.push(i);
|
|
2480
|
+
pe.push(i + w);
|
|
2481
|
+
pdot.push(dotted ? 1 : 0);
|
|
2482
|
+
}
|
|
2483
|
+
if (dotted)
|
|
2484
|
+
for (let x = i; x < i + w; x++)
|
|
2485
|
+
rawDot[x] = 1;
|
|
2486
|
+
}
|
|
2487
|
+
i += w;
|
|
2488
|
+
}
|
|
2489
|
+
if (!changed)
|
|
2490
|
+
return { surfaces: [], rawSoft: null, rawPlain: null, contents: [] };
|
|
2491
|
+
const screen = chars.join('');
|
|
2492
|
+
const M = { ps: Int32Array.from(ps), pe: Int32Array.from(pe), pdot: Uint8Array.from(pdot), rawDot };
|
|
2493
|
+
const V = buildScanView(screen, o);
|
|
2494
|
+
const rawSoft = Uint8Array.from(rawDot);
|
|
2495
|
+
for (let k = 0; k < V.view.length; k++) {
|
|
2496
|
+
if (V.dropped[k] !== 1)
|
|
2497
|
+
continue;
|
|
2498
|
+
for (let x = viewRawStart(V, M, k); x < viewRawEnd(V, M, k); x++) {
|
|
2499
|
+
rawSoft[x] = 1;
|
|
2500
|
+
rawPlain[x] = 0;
|
|
2501
|
+
}
|
|
2502
|
+
}
|
|
2503
|
+
const out = [separatedSurface(text, V, M)];
|
|
2504
|
+
const joined = joinedSurface(text, V, false, M);
|
|
2505
|
+
if (joined !== null)
|
|
2506
|
+
out.push(joined);
|
|
2507
|
+
const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true, M) : null;
|
|
2508
|
+
if (joinedSpaced !== null)
|
|
2509
|
+
out.push(joinedSpaced);
|
|
2510
|
+
const contents = V.contents.map(([a, b]) => ({ sub: screen.slice(a, b), rawAt: (k) => M.ps[a + k] ?? 0, rawEndAt: (k) => M.pe[a + k] ?? 0 }));
|
|
2511
|
+
return { surfaces: out, rawSoft, rawPlain, contents };
|
|
1473
2512
|
}
|
|
1474
2513
|
function finalIsStructural(text, at) {
|
|
1475
2514
|
if (text.charCodeAt(at) !== 0x5c)
|
|
@@ -1497,260 +2536,350 @@ function credentialWordBeginsWith(text, at) {
|
|
|
1497
2536
|
let end = Math.min(text.length, at + 64);
|
|
1498
2537
|
while (end < text.length && isProbeFiller(text.charCodeAt(end - 1)))
|
|
1499
2538
|
end++;
|
|
1500
|
-
const probe = ' ' + text.slice(at, Math.min(text.length, end + 2));
|
|
2539
|
+
const probe = ' ' + text.slice(at, Math.min(text.length, end + 2)) + (end + 2 >= text.length ? ' ' : '');
|
|
1501
2540
|
if (letter)
|
|
1502
2541
|
return credentialWordStartsAt(probe, 1);
|
|
1503
2542
|
return credentialWordStartsAt(probe, 2) || credentialWordStartsAt(probe, 3);
|
|
1504
2543
|
}
|
|
1505
|
-
function
|
|
2544
|
+
function editsToCandidates(edits, cls, out, te = -1) {
|
|
2545
|
+
for (const e of edits) {
|
|
2546
|
+
const kind = KIND_BY_TEXT.get(e.text);
|
|
2547
|
+
if (kind === undefined || e.end <= e.start)
|
|
2548
|
+
continue;
|
|
2549
|
+
out.push({ a: e.start, b: e.end, kind, cls, te });
|
|
2550
|
+
}
|
|
2551
|
+
}
|
|
2552
|
+
function pemRegions(recognized) {
|
|
2553
|
+
const seen = new Set();
|
|
1506
2554
|
const out = [];
|
|
1507
|
-
const
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
origin.push(x);
|
|
1513
|
-
};
|
|
1514
|
-
const inserted = (s) => {
|
|
1515
|
-
out.push(s);
|
|
1516
|
-
if (origin !== null)
|
|
1517
|
-
for (let x = 0; x < s.length; x++)
|
|
1518
|
-
origin.push(-1);
|
|
1519
|
-
};
|
|
1520
|
-
const viewLength = view.view.length;
|
|
1521
|
-
let cursor = 0;
|
|
1522
|
-
let k = 0;
|
|
1523
|
-
const text = target.text;
|
|
1524
|
-
while (k < text.length) {
|
|
1525
|
-
const s = target.src[k] ?? 0;
|
|
1526
|
-
if (s >= 0) {
|
|
1527
|
-
const end = view.ends[s] ?? cursor;
|
|
1528
|
-
if ((view.starts[s] ?? 0) >= cursor) {
|
|
1529
|
-
fromOriginal(cursor, end);
|
|
1530
|
-
cursor = end;
|
|
1531
|
-
}
|
|
1532
|
-
else
|
|
1533
|
-
inserted(text.charAt(k));
|
|
1534
|
-
k++;
|
|
2555
|
+
for (const blk of recognized) {
|
|
2556
|
+
if (blk.regionEnd <= blk.begin)
|
|
2557
|
+
continue;
|
|
2558
|
+
const key = `${blk.begin}:${blk.bodyStart}:${blk.bodyEnd}:${blk.regionEnd}`;
|
|
2559
|
+
if (seen.has(key))
|
|
1535
2560
|
continue;
|
|
2561
|
+
seen.add(key);
|
|
2562
|
+
out.push(blk);
|
|
2563
|
+
}
|
|
2564
|
+
return out;
|
|
2565
|
+
}
|
|
2566
|
+
function resolveCandidates(text, cands, rawUnit, rawDot, pems, rawSoft = rawDot, rawPlain = null) {
|
|
2567
|
+
const items = [];
|
|
2568
|
+
for (const c of cands) {
|
|
2569
|
+
let a = c.start;
|
|
2570
|
+
let b = c.end;
|
|
2571
|
+
if (c.cls === 'pem') {
|
|
1536
2572
|
}
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
2573
|
+
else if (c.dotted === true) {
|
|
2574
|
+
while (a < b && rawDot !== null && rawDot[a] === 1)
|
|
2575
|
+
a++;
|
|
2576
|
+
if (a >= b)
|
|
2577
|
+
a = c.start;
|
|
2578
|
+
else if (rawDot !== null && b < text.length && isKeyWordChar(text.charCodeAt(b))) {
|
|
2579
|
+
while (b > a && rawDot[b - 1] === 1)
|
|
2580
|
+
b--;
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
else if (c.pretrimmed !== true) {
|
|
2584
|
+
while (a < b && rawUnit[a] === 1)
|
|
2585
|
+
a++;
|
|
2586
|
+
while (b > a && rawUnit[b - 1] === 1)
|
|
2587
|
+
b--;
|
|
2588
|
+
if (rawPlain !== null && rawDot !== null) {
|
|
2589
|
+
let k = a;
|
|
2590
|
+
while (k > 0 && rawUnit[k - 1] === 1 && rawPlain[k - 1] === 1 && isKeyWordChar(text.charCodeAt(k - 1)))
|
|
2591
|
+
k--;
|
|
2592
|
+
if (k < a && k > 0 && (rawDot[k - 1] === 1 || isRegexSpace(text.charCodeAt(k - 1))))
|
|
2593
|
+
a = k;
|
|
2594
|
+
}
|
|
2595
|
+
}
|
|
2596
|
+
if (a >= b)
|
|
2597
|
+
continue;
|
|
2598
|
+
const body = text.slice(a, b);
|
|
2599
|
+
if (isMarker(body) || MARKER_WITH_DOTS.test(body))
|
|
2600
|
+
continue;
|
|
2601
|
+
items.push({ start: a, end: b, kind: c.kind, cls: c.cls, tokenEnd: c.tokenEnd, origin: c.dotted === true ? c.start : a });
|
|
1553
2602
|
}
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
function washThroughView(original, view, urls, words, trace = false, literalBlanks = false) {
|
|
1559
|
-
const target = new TrackedTarget(view.view, view.view.length, view.units);
|
|
1560
|
-
runCredentialNets(target, urls, words, literalBlanks);
|
|
1561
|
-
return target.text === view.view ? { text: original } : rebuildFromView(original, view, target, trace);
|
|
1562
|
-
}
|
|
1563
|
-
function washCredentialNetsOnce(text, urls, words, nested, blankTokens, trace) {
|
|
1564
|
-
const contents = nested ? { text } : washControlStringContents(text, urls, words, blankTokens, trace);
|
|
1565
|
-
if (blankTokens === 'off')
|
|
1566
|
-
return washThroughViews(contents, urls, words, 'off', trace);
|
|
1567
|
-
const tokens = hasNameBlankToken(contents.text);
|
|
1568
|
-
const readings = [];
|
|
1569
|
-
if (tokens)
|
|
1570
|
-
readings.push('literal');
|
|
1571
|
-
const rawFolded = blankTokens === 'fold' && hasRawFoldedBlank(contents.text);
|
|
1572
|
-
if (blankTokens === 'fold' && (tokens || rawFolded))
|
|
1573
|
-
readings.push('fold');
|
|
1574
|
-
if (blankTokens === 'fold' && tokens && hasNameBlankToken(contents.text, false) && (rawFolded || hasNameBlankToken(contents.text, true)))
|
|
1575
|
-
readings.push('foldlit');
|
|
1576
|
-
readings.push('blank');
|
|
1577
|
-
if (readings.length === 1)
|
|
1578
|
-
return washThroughViews(contents, urls, words, 'blank', trace);
|
|
1579
|
-
const base = { text: contents.text };
|
|
1580
|
-
const union = unionMaskedRanges(base.text, readings.map((r) => washThroughViews(base, urls, words, r, true)));
|
|
1581
|
-
return traced(union.text, composeOrigin(contents.origin, union.origin));
|
|
1582
|
-
}
|
|
1583
|
-
function unionMaskedRanges(text, results) {
|
|
1584
|
-
const ranges = [];
|
|
1585
|
-
for (const r of results) {
|
|
1586
|
-
if (r.text === text || r.origin === undefined)
|
|
2603
|
+
items.sort((x, y) => (x.start !== y.start ? x.start - y.start : y.end - x.end));
|
|
2604
|
+
let maxTokenEnd = null;
|
|
2605
|
+
for (const x of items) {
|
|
2606
|
+
if (!isStructuralKind(x.kind) || x.cls !== 'url')
|
|
1587
2607
|
continue;
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
let k =
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
2608
|
+
if (maxTokenEnd === null)
|
|
2609
|
+
maxTokenEnd = new Int32Array(text.length).fill(-1);
|
|
2610
|
+
for (let k = x.start; k < x.end && k < text.length; k++)
|
|
2611
|
+
if (x.tokenEnd > (maxTokenEnd[k] ?? -1))
|
|
2612
|
+
maxTokenEnd[k] = x.tokenEnd;
|
|
2613
|
+
}
|
|
2614
|
+
const p2Drops = (x) => !isStructuralKind(x.kind) && x.cls !== 'pem' && x.cls !== 'glued' && maxTokenEnd !== null && x.end <= (maxTokenEnd[x.start] ?? -1);
|
|
2615
|
+
let pemCover = null;
|
|
2616
|
+
if (pems.length > 0) {
|
|
2617
|
+
const reachBefore = new Int32Array(text.length + 1).fill(-1);
|
|
2618
|
+
for (const x of items)
|
|
2619
|
+
if (x.cls !== 'pem' && !p2Drops(x) && x.origin < text.length && x.end > (reachBefore[x.origin + 1] ?? -1))
|
|
2620
|
+
reachBefore[x.origin + 1] = x.end;
|
|
2621
|
+
for (let k = 1; k <= text.length; k++)
|
|
2622
|
+
if ((reachBefore[k - 1] ?? -1) > (reachBefore[k] ?? -1))
|
|
2623
|
+
reachBefore[k] = reachBefore[k - 1] ?? -1;
|
|
2624
|
+
let pemMask = null;
|
|
2625
|
+
for (const x of items) {
|
|
2626
|
+
if (x.cls !== 'pem')
|
|
1595
2627
|
continue;
|
|
2628
|
+
pemMask ??= new Uint8Array(text.length);
|
|
2629
|
+
for (let k = x.start; k < x.end && k < text.length; k++)
|
|
2630
|
+
pemMask[k] = 1;
|
|
2631
|
+
}
|
|
2632
|
+
for (const blk of pems) {
|
|
2633
|
+
const a = blk.begin;
|
|
2634
|
+
const b = blk.regionEnd;
|
|
2635
|
+
if ((reachBefore[a] ?? -1) > a)
|
|
2636
|
+
continue;
|
|
2637
|
+
pemCover ??= new Uint8Array(text.length);
|
|
2638
|
+
for (let k = a; k < b && k < text.length; k++) {
|
|
2639
|
+
if (k >= blk.bodyStart && k < blk.bodyEnd && rawUnit[k] === 1 && pemMask?.[k] !== 1)
|
|
2640
|
+
continue;
|
|
2641
|
+
pemCover[k] = 1;
|
|
1596
2642
|
}
|
|
1597
|
-
let j = k;
|
|
1598
|
-
while (j < origin.length && (origin[j] ?? -1) < 0)
|
|
1599
|
-
j++;
|
|
1600
|
-
const next = j < origin.length ? (origin[j] ?? text.length) : text.length;
|
|
1601
|
-
ranges.push({ start: lastKept + 1, end: next, text: r.text.slice(k, j) });
|
|
1602
|
-
k = j;
|
|
1603
2643
|
}
|
|
1604
2644
|
}
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
2645
|
+
const kept = items.filter((x) => {
|
|
2646
|
+
if (x.cls !== 'pem' && pemCover !== null && pemCover[x.origin] === 1)
|
|
2647
|
+
return false;
|
|
2648
|
+
if (p2Drops(x))
|
|
2649
|
+
return false;
|
|
2650
|
+
return true;
|
|
2651
|
+
});
|
|
2652
|
+
const groups = [];
|
|
2653
|
+
for (const it of kept) {
|
|
2654
|
+
const g = groups[groups.length - 1];
|
|
2655
|
+
if (g !== undefined && it.start < g.end) {
|
|
2656
|
+
if (it.end > g.end)
|
|
2657
|
+
g.end = it.end;
|
|
2658
|
+
g.members.push(it);
|
|
1617
2659
|
continue;
|
|
1618
2660
|
}
|
|
1619
|
-
|
|
2661
|
+
groups.push({ start: it.start, end: it.end, members: [it] });
|
|
2662
|
+
}
|
|
2663
|
+
const out = [];
|
|
2664
|
+
for (const g of groups) {
|
|
2665
|
+
let structStart = Infinity;
|
|
2666
|
+
let structEnd = -Infinity;
|
|
2667
|
+
let firstStructKind = null;
|
|
2668
|
+
let firstStructStart = Infinity;
|
|
2669
|
+
let anySecret = false;
|
|
2670
|
+
const secretLike = (m) => !isStructuralKind(m.kind) || m.cls === 'userinfo-hidden';
|
|
2671
|
+
for (const m of g.members) {
|
|
2672
|
+
if (!secretLike(m)) {
|
|
2673
|
+
if (m.start < structStart)
|
|
2674
|
+
structStart = m.start;
|
|
2675
|
+
if (m.end > structEnd)
|
|
2676
|
+
structEnd = m.end;
|
|
2677
|
+
if (m.start < firstStructStart) {
|
|
2678
|
+
firstStructStart = m.start;
|
|
2679
|
+
firstStructKind = m.kind;
|
|
2680
|
+
}
|
|
2681
|
+
}
|
|
2682
|
+
else
|
|
2683
|
+
anySecret = true;
|
|
2684
|
+
}
|
|
2685
|
+
let secretWider = false;
|
|
2686
|
+
if (anySecret)
|
|
2687
|
+
for (const m of g.members)
|
|
2688
|
+
if (secretLike(m) && (m.start < structStart || m.end > structEnd))
|
|
2689
|
+
secretWider = true;
|
|
2690
|
+
const onlyHidden = g.members.every((m) => m.cls === 'userinfo-hidden');
|
|
2691
|
+
const kind = onlyHidden ? 'userinfo' : firstStructKind === null || secretWider ? 'secret' : firstStructKind;
|
|
2692
|
+
const value = g.members.some((m) => m.cls === 'word' || m.cls === 'glued' || m.cls === 'param' || (m.kind !== 'userinfo' && m.kind !== 'secret' && (m.cls === 'url' || m.cls === 'structural')));
|
|
2693
|
+
out.push({ start: g.start, end: g.end, kind, value });
|
|
2694
|
+
}
|
|
2695
|
+
return continueValues(text, out, rawSoft ?? rawUnit);
|
|
2696
|
+
}
|
|
2697
|
+
function continueValues(text, drafts, rawSoft) {
|
|
2698
|
+
const out = [];
|
|
2699
|
+
const soft = (k) => rawSoft[k] === 1;
|
|
2700
|
+
let i = 0;
|
|
2701
|
+
while (i < drafts.length) {
|
|
2702
|
+
const cur = drafts[i];
|
|
2703
|
+
i++;
|
|
2704
|
+
if (cur === undefined)
|
|
1620
2705
|
continue;
|
|
1621
|
-
|
|
2706
|
+
const d = { start: cur.start, end: cur.end, kind: cur.kind, value: cur.value };
|
|
2707
|
+
for (;;) {
|
|
2708
|
+
let k = d.end;
|
|
2709
|
+
if (!d.value || k >= text.length || !soft(k))
|
|
2710
|
+
break;
|
|
2711
|
+
while (k < text.length && soft(k))
|
|
2712
|
+
k++;
|
|
2713
|
+
if (k >= text.length || !isKeyWordChar(text.charCodeAt(k)) || credentialWordStartsAt(text, k))
|
|
2714
|
+
break;
|
|
2715
|
+
let e = k;
|
|
2716
|
+
while (e < text.length && (soft(e) || !isBareValueStop(text.charAt(e))))
|
|
2717
|
+
e++;
|
|
2718
|
+
d.end = e;
|
|
2719
|
+
while (i < drafts.length && (drafts[i]?.start ?? Infinity) < d.end) {
|
|
2720
|
+
const next = drafts[i];
|
|
2721
|
+
i++;
|
|
2722
|
+
if (next === undefined)
|
|
2723
|
+
continue;
|
|
2724
|
+
if (next.end > d.end)
|
|
2725
|
+
d.end = next.end;
|
|
2726
|
+
if (next.kind !== d.kind)
|
|
2727
|
+
d.kind = 'secret';
|
|
2728
|
+
}
|
|
2729
|
+
}
|
|
2730
|
+
out.push(d);
|
|
1622
2731
|
}
|
|
2732
|
+
return out;
|
|
2733
|
+
}
|
|
2734
|
+
function materialize(text, intervals, trace) {
|
|
2735
|
+
if (intervals.length === 0)
|
|
2736
|
+
return { text };
|
|
1623
2737
|
const parts = [];
|
|
1624
|
-
const origin = [];
|
|
2738
|
+
const origin = trace ? [] : null;
|
|
1625
2739
|
let cursor = 0;
|
|
1626
|
-
for (const
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
2740
|
+
for (const iv of intervals) {
|
|
2741
|
+
const token = TEXT_BY_KIND[iv.kind];
|
|
2742
|
+
parts.push(text.slice(cursor, iv.start), token);
|
|
2743
|
+
if (origin !== null) {
|
|
2744
|
+
for (let x = cursor; x < iv.start; x++)
|
|
2745
|
+
origin.push(x);
|
|
2746
|
+
for (let x = 0; x < token.length; x++)
|
|
2747
|
+
origin.push(-1);
|
|
2748
|
+
}
|
|
2749
|
+
cursor = iv.end;
|
|
1633
2750
|
}
|
|
1634
2751
|
parts.push(text.slice(cursor));
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
}
|
|
1639
|
-
function hasRawFoldedBlank(text) {
|
|
1640
|
-
for (let k = 0; k < text.length; k++)
|
|
1641
|
-
if (isFoldedBlank(text.charCodeAt(k)))
|
|
1642
|
-
return true;
|
|
1643
|
-
return false;
|
|
2752
|
+
if (origin !== null)
|
|
2753
|
+
for (let x = cursor; x < text.length; x++)
|
|
2754
|
+
origin.push(x);
|
|
2755
|
+
return origin === null ? { text: parts.join('') } : { text: parts.join(''), origin: Int32Array.from(origin) };
|
|
1644
2756
|
}
|
|
1645
|
-
function
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
2757
|
+
function toRaw(S, c) {
|
|
2758
|
+
let a = c.a;
|
|
2759
|
+
let b = c.b;
|
|
2760
|
+
while (a < b && S.dropped(a))
|
|
2761
|
+
a++;
|
|
2762
|
+
while (b > a && S.dropped(b - 1))
|
|
2763
|
+
b--;
|
|
2764
|
+
if (a >= b)
|
|
2765
|
+
return null;
|
|
2766
|
+
const te = c.te > b ? S.rawEnd(c.te - 1) : S.rawEnd(b - 1);
|
|
2767
|
+
return { start: S.rawStart(a), end: S.rawEnd(b - 1), kind: c.kind, cls: c.cls, tokenEnd: te };
|
|
2768
|
+
}
|
|
2769
|
+
function collectCandidates(text, o, nested) {
|
|
2770
|
+
const V = buildScanView(text, o);
|
|
2771
|
+
const separated = separatedSurface(text, V);
|
|
2772
|
+
const surfaces = [separated];
|
|
2773
|
+
const joined = joinedSurface(text, V);
|
|
2774
|
+
if (joined !== null)
|
|
2775
|
+
surfaces.push(joined);
|
|
2776
|
+
const joinedSpaced = o.escapeSpaces !== null ? joinedSurface(text, V, true) : null;
|
|
2777
|
+
if (joinedSpaced !== null)
|
|
2778
|
+
surfaces.push(joinedSpaced);
|
|
2779
|
+
const screen = screenSurfaces(text, o);
|
|
2780
|
+
surfaces.push(...screen.surfaces);
|
|
2781
|
+
const raw = [];
|
|
2782
|
+
const regionSurface = screen.surfaces[0] ?? separated;
|
|
2783
|
+
const pemBlocksSeen = [];
|
|
2784
|
+
for (const S of surfaces) {
|
|
2785
|
+
const cands = [];
|
|
2786
|
+
const regions = [];
|
|
2787
|
+
if (o.urls) {
|
|
2788
|
+
const unitPos = (k) => S.dropped(k) || V.rawUnit[S.rawStart(k)] === 1;
|
|
2789
|
+
urlCandidates(S.text, cands, regions, unitPos, (k) => unitPos(k) || S.painted(k));
|
|
2790
|
+
userinfoCandidates(S, cands, o.words);
|
|
2791
|
+
}
|
|
2792
|
+
if (o.words) {
|
|
2793
|
+
editsToCandidates(pemEdits(S, (blk) => { if (S === regionSurface)
|
|
2794
|
+
pemBlocksSeen.push(pemBlockToRaw(S, blk)); }), 'pem', cands);
|
|
2795
|
+
labelCandidates(S, 'scheme', regions, cands);
|
|
2796
|
+
labelCandidates(S, 'token', regions, cands);
|
|
2797
|
+
labelCandidates(S, 'label', regions, cands);
|
|
2798
|
+
editsToCandidates(authParamEdits(S.text), 'param', cands);
|
|
2799
|
+
editsToCandidates(proseLiteralEdits(S.text), 'prose', cands);
|
|
2800
|
+
}
|
|
2801
|
+
for (const c of cands) {
|
|
2802
|
+
const r = toRaw(S, c);
|
|
2803
|
+
if (r !== null)
|
|
2804
|
+
raw.push(S.rawDot !== null ? { ...r, dotted: true } : r);
|
|
2805
|
+
}
|
|
1652
2806
|
}
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
const letteredWashed = lettered === null ? { text: start } : washThroughView(start, lettered, urls, words, trace, literal);
|
|
1660
|
-
const pre = letteredWashed.text;
|
|
1661
|
-
let origin = composeOrigin(input.origin, letteredWashed.origin);
|
|
1662
|
-
const separated = buildCredentialView(pre, true, false, blankRead);
|
|
1663
|
-
if (separated === null) {
|
|
1664
|
-
const target = new PlainTarget(pre, trace);
|
|
1665
|
-
runCredentialNets(target, urls, words, literal);
|
|
1666
|
-
return traced(target.text, composeOrigin(origin, target.origin));
|
|
1667
|
-
}
|
|
1668
|
-
const afterSeparated = washThroughView(pre, separated, urls, words, trace, literal);
|
|
1669
|
-
origin = composeOrigin(origin, afterSeparated.origin);
|
|
1670
|
-
const joined = buildCredentialView(afterSeparated.text, false, false, blankRead);
|
|
1671
|
-
if (joined === null)
|
|
1672
|
-
return traced(afterSeparated.text, origin);
|
|
1673
|
-
const afterJoined = washThroughView(afterSeparated.text, joined, urls, words, trace, literal);
|
|
1674
|
-
return traced(afterJoined.text, composeOrigin(origin, afterJoined.origin));
|
|
1675
|
-
}
|
|
1676
|
-
function washControlStringContents(text, urls, words, blankTokens, trace) {
|
|
1677
|
-
if (!/[\u001b\u0090\u0098\u009d-\u009f]|\\u/.test(text))
|
|
1678
|
-
return { text };
|
|
1679
|
-
const ends = new SequenceReader(text, 'none');
|
|
1680
|
-
const parts = [];
|
|
1681
|
-
const origin = trace ? [] : null;
|
|
1682
|
-
const keptUpTo = (a, b) => {
|
|
1683
|
-
if (origin !== null)
|
|
1684
|
-
for (let x = a; x < b; x++)
|
|
1685
|
-
origin.push(x);
|
|
1686
|
-
};
|
|
1687
|
-
let cursor = 0;
|
|
1688
|
-
let i = 0;
|
|
1689
|
-
while (i < text.length) {
|
|
1690
|
-
const c = text.charCodeAt(i);
|
|
1691
|
-
let unit = null;
|
|
1692
|
-
let next = i + 1;
|
|
1693
|
-
if (c === 0x1b)
|
|
1694
|
-
unit = escapeSequenceAfterEsc(text, i + 1, ends);
|
|
1695
|
-
else if (c >= 0x90 && c <= 0x9f)
|
|
1696
|
-
unit = c1SequenceAfter(text, c, i + 1, ends);
|
|
1697
|
-
else if (c === 0x5c && text.charCodeAt(i + 1) === 0x75) {
|
|
1698
|
-
const tok = escapeTokenAt(text, i);
|
|
1699
|
-
if (tok !== null) {
|
|
1700
|
-
next = tok.end;
|
|
1701
|
-
unit = tok.cp === 0x1b ? escapeSequenceAfterEsc(text, tok.end, ends) : tok.cp >= 0x90 && tok.cp <= 0x9f ? c1SequenceAfter(text, tok.cp, tok.end, ends) : null;
|
|
2807
|
+
if (!nested) {
|
|
2808
|
+
for (const [a, b] of V.contents) {
|
|
2809
|
+
const sub = text.slice(a, b);
|
|
2810
|
+
const inner = collectCandidates(sub, o, true);
|
|
2811
|
+
for (const iv of resolveCandidates(sub, inner.cands, inner.rawUnit, inner.rawDot, o.words ? pemRegions(inner.pemBlocksSeen) : [], inner.rawSoft, inner.rawPlain)) {
|
|
2812
|
+
raw.push({ start: a + iv.start, end: a + iv.end, kind: iv.kind, cls: 'nested', tokenEnd: a + iv.end, pretrimmed: true });
|
|
1702
2813
|
}
|
|
1703
2814
|
}
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
const washed = washNets(inner, urls, words, true, blankTokens, trace);
|
|
1712
|
-
if (washed.text !== inner) {
|
|
1713
|
-
parts.push(text.slice(cursor, a), washed.text);
|
|
1714
|
-
keptUpTo(cursor, a);
|
|
1715
|
-
if (origin !== null) {
|
|
1716
|
-
const w = washed.origin;
|
|
1717
|
-
for (let x = 0; x < washed.text.length; x++) {
|
|
1718
|
-
const s = w === undefined ? x : (w[x] ?? -1);
|
|
1719
|
-
origin.push(s < 0 ? -1 : a + s);
|
|
1720
|
-
}
|
|
1721
|
-
}
|
|
1722
|
-
cursor = b;
|
|
2815
|
+
for (const c of screen.contents) {
|
|
2816
|
+
const inner = collectCandidates(c.sub, o, true);
|
|
2817
|
+
for (const iv of resolveCandidates(c.sub, inner.cands, inner.rawUnit, inner.rawDot, o.words ? pemRegions(inner.pemBlocksSeen) : [], inner.rawSoft, inner.rawPlain)) {
|
|
2818
|
+
const start = c.rawAt(iv.start);
|
|
2819
|
+
const end = c.rawEndAt(iv.end - 1);
|
|
2820
|
+
if (end > start)
|
|
2821
|
+
raw.push({ start, end, kind: iv.kind, cls: 'nested', tokenEnd: end, pretrimmed: true });
|
|
1723
2822
|
}
|
|
1724
2823
|
}
|
|
1725
|
-
i = Math.max(unit.end, i + 1);
|
|
1726
2824
|
}
|
|
1727
|
-
|
|
1728
|
-
return { text };
|
|
1729
|
-
parts.push(text.slice(cursor));
|
|
1730
|
-
keptUpTo(cursor, text.length);
|
|
1731
|
-
const joined = parts.join('');
|
|
1732
|
-
return origin === null ? { text: joined } : { text: joined, origin: Int32Array.from(origin) };
|
|
2825
|
+
return { cands: raw, rawUnit: V.rawUnit, rawDot: screen.surfaces[0]?.rawDot ?? null, rawSoft: screen.rawSoft, rawPlain: screen.rawPlain, pemBlocksSeen };
|
|
1733
2826
|
}
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
2827
|
+
function scanOnce(text, o, trace) {
|
|
2828
|
+
const { cands, rawUnit, rawDot, rawSoft, rawPlain, pemBlocksSeen } = collectCandidates(text, o, false);
|
|
2829
|
+
const intervals = resolveCandidates(text, cands, rawUnit, rawDot, o.words ? pemRegions(pemBlocksSeen) : [], rawSoft, rawPlain);
|
|
2830
|
+
return materialize(text, intervals, trace);
|
|
2831
|
+
}
|
|
2832
|
+
const MAX_WASH_REWRITES = 8;
|
|
2833
|
+
function composeOrigin(first, second) {
|
|
2834
|
+
if (second === undefined)
|
|
2835
|
+
return first;
|
|
2836
|
+
if (first === undefined)
|
|
2837
|
+
return second;
|
|
2838
|
+
const out = new Int32Array(second.length);
|
|
2839
|
+
for (let k = 0; k < second.length; k++) {
|
|
2840
|
+
const s = second[k] ?? -1;
|
|
2841
|
+
out[k] = s < 0 ? -1 : (first[s] ?? -1);
|
|
1744
2842
|
}
|
|
1745
|
-
return
|
|
2843
|
+
return out;
|
|
1746
2844
|
}
|
|
1747
|
-
function
|
|
1748
|
-
|
|
2845
|
+
function scanAndMask(text, o, trace, stats) {
|
|
2846
|
+
if (!o.urls && !o.words) {
|
|
2847
|
+
if (stats !== undefined) {
|
|
2848
|
+
stats.passes = 0;
|
|
2849
|
+
stats.capped = false;
|
|
2850
|
+
}
|
|
2851
|
+
return { text };
|
|
2852
|
+
}
|
|
2853
|
+
let cur = scanOnce(text, o, trace);
|
|
2854
|
+
let passes = 1;
|
|
2855
|
+
let rewrites = cur.text !== text ? 1 : 0;
|
|
2856
|
+
let changed = rewrites > 0;
|
|
2857
|
+
while (changed && rewrites < MAX_WASH_REWRITES) {
|
|
2858
|
+
const next = scanOnce(cur.text, o, trace);
|
|
2859
|
+
passes++;
|
|
2860
|
+
changed = next.text !== cur.text;
|
|
2861
|
+
if (!changed)
|
|
2862
|
+
break;
|
|
2863
|
+
rewrites++;
|
|
2864
|
+
const origin = trace ? composeOrigin(cur.origin, next.origin) : undefined;
|
|
2865
|
+
cur = origin === undefined ? { text: next.text } : { text: next.text, origin };
|
|
2866
|
+
}
|
|
2867
|
+
if (stats !== undefined) {
|
|
2868
|
+
let capped = false;
|
|
2869
|
+
if (changed) {
|
|
2870
|
+
passes++;
|
|
2871
|
+
capped = scanOnce(cur.text, o, false).text !== cur.text;
|
|
2872
|
+
}
|
|
2873
|
+
stats.passes = passes;
|
|
2874
|
+
stats.capped = capped;
|
|
2875
|
+
}
|
|
2876
|
+
return cur;
|
|
1749
2877
|
}
|
|
2878
|
+
const WASH_OPTIONS = Object.freeze({ urls: true, words: true, blanks: false, fold: false, face: null, escapeSpaces: null });
|
|
1750
2879
|
export function washCredentials(text) {
|
|
1751
2880
|
if (typeof text !== 'string')
|
|
1752
2881
|
return text;
|
|
1753
|
-
return
|
|
2882
|
+
return scanAndMask(text, WASH_OPTIONS, false).text;
|
|
1754
2883
|
}
|
|
1755
2884
|
const C0_ALL = '\\u0000-\\u001F';
|
|
1756
2885
|
const C0_KEEP_LAYOUT = '\\u0000-\\u0008\\u000B-\\u001F';
|
|
@@ -2089,42 +3218,64 @@ function capDisplay(text, max, mark) {
|
|
|
2089
3218
|
}
|
|
2090
3219
|
const MAX_CAP_SETTLE_ROUNDS = 4;
|
|
2091
3220
|
function runDisplaySteps(start, steps) {
|
|
2092
|
-
const
|
|
2093
|
-
|
|
2094
|
-
if (steps.text(current) === steps.text(s))
|
|
2095
|
-
return current;
|
|
2096
|
-
for (let round = 1; round < MAX_SETTLE_ROUNDS; round++) {
|
|
2097
|
-
const next = steps.paint(steps.wash(current));
|
|
2098
|
-
if (steps.text(next) === steps.text(current))
|
|
2099
|
-
return current;
|
|
2100
|
-
current = next;
|
|
2101
|
-
}
|
|
2102
|
-
return current;
|
|
2103
|
-
};
|
|
2104
|
-
const settled = settle(start);
|
|
3221
|
+
const once = (s) => steps.paint(steps.wash(s));
|
|
3222
|
+
const settled = once(start);
|
|
2105
3223
|
let out = steps.cap(settled);
|
|
2106
3224
|
if (steps.text(out) === steps.text(settled))
|
|
2107
3225
|
return out;
|
|
2108
3226
|
for (let round = 0; round < MAX_CAP_SETTLE_ROUNDS; round++) {
|
|
2109
|
-
const again = steps.cap(
|
|
3227
|
+
const again = steps.cap(once(out));
|
|
2110
3228
|
if (steps.text(again) === steps.text(out))
|
|
2111
3229
|
break;
|
|
2112
3230
|
out = again;
|
|
2113
3231
|
}
|
|
2114
3232
|
return out;
|
|
2115
3233
|
}
|
|
3234
|
+
function escapeSpacesFor(o, face) {
|
|
3235
|
+
if (face.mark !== 'escape' || face.fold)
|
|
3236
|
+
return null;
|
|
3237
|
+
const set = new Set();
|
|
3238
|
+
if (o.controls !== false) {
|
|
3239
|
+
for (const c of [0x09, 0x0a, 0x0b, 0x0c, 0x0d])
|
|
3240
|
+
set.add(c);
|
|
3241
|
+
if (o.keepLayout === true) {
|
|
3242
|
+
set.delete(0x09);
|
|
3243
|
+
set.delete(0x0a);
|
|
3244
|
+
}
|
|
3245
|
+
}
|
|
3246
|
+
if (o.bidi !== false) {
|
|
3247
|
+
set.add(0x2028);
|
|
3248
|
+
set.add(0x2029);
|
|
3249
|
+
}
|
|
3250
|
+
return set.size === 0 ? null : set;
|
|
3251
|
+
}
|
|
3252
|
+
function scanOptionsOf(o, face) {
|
|
3253
|
+
return {
|
|
3254
|
+
urls: o.credentialUrls !== false,
|
|
3255
|
+
words: o.credentialWords !== false,
|
|
3256
|
+
blanks: face.keepBlanks === true,
|
|
3257
|
+
fold: face.fold,
|
|
3258
|
+
face,
|
|
3259
|
+
escapeSpaces: escapeSpacesFor(o, face),
|
|
3260
|
+
};
|
|
3261
|
+
}
|
|
3262
|
+
export function credentialWashPasses(text, opts = {}) {
|
|
3263
|
+
const input = typeof text === 'string' ? text : String(text ?? '');
|
|
3264
|
+
const stats = { passes: 0, capped: false };
|
|
3265
|
+
const scan = opts === null ? WASH_OPTIONS : scanOptionsOf(opts, publicFace(opts));
|
|
3266
|
+
scanAndMask(input, scan, false, stats);
|
|
3267
|
+
return { passes: stats.passes, capped: stats.capped };
|
|
3268
|
+
}
|
|
2116
3269
|
function renderUntrusted(text, opts, ledger) {
|
|
2117
3270
|
const input = typeof text === 'string' ? text : String(text ?? '');
|
|
2118
3271
|
const o = opts ?? {};
|
|
2119
|
-
const urls = o.credentialUrls !== false;
|
|
2120
|
-
const words = o.credentialWords !== false;
|
|
2121
3272
|
const face = publicFace(o);
|
|
2122
|
-
const
|
|
3273
|
+
const scan = scanOptionsOf(o, face);
|
|
2123
3274
|
const pairBackslashes = o.escapeBackslashes === true && face.mark !== 'dot' && face.mark !== 'space';
|
|
2124
3275
|
const book = ledger ?? (pairBackslashes ? { cps: [], lens: [] } : null);
|
|
2125
3276
|
if (book === null) {
|
|
2126
3277
|
return runDisplaySteps(input, {
|
|
2127
|
-
wash: (s) =>
|
|
3278
|
+
wash: (s) => scanAndMask(s, scan, false).text,
|
|
2128
3279
|
paint: (s) => applyDisplayFace(s, face),
|
|
2129
3280
|
cap: (s) => capDisplay(s, o.max, face.mark),
|
|
2130
3281
|
text: (s) => s,
|
|
@@ -2132,7 +3283,7 @@ function renderUntrusted(text, opts, ledger) {
|
|
|
2132
3283
|
}
|
|
2133
3284
|
const out = runDisplaySteps({ text: input, ids: new Int32Array(input.length).fill(-1) }, {
|
|
2134
3285
|
wash: (t) => {
|
|
2135
|
-
const r =
|
|
3286
|
+
const r = scanAndMask(t.text, scan, true);
|
|
2136
3287
|
return r.text === t.text ? t : { text: r.text, ids: carryIds(t.ids, r.origin) };
|
|
2137
3288
|
},
|
|
2138
3289
|
paint: (t) => faceTracked(t, face, book),
|