@hue-run/sdk 0.11.2 → 0.11.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +62 -25
- package/README.md +10 -7
- package/dist/cli/eval.js +16 -10
- package/dist/cli/listen.js +21 -2
- package/dist/cli/login.js +2 -1
- package/dist/cli/mcp.d.ts +5 -4
- package/dist/cli/mcp.js +480 -48
- package/dist/environment/world.d.ts +4 -2
- package/dist/environment/world.js +13 -4
- package/dist/evals/checkpoint.d.ts +5 -0
- package/dist/evals/checkpoint.js +17 -1
- package/dist/evals/local-worker.js +9 -5
- package/dist/evals/simulation.js +7 -5
- package/dist/provider-tools.d.ts +3 -1
- package/dist/provider-tools.js +4 -2
- package/dist/tool-definitions.d.ts +14 -3
- package/dist/tool-definitions.js +635 -53
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/tool-definitions.js
CHANGED
|
@@ -63,7 +63,9 @@ function scrubUrl(value, state) {
|
|
|
63
63
|
state.changed = true;
|
|
64
64
|
return changed ? url.toString() : value;
|
|
65
65
|
}
|
|
66
|
-
|
|
66
|
+
/** Whether a field name names a credential: one of `credentialKeys`, or a name ending in `token`,
|
|
67
|
+
* `secret`, `password`, `apikey` or `credential`, ignoring case, `-` and `_`. */
|
|
68
|
+
export function isCredentialKey(key) {
|
|
67
69
|
const normalized = key.toLowerCase().replace(/[-_]/g, "");
|
|
68
70
|
return (credentialKeys.has(normalized) ||
|
|
69
71
|
normalized.endsWith("token") ||
|
|
@@ -72,9 +74,40 @@ function isCredentialKey(key) {
|
|
|
72
74
|
normalized.endsWith("apikey") ||
|
|
73
75
|
normalized.endsWith("credential"));
|
|
74
76
|
}
|
|
75
|
-
/**
|
|
76
|
-
*
|
|
77
|
-
|
|
77
|
+
/** One piece of a value between backslash-escaped quotes (JSON inside a JSON string): a character,
|
|
78
|
+
* or a whole run of backslashes with what it escapes. The run's length decides what it is: after
|
|
79
|
+
* 4m backslashes, three more and the quote are the inner text's escaped quote, one or three more
|
|
80
|
+
* escape another character, two more (or none, after at least four) are escaped backslashes. Each
|
|
81
|
+
* run can be read only one way, so a value that does not close is given up in linear time; 4m + 1
|
|
82
|
+
* backslashes and the quote close it. `line` holds the characters that end a value's line. */
|
|
83
|
+
const escapedUnit = (quote, line = String.raw `\r\n`) => String.raw `[^${quote}\\${line}]|(?:\\\\\\\\)*(?:\\\\\\${quote}|\\\\\\[^${quote}\\${line}]|\\[^${quote}\\${line}]|\\\\(?!\\))|(?:\\\\\\\\)+(?!\\)`;
|
|
84
|
+
/** Not a quoted value whose closing quote opens the next key of the JSON around the URL
|
|
85
|
+
* (`?code=","password":…`): only JSON punctuation lies between its quotes, and a key and `:` or
|
|
86
|
+
* `=` follow. */
|
|
87
|
+
const notJsonBoundary = (quote) => String.raw `(?![\s,:{}[\]]*${quote}[a-zA-Z0-9_.$-]*\\?["']\s*[:=])`;
|
|
88
|
+
/** A value between backslash-escaped quotes (JSON inside a string) in a URL, whole, when its
|
|
89
|
+
* closing quote ends the URL or its query value: whitespace, the string's own quote, `&`, `#` or
|
|
90
|
+
* the end follows. `openEscapedValue` is one whose closing quote is missing, to where its escapes
|
|
91
|
+
* end. */
|
|
92
|
+
const urlEscapedValue = (quote) => String.raw `\\${quote}(?:${escapedUnit(quote, String.raw `<>${"`"}\r\n`)})*(?:\\\\\\\\)*\\${quote}(?=[\s"&#]|$)`;
|
|
93
|
+
const openEscapedValue = (quote) => String.raw `\\${quote}(?:${escapedUnit(quote)})*`;
|
|
94
|
+
/** One piece of a URL after its `://`, which runs to whitespace, a quote, `<>` or a
|
|
95
|
+
* backslash-escaped quote: a run of other characters; a quoted value right after `=`
|
|
96
|
+
* (`?token="…"`), to its closing quote unless that quote opens the next key of the JSON around it;
|
|
97
|
+
* one between backslash-escaped quotes (`?token=\"…\"`) that ends the URL or its query value; one
|
|
98
|
+
* whose quote does not close on its line, to the end of the line whatever it holds; a backslash
|
|
99
|
+
* that opens no escaped quote; or a quote after `=` that opens no value. `scrubTextUrl` replaces
|
|
100
|
+
* the quoted values. Pieces are read one at a time, not by one repeated pattern, so no number of
|
|
101
|
+
* pieces can exhaust a regular expression engine's backtracking stack. */
|
|
102
|
+
const urlPiece = new RegExp(String.raw `[^\s"'<>${"`"}\\]+|(?<==)(?:"${notJsonBoundary('"')}[^"<>${"`"}\r\n]*"|'${notJsonBoundary("'")}[^'<>${"`"}\r\n]*'|${urlEscapedValue('"')}|${urlEscapedValue("'")}|(?:"[^"\r\n]*|'[^'\r\n]*)(?=[\r\n]|$)|${openEscapedValue('"')}\\*(?=[\r\n]|$)|${openEscapedValue("'")}\\*(?=[\r\n]|$))|\\(?!["'])|(?<==)["']`, "y");
|
|
103
|
+
/** Where the URL whose `://` is at `index` ends: `index + 3` when nothing after it is one. */
|
|
104
|
+
function urlEnd(text, index) {
|
|
105
|
+
let end = index + 3;
|
|
106
|
+
urlPiece.lastIndex = end;
|
|
107
|
+
while (urlPiece.test(text))
|
|
108
|
+
end = urlPiece.lastIndex;
|
|
109
|
+
return end;
|
|
110
|
+
}
|
|
78
111
|
const isSchemeLetter = (code) => (code | 0x20) >= 0x61 && (code | 0x20) <= 0x7a;
|
|
79
112
|
/** `a-z`, `0-9`, `+`, `.` and `-`, case-insensitively. */
|
|
80
113
|
const isSchemeCharacter = (code) => isSchemeLetter(code) ||
|
|
@@ -84,8 +117,22 @@ const isSchemeCharacter = (code) => isSchemeLetter(code) ||
|
|
|
84
117
|
code === 0x2d;
|
|
85
118
|
/** Scrubs each URL in free text. Each `://` is found by search and its scheme read back from it:
|
|
86
119
|
* up to 64 scheme characters, starting at a letter, so a longer run before `://` still leaves a
|
|
87
|
-
* URL to scrub and a long run such as `a.a.a…` costs one pass.
|
|
88
|
-
|
|
120
|
+
* URL to scrub and a long run such as `a.a.a…` costs one pass. A URL that runs to the end of a
|
|
121
|
+
* `cut` text may have lost its `@` or `?` there, so it is replaced whole. Each URL it replaces is
|
|
122
|
+
* added to `changes`, and where each URL it does not replace whole lies in the scrubbed text to
|
|
123
|
+
* `kept`. One replaced whole because a span of `closing` runs past its `://` from before it or
|
|
124
|
+
* starts in its host or path (not in its own userinfo), or the userinfo in `nested` of a URL nested
|
|
125
|
+
* in it starts there, is added to `whole`: where its `[redacted]` lies in the
|
|
126
|
+
* scrubbed text, and the URL as it would have been rewritten. A query name overlapping a span of
|
|
127
|
+
* `hidden` is replaced. Each list of spans is sorted by start. */
|
|
128
|
+
function scrubTextUrls(text, state, cut = false, changes = [], kept = [], closing = [], whole = [], nested = [], hidden = []) {
|
|
129
|
+
// The furthest any closing span starting at or before each one reaches.
|
|
130
|
+
const reach = [];
|
|
131
|
+
for (const [, end] of closing)
|
|
132
|
+
reach.push(Math.max(end, reach.at(-1) ?? 0));
|
|
133
|
+
const hiddenReach = [];
|
|
134
|
+
for (const [, end] of hidden)
|
|
135
|
+
hiddenReach.push(Math.max(end, hiddenReach.at(-1) ?? 0));
|
|
89
136
|
let result = "";
|
|
90
137
|
let copied = 0;
|
|
91
138
|
for (let index = text.indexOf("://"); index !== -1; index = text.indexOf("://", index + 1)) {
|
|
@@ -98,47 +145,228 @@ function scrubTextUrls(text, state) {
|
|
|
98
145
|
start++;
|
|
99
146
|
if (start === index)
|
|
100
147
|
continue;
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
if (!
|
|
148
|
+
const end = urlEnd(text, index);
|
|
149
|
+
// A scheme at the end of a cut text may have lost its URL there, so it is replaced whole too.
|
|
150
|
+
if (end === index + 3 && !(cut && end === text.length))
|
|
104
151
|
continue;
|
|
105
|
-
const
|
|
106
|
-
|
|
152
|
+
const url = text.slice(start, end);
|
|
153
|
+
let scrubbed = cut && end === text.length
|
|
154
|
+
? REDACTED
|
|
155
|
+
: scrubTextUrl(url, state, (from, to) => reaches(hidden, hiddenReach, start + from, start + to));
|
|
156
|
+
// A URL rewritten around a pair or scheme credential before its query, where the rewrite can
|
|
157
|
+
// take the key or scheme word apart from the value, is replaced whole instead, as is one
|
|
158
|
+
// around what the text read on its own hid there, which the rewrite keeps: its host and path
|
|
159
|
+
// are kept. A span starting in the URL's own userinfo, which the rewrite drops, is not such a
|
|
160
|
+
// span (`https://x-access-token:…@github.com/…`).
|
|
161
|
+
if (scrubbed !== url && scrubbed !== REDACTED) {
|
|
162
|
+
const before = /[?#]/.exec(url);
|
|
163
|
+
const to = before ? start + before.index : end;
|
|
164
|
+
const [authority, at] = userinfoSpan(text, index, end);
|
|
165
|
+
const host = at > authority ? at + 1 : index + 3;
|
|
166
|
+
if (reaches(closing, reach, index + 3, index + 3) ||
|
|
167
|
+
startsBetween(closing, host, to) ||
|
|
168
|
+
startsBetween(nested, authority + 1, to)) {
|
|
169
|
+
whole.push([result.length + start - copied, scrubbed]);
|
|
170
|
+
scrubbed = REDACTED;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (scrubbed !== url)
|
|
174
|
+
changes.push([start, end, scrubbed.length - url.length]);
|
|
175
|
+
result += text.slice(copied, start);
|
|
176
|
+
if (scrubbed !== REDACTED)
|
|
177
|
+
kept.push([result.length, result.length + scrubbed.length]);
|
|
178
|
+
result += scrubbed;
|
|
107
179
|
copied = end;
|
|
108
180
|
}
|
|
109
181
|
return result + text.slice(copied);
|
|
110
182
|
}
|
|
183
|
+
/** Where the userinfo of the URL whose `://` is at `index`, ending at `end`, lies: after any `/`
|
|
184
|
+
* or `\` that follow `://`, as the parser skips them, to the last `@` before a `/`, `\`, `?` or
|
|
185
|
+
* `#`; an empty span at its start when it has no `@`. */
|
|
186
|
+
function userinfoSpan(text, index, end) {
|
|
187
|
+
let authority = index + 3;
|
|
188
|
+
while (authority < end && (text[authority] === "/" || text[authority] === "\\"))
|
|
189
|
+
authority++;
|
|
190
|
+
const rest = text.slice(authority, end);
|
|
191
|
+
const boundary = rest.search(/[/\\?#]/);
|
|
192
|
+
const at = (boundary === -1 ? rest : rest.slice(0, boundary)).lastIndexOf("@");
|
|
193
|
+
return [authority, at === -1 ? authority : authority + at];
|
|
194
|
+
}
|
|
195
|
+
/** Whether a span of `spans` (sorted by start) starts at `from` or after it and before `to`. */
|
|
196
|
+
function startsBetween(spans, from, to) {
|
|
197
|
+
let low = 0;
|
|
198
|
+
let high = spans.length;
|
|
199
|
+
while (low < high) {
|
|
200
|
+
const middle = (low + high) >> 1;
|
|
201
|
+
if (spans[middle][0] < from)
|
|
202
|
+
low = middle + 1;
|
|
203
|
+
else
|
|
204
|
+
high = middle;
|
|
205
|
+
}
|
|
206
|
+
return low < spans.length && spans[low][0] < to;
|
|
207
|
+
}
|
|
208
|
+
/** Whether a span of `spans` (sorted by start, `reach` their running furthest end) overlaps
|
|
209
|
+
* `from` to `to`. */
|
|
210
|
+
function reaches(spans, reach, from, to) {
|
|
211
|
+
let low = 0;
|
|
212
|
+
let high = spans.length;
|
|
213
|
+
while (low < high) {
|
|
214
|
+
const middle = (low + high) >> 1;
|
|
215
|
+
if (spans[middle][0] < to)
|
|
216
|
+
low = middle + 1;
|
|
217
|
+
else
|
|
218
|
+
high = middle;
|
|
219
|
+
}
|
|
220
|
+
return low > 0 && reach[low - 1] > from;
|
|
221
|
+
}
|
|
111
222
|
/** Schemes WHATWG parses as hierarchical, which both SDKs serialize alike. */
|
|
112
223
|
const specialScheme = /^(?:https?|wss?|ftp):/i;
|
|
113
|
-
/**
|
|
114
|
-
*
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
const
|
|
224
|
+
/** The quoted values of a URL's text, as `urlPiece` reads them, replaced before the URL is parsed,
|
|
225
|
+
* so an `&` or `=` inside one cannot make the rest of it a query name. One whose quote does not
|
|
226
|
+
* close before the URL's end, and, before its first `#`, one between backslash-escaped quotes,
|
|
227
|
+
* which is never part of a URL in the text around it, are replaced wherever they are; any other
|
|
228
|
+
* only in the query, since one in the host or path can hold the `?` that starts the query or make
|
|
229
|
+
* the URL unparseable. */
|
|
230
|
+
const urlOpenValue = /(?<==)(?:"[^"\r\n]*|'[^'\r\n]*)$/;
|
|
231
|
+
const urlEscapedValues = new RegExp(String.raw `(?<==)(?:(${urlEscapedValue('"')}|${urlEscapedValue("'")})|${openEscapedValue('"')}\\*$|${openEscapedValue("'")}\\*$)`, "g");
|
|
232
|
+
const urlQueryValue = new RegExp(String.raw `(?<==)(?:"[^"<>${"`"}\r\n]*"|'[^'<>${"`"}\r\n]*'|\\?["'][^\r\n]*$)`, "g");
|
|
233
|
+
/** Each query name: from the query's `?` or an `&` to its `=`, not from a `?` inside a value. */
|
|
234
|
+
const queryName = /(?<=^\?|&)[^=&#]+/g;
|
|
235
|
+
/** A scheme word, where no letter or digit comes before it or after an escape, before an escape,
|
|
236
|
+
* which may be of any space (`%20`, `%0B`, `%C2%A0`, `\t`, `\\t`, `\u0020`), or `+`, a form's
|
|
237
|
+
* space, as in `&Bearer%20…` or `&%20Basic%2B…`; not before an escaped bracket, as in an
|
|
238
|
+
* array's name (`token%5B%5D`). */
|
|
239
|
+
const escapedScheme = /(?:(?<![a-z0-9])|(?<=%[0-9a-f]{2})|(?<=\\[bfnrt])|(?<=\\u[0-9a-f]{4}))(?:bearer|basic|token)(?:%(?!5b|5d)|\\|\+)/i;
|
|
240
|
+
/** Whether a query name, which would be exported as a name, the text rules never reading it, holds
|
|
241
|
+
* a credential: a `:` (another URL, `?mongodb://u:…@…`, or a pair, `&token:…`), a scheme word
|
|
242
|
+
* before an escape, or a pair or scheme credential the text rules find once `%3A` and
|
|
243
|
+
* `%3D` are read as `:` and `=` (`&auth.token%20…`, `&Authorization%3ABearer%20…`). The URL keeps
|
|
244
|
+
* its extent, so its later values are replaced as ever. */
|
|
245
|
+
function isCredentialName(name) {
|
|
246
|
+
if (name.includes(":") || escapedScheme.test(name))
|
|
247
|
+
return true;
|
|
248
|
+
// A pair needs a `:` or `=`, which a name holds only escaped, and a scheme credential its word,
|
|
249
|
+
// so most names are never read.
|
|
250
|
+
const separated = name.includes("%") ? name.replace(/%3a/gi, ":").replace(/%3d/gi, "=") : name;
|
|
251
|
+
return ((separated !== name && pairSpans(separated).length > 0) ||
|
|
252
|
+
(schemeWord.test(name) && authorizationSpans(name).length > 0));
|
|
253
|
+
}
|
|
254
|
+
const schemeWord = /bearer|basic|token/i;
|
|
255
|
+
/** What any name `isCredentialName` holds has: a `:`, `%3A`, `%3D` or a scheme word. */
|
|
256
|
+
const credentialNameMark = /:|%3[ad]|bearer|basic|token/i;
|
|
257
|
+
/** Maps a position in a text after `edits` (in order) back to the text before them. Positions must
|
|
258
|
+
* be asked in order, so each edit is passed once. */
|
|
259
|
+
function positionBefore(edits) {
|
|
260
|
+
let next = 0;
|
|
261
|
+
let growth = 0;
|
|
262
|
+
return (at) => {
|
|
263
|
+
for (; next < edits.length && edits[next][0] + edits[next][1] <= at; next++)
|
|
264
|
+
growth += edits[next][2];
|
|
265
|
+
return at - growth;
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
/** A URL in free text, its quoted values replaced. One with another scheme, which runtimes parse
|
|
269
|
+
* differently, is replaced whole when it could carry userinfo, a query or a fragment. A query name
|
|
270
|
+
* that holds a credential, or where `hides` says the text around the URL, read as it was before
|
|
271
|
+
* escaped quotes were, hides something, is replaced. */
|
|
272
|
+
function scrubTextUrl(url, state, hides = () => false) {
|
|
273
|
+
if (!specialScheme.test(url))
|
|
274
|
+
return /[@?#]/.test(url) ? REDACTED : url;
|
|
275
|
+
// The first `#` starts the fragment, which is dropped, even inside a quoted value, as the URL
|
|
276
|
+
// parser reads the text; other values are replaced only before it. A closed value between
|
|
277
|
+
// escaped quotes that holds the first `?`, which starts the query as the parser reads it, keeps
|
|
278
|
+
// that `?`, so what follows the value stays in the query.
|
|
279
|
+
const whole = url.replace(urlOpenValue, REDACTED);
|
|
280
|
+
const hash = whole.indexOf("#");
|
|
281
|
+
const fragment = hash === -1 ? "" : whole.slice(hash);
|
|
282
|
+
const head = hash === -1 ? whole : whole.slice(0, hash);
|
|
283
|
+
const question = head.indexOf("?");
|
|
284
|
+
const escaped = [];
|
|
285
|
+
let grown = 0;
|
|
286
|
+
const opened = head.replace(urlEscapedValues, (value, closed, at) => {
|
|
287
|
+
const replaced = closed !== undefined && at < question && question < at + value.length
|
|
288
|
+
? `${REDACTED}?${REDACTED}`
|
|
289
|
+
: REDACTED;
|
|
290
|
+
escaped.push([at + grown, replaced.length, replaced.length - value.length]);
|
|
291
|
+
grown += replaced.length - value.length;
|
|
292
|
+
return replaced;
|
|
293
|
+
});
|
|
294
|
+
const query = opened.indexOf("?");
|
|
295
|
+
if (query === -1)
|
|
296
|
+
return scrubUrl(opened + fragment, state);
|
|
297
|
+
const quoted = [];
|
|
298
|
+
grown = 0;
|
|
299
|
+
const valued = opened.slice(query).replace(urlQueryValue, (value, at) => {
|
|
300
|
+
quoted.push([query + at + grown, REDACTED.length, REDACTED.length - value.length]);
|
|
301
|
+
grown += REDACTED.length - value.length;
|
|
302
|
+
return REDACTED;
|
|
303
|
+
});
|
|
304
|
+
// A name can hold a credential only where the query holds a `:`, `%3A`, `%3D` or a scheme word,
|
|
305
|
+
// and overlap what `hides` covers only where that covers any of the URL, so most queries' names
|
|
306
|
+
// are not read one by one.
|
|
307
|
+
const overlaps = hides(0, url.length);
|
|
308
|
+
if (!overlaps && !credentialNameMark.test(valued))
|
|
309
|
+
return scrubUrl(opened.slice(0, query) + valued + fragment, state);
|
|
310
|
+
const beforeQuoted = positionBefore(quoted);
|
|
311
|
+
const beforeEscaped = positionBefore(escaped);
|
|
312
|
+
const named = valued.replace(queryName, (name, at) => {
|
|
313
|
+
if (isCredentialName(name))
|
|
314
|
+
return REDACTED;
|
|
315
|
+
if (!overlaps)
|
|
316
|
+
return name;
|
|
317
|
+
const from = beforeEscaped(beforeQuoted(query + at));
|
|
318
|
+
return hides(from, from + name.length) ? REDACTED : name;
|
|
319
|
+
});
|
|
320
|
+
return scrubUrl(opened.slice(0, query) + named + fragment, state);
|
|
321
|
+
}
|
|
322
|
+
/** Where a word starts: after a character that is not a word character, or after a JSON escape
|
|
323
|
+
* (`\n`, `\t`, `\u0022`) or `%` escape, which ends in one. */
|
|
324
|
+
const wordStart = String.raw `(?:(?<![a-z0-9_])|(?<=\\[bfnrt])|(?<=\\u[0-9a-f]{4})|(?<=%[0-9a-f]{2}))`;
|
|
325
|
+
/** Hue's API, MCP, world, attempt, simulation, setup, install, invocation, OAuth and other
|
|
326
|
+
* tokens, and OpenAI and Anthropic (`sk-`), Stripe, Slack, Google OAuth, GitHub and GitLab ones. */
|
|
327
|
+
const tokenPrefix = String.raw `(?:hue_(?:sk|mcp|world|attempt|sim|setup|install|inv|at|rt|oauth|ss|vt)_|sk-|[rs]k_(?:live|test)_|xox[abcdeoprs]-|xapp-|ya29\.|gocspx-|gh[opsur]_|github_pat_|glpat-)`;
|
|
328
|
+
/** A credential its own prefix identifies wherever a word starts, `%` escapes included. At the end
|
|
329
|
+
* of a cut text, any length of it is a credential, since the rest may have been cut. The
|
|
330
|
+
* lookahead for a first letter, here and before `bearer`, lets a search skip other characters
|
|
331
|
+
* before testing a word start. */
|
|
332
|
+
const prefixedToken = new RegExp(`(?=[ghrsxy])${wordStart}${tokenPrefix}[a-z0-9_.~+/=%-]{8,}`, "gi");
|
|
333
|
+
const cutPrefixedToken = new RegExp(`(?=[ghrsxy])${wordStart}${tokenPrefix}(?:[a-z0-9_.~+/=%-]{8,}|[a-z0-9_.~+/=%-]*$)`, "gi");
|
|
124
334
|
/** An authorization scheme followed by its credential, as in an `Authorization` header, up to
|
|
125
|
-
* whitespace, a quote, a delimiter or a backslash. The
|
|
126
|
-
* `
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
335
|
+
* whitespace, a quote, a delimiter or a backslash. The space between them may be escaped (`%20`,
|
|
336
|
+
* `\t`), and then an escaped space also ends the credential, which keeps a run of them linear.
|
|
337
|
+
* The credential cannot start with `=` or `:`, so `token = value` and `Token : value` are left to
|
|
338
|
+
* the key-value rule. */
|
|
339
|
+
const credentialStart = String.raw `[^\s"'${"`"}<>=:,;(){}[\]\\]`;
|
|
340
|
+
const credentialRest = String.raw `[^\s"'${"`"}<>,;(){}[\]\\]`;
|
|
341
|
+
const authorizationValue = new RegExp(String.raw `(?=[bt])${wordStart}(bearer|basic|token)(?:(\s+)${credentialStart}${credentialRest}*|((?:\s|%20|%09|\\[nrt]|\\u0020)+)(?!%20|%09)${credentialStart}(?:(?!%20|%09)${credentialRest})*)`, "gi");
|
|
342
|
+
/** The key and separator of a `key=value`, `key: value` or `key => value` pair, the key optionally
|
|
343
|
+
* quoted, with a backslash-escaped quote too (JSON inside a string). A key starts where no key
|
|
344
|
+
* character precedes it, so each word is tried once and a long run stays linear. Only a key whose
|
|
345
|
+
* last letter a credential key's can be (`token`, `apiKey`, `headers`, `basic` …) is read. The
|
|
346
|
+
* value is not consumed, so a pair inside another pair's value (`error: token=…`) is found. */
|
|
347
|
+
const pairKey = /(\\?["']|)(?<![a-z0-9_-])([a-z0-9_-]*[cdlnrsty][-_]*)\1(\s*(?:=>|[:=])\s*)/gi;
|
|
133
348
|
/** A quoted value to its closing quote on the same line, spaces and escaped quotes included, or
|
|
134
349
|
* one between backslash-escaped quotes, double or single. */
|
|
135
|
-
const quotedValue =
|
|
350
|
+
const quotedValue = new RegExp(String.raw `"(?:[^"\\\r\n]|\\[^\r\n])+"|'(?:[^'\\\r\n]|\\[^\r\n])+'|\\"(?:${escapedUnit('"')})+(?:\\\\\\\\)*\\"|\\'(?:${escapedUnit("'")})+(?:\\\\\\\\)*\\'`, "y");
|
|
351
|
+
/** A quoted value whose quote does not close on its line, as when the text was cut inside it:
|
|
352
|
+
* the value runs to the end of the line, or one between backslash-escaped quotes to the quote that
|
|
353
|
+
* ends the string holding it. */
|
|
354
|
+
const openQuotedValue = new RegExp(String.raw `(?:"(?:[^"\\\r\n]|\\[^\r\n])+|'(?:[^'\\\r\n]|\\[^\r\n])+)\\?(?=[\r\n]|$)|\\"(?:${escapedUnit('"')})+\\*(?=["\r\n]|$)|\\'(?:${escapedUnit("'")})+\\*(?=['\r\n]|$)`, "y");
|
|
355
|
+
/** What a `[…]` or `{…}` value's brackets are counted between: a bracket, a string (double or
|
|
356
|
+
* single quotes), which runs to the end of the text when it does not close, one between
|
|
357
|
+
* backslash-escaped quotes, which ends where its escapes do, or another escaped character. */
|
|
358
|
+
const bracketToken = new RegExp(String.raw `[[\]{}]|"(?:[^"\\]|\\[\s\S])*"?|'(?:[^'\\]|\\[\s\S])*'?|\\"(?:${escapedUnit('"', "")})*(?:(?:\\\\\\\\)*\\")?|\\'(?:${escapedUnit("'", "")})*(?:(?:\\\\\\\\)*\\')?|\\[\s\S]`, "g");
|
|
136
359
|
/** An unquoted value, or one whose quote does not close on its line, up to whitespace, a quote or
|
|
137
360
|
* a delimiter; a value already replaced, or a scheme whose credential was, is left alone. */
|
|
138
|
-
const bareValue = /(\\?["']?)(
|
|
361
|
+
const bareValue = /(\\?["']?)(?!(?:\[redacted\]|%5Bredacted%5D)(?![^\s"',;&})\]\\])|(?:bearer|basic|token)\s)(?:\[redacted\](?=[^\s"',;&})\]\\]))?[^\s"',;&})\]]+/iy;
|
|
139
362
|
/** An `Authorization` header's unquoted value: its scheme and the credential after it (`Bot …`,
|
|
140
363
|
* `OAuth1 …`), or a lone credential. One already replaced is left alone. */
|
|
141
|
-
const authorizationBare = /(\\?["']?)(
|
|
364
|
+
const authorizationBare = /(\\?["']?)(?!(?:\[redacted\]|%5Bredacted%5D)(?![^\s"',;})\]\\]))((?:\[redacted\](?=[^\s"',;})\]\\]))?[^\s"',;})\]]+)(?:[ \t]+(?:\[redacted\]|[^\s"',;})\]]+))?/y;
|
|
365
|
+
/** `bareValue` and `authorizationBare` for a value that starts with a placeholder after `=>`,
|
|
366
|
+
* which is read to its end as any other value is, a `[redacted]` it starts with included. */
|
|
367
|
+
const arrowedBareValue = /(\\?["']?)(?:\[redacted\][^\s"',;&})\]]*|[^\s"',;&})\]]+)/iy;
|
|
368
|
+
const arrowedAuthorizationBare = /(\\?["']?)(\[redacted\][^\s"',;})\]]*|[^\s"',;})\]]+)(?:[ \t]+(?:\[redacted\][^\s"',;})\]]*|[^\s"',;})\]]+))?/iy;
|
|
369
|
+
const placeholderStart = /^(?:\[redacted\]|%5Bredacted%5D)/i;
|
|
142
370
|
function normalizedKey(key) {
|
|
143
371
|
return key.toLowerCase().replace(/[-_]/g, "");
|
|
144
372
|
}
|
|
@@ -152,10 +380,39 @@ function isTextCredentialKey(key) {
|
|
|
152
380
|
normalized === "basic");
|
|
153
381
|
}
|
|
154
382
|
/** `API key: …`: a credential named in two words, `key` right after `API`. */
|
|
155
|
-
const apiBefore = /(?:^|[^a-z0-9_])api[ \t]+$/i;
|
|
383
|
+
const apiBefore = /(?:^|[^a-z0-9_]|\\[bfnrt]|\\u[0-9a-f]{4}|%[0-9a-f]{2})api[ \t]+$/i;
|
|
156
384
|
function isApiKeyPhrase(text, keyStart, key) {
|
|
157
385
|
return (key.toLowerCase() === "key" && apiBefore.test(text.slice(Math.max(0, keyStart - 16), keyStart)));
|
|
158
386
|
}
|
|
387
|
+
/** A key that a JSON or `%` escape before it runs into (`\nheaders`, `%20credentials`), without
|
|
388
|
+
* the escape's characters; `undefined` when no escape precedes it. */
|
|
389
|
+
function unescapedKey(text, keyStart, key) {
|
|
390
|
+
const escape = text[keyStart - 1] === "\\"
|
|
391
|
+
? /^(?:[bfnrt]|u[0-9a-f]{4})/i.exec(key)
|
|
392
|
+
: text[keyStart - 1] === "%"
|
|
393
|
+
? /^[0-9a-f]{2}/i.exec(key)
|
|
394
|
+
: null;
|
|
395
|
+
return escape ? key.slice(escape[0].length) : undefined;
|
|
396
|
+
}
|
|
397
|
+
/** Where a `[…]` or `{…}` value that opens at `start` ends: after the bracket that closes it,
|
|
398
|
+
* a closing bracket of another kind and brackets inside strings being part of the value, or at the
|
|
399
|
+
* end of the text when it does not close. */
|
|
400
|
+
function bracketEnd(text, start) {
|
|
401
|
+
const closers = [];
|
|
402
|
+
bracketToken.lastIndex = start;
|
|
403
|
+
for (let token = bracketToken.exec(text); token; token = bracketToken.exec(text)) {
|
|
404
|
+
if (token[0] === "[")
|
|
405
|
+
closers.push("]");
|
|
406
|
+
else if (token[0] === "{")
|
|
407
|
+
closers.push("}");
|
|
408
|
+
else if (token[0] === closers.at(-1)) {
|
|
409
|
+
closers.pop();
|
|
410
|
+
if (!closers.length)
|
|
411
|
+
return bracketToken.lastIndex;
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
return text.length;
|
|
415
|
+
}
|
|
159
416
|
/** Whether a quote, or a backslash-escaped quote, opens at `index`. */
|
|
160
417
|
function opensQuote(text, index) {
|
|
161
418
|
const character = text[index];
|
|
@@ -172,40 +429,60 @@ function pairSpans(text) {
|
|
|
172
429
|
// The last `Authorization` value read: where it starts, where its first word ends and where it
|
|
173
430
|
// ends. A value starting inside that first word ends where it does, so each run is read once.
|
|
174
431
|
let authorization;
|
|
432
|
+
// Where the last `[…]` or `{…}` value ends. One opening inside it closes inside it.
|
|
433
|
+
let bracketed = 0;
|
|
175
434
|
for (const match of text.matchAll(pairKey)) {
|
|
176
435
|
const start = match.index + match[0].length;
|
|
177
436
|
const key = match[2];
|
|
178
437
|
const keyStart = match.index + match[1].length;
|
|
179
|
-
|
|
438
|
+
const unescaped = unescapedKey(text, keyStart, key);
|
|
439
|
+
if (!(isTextCredentialKey(key) ||
|
|
440
|
+
isApiKeyPhrase(text, keyStart, key) ||
|
|
441
|
+
(unescaped !== undefined && isTextCredentialKey(unescaped))))
|
|
180
442
|
continue;
|
|
181
443
|
const nested = start < covered;
|
|
182
444
|
const isAuthorization = normalizedKey(key).endsWith("authorization");
|
|
445
|
+
const bracket = (text[start] === "[" || text[start] === "{") && !text.startsWith("[redacted]", start);
|
|
183
446
|
// A value inside an earlier value ends where that value's run or quote ends, so it can run
|
|
184
|
-
// past it only by opening a quote (which may be the one that closes the earlier value) or
|
|
185
|
-
// for `Authorization`, by the word after its scheme.
|
|
186
|
-
if (nested && !isAuthorization && !opensQuote(text, start))
|
|
447
|
+
// past it only by opening a quote (which may be the one that closes the earlier value) or a
|
|
448
|
+
// bracket outside an earlier bracket or, for `Authorization`, by the word after its scheme.
|
|
449
|
+
if (nested && (bracket ? start < bracketed : !isAuthorization && !opensQuote(text, start)))
|
|
187
450
|
continue;
|
|
451
|
+
// 0.11.0 read `=>` as `=` and the value from the `>`, whatever followed it, so a placeholder
|
|
452
|
+
// right after `=>` is not taken for a value already replaced.
|
|
453
|
+
const arrowed = match[3].includes("=>") && placeholderStart.test(text.slice(start, start + 14));
|
|
188
454
|
quotedValue.lastIndex = start;
|
|
189
455
|
const quoted = quotedValue.exec(text);
|
|
456
|
+
openQuotedValue.lastIndex = start;
|
|
457
|
+
const openQuoted = quoted ? null : openQuotedValue.exec(text);
|
|
190
458
|
let open;
|
|
191
459
|
let end;
|
|
192
460
|
if (quoted) {
|
|
193
461
|
open = quoted[0].startsWith("\\") ? 2 : 1;
|
|
194
462
|
end = start + quoted[0].length;
|
|
195
463
|
}
|
|
464
|
+
else if (openQuoted) {
|
|
465
|
+
open = openQuoted[0].startsWith("\\") ? 2 : 1;
|
|
466
|
+
end = start + openQuoted[0].length;
|
|
467
|
+
}
|
|
468
|
+
else if (bracket) {
|
|
469
|
+
open = 0;
|
|
470
|
+
end = bracketEnd(text, start);
|
|
471
|
+
bracketed = end;
|
|
472
|
+
}
|
|
196
473
|
else if (isAuthorization) {
|
|
197
474
|
if (authorization &&
|
|
198
475
|
start > authorization.from &&
|
|
199
476
|
start < authorization.firstEnd &&
|
|
200
477
|
!opensQuote(text, start) &&
|
|
201
|
-
!text.startsWith("[redacted]", start)
|
|
202
|
-
!text.startsWith("%5Bredacted%5D", start)) {
|
|
478
|
+
!text.startsWith("[redacted]", start)) {
|
|
203
479
|
open = 0;
|
|
204
480
|
end = authorization.end;
|
|
205
481
|
}
|
|
206
482
|
else {
|
|
207
|
-
|
|
208
|
-
|
|
483
|
+
const pattern = arrowed ? arrowedAuthorizationBare : authorizationBare;
|
|
484
|
+
pattern.lastIndex = start;
|
|
485
|
+
const value = pattern.exec(text);
|
|
209
486
|
if (!value)
|
|
210
487
|
continue;
|
|
211
488
|
open = value[1].length;
|
|
@@ -214,8 +491,9 @@ function pairSpans(text) {
|
|
|
214
491
|
}
|
|
215
492
|
}
|
|
216
493
|
else {
|
|
217
|
-
|
|
218
|
-
|
|
494
|
+
const pattern = arrowed ? arrowedBareValue : bareValue;
|
|
495
|
+
pattern.lastIndex = start;
|
|
496
|
+
const value = pattern.exec(text);
|
|
219
497
|
if (!value)
|
|
220
498
|
continue;
|
|
221
499
|
open = value[1].length;
|
|
@@ -229,14 +507,16 @@ function pairSpans(text) {
|
|
|
229
507
|
return spans;
|
|
230
508
|
}
|
|
231
509
|
/** The credential after each authorization scheme. A scheme word can end an earlier credential
|
|
232
|
-
* (`…~bearer SECRET`)
|
|
510
|
+
* (`…~bearer SECRET`) or start inside its escaped separator (`Bearer\token SECRET`, `\t` being the
|
|
511
|
+
* separator), so the search resumes after each scheme word, reading its credential at most once
|
|
512
|
+
* more. */
|
|
233
513
|
function authorizationSpans(text) {
|
|
234
514
|
const spans = [];
|
|
235
515
|
authorizationValue.lastIndex = 0;
|
|
236
516
|
for (let match = authorizationValue.exec(text); match; match = authorizationValue.exec(text)) {
|
|
237
|
-
const credential = match.index + match[1].length + match[2].length;
|
|
517
|
+
const credential = match.index + match[1].length + (match[2] ?? match[3]).length;
|
|
238
518
|
spans.push([credential, match.index + match[0].length]);
|
|
239
|
-
authorizationValue.lastIndex =
|
|
519
|
+
authorizationValue.lastIndex = match.index + match[1].length;
|
|
240
520
|
}
|
|
241
521
|
return spans;
|
|
242
522
|
}
|
|
@@ -276,19 +556,321 @@ function redactSpans(text, spans) {
|
|
|
276
556
|
* becomes `[redacted]` when it has an `@`, `?` or `#`; a token with a known credential prefix
|
|
277
557
|
* (`hue_sk_`, `sk-`, `xoxb-`, `ya29.` and others), the credential after an authorization scheme
|
|
278
558
|
* (`Bearer`, `Basic`, `Token`), the whole value of an `Authorization` header, and the value of a
|
|
279
|
-
* `key=value` or `key
|
|
280
|
-
* become `[redacted]`.
|
|
559
|
+
* `key=value`, `key: value` or `key => value` pair whose key names a credential (quoted,
|
|
560
|
+
* escaped-quoted, bare, or a whole `[…]` or `{…}`) become `[redacted]`. A JSON or `%` escape
|
|
561
|
+
* (`\n`, `\u0022`, `%20`) ends a word as a space does. A quote that does not close on its line
|
|
562
|
+
* runs to the end of the line, and a URL's quoted query value is replaced whole. `cut` says the
|
|
563
|
+
* text was cut from a longer one, so a URL or prefixed token that runs to its end is replaced
|
|
564
|
+
* whole, as is the word of scheme characters it ends in, which may start a URL's scheme. Only the
|
|
565
|
+
* first 16,384 code points are read: a longer text is cut there, and `…` marks it.
|
|
281
566
|
*/
|
|
282
|
-
export function scrubCredentialText(text) {
|
|
283
|
-
|
|
567
|
+
export function scrubCredentialText(text, cut = false) {
|
|
568
|
+
if (text.length > MAX_SCRUBBED_TEXT) {
|
|
569
|
+
let kept = "";
|
|
570
|
+
let seen = 0;
|
|
571
|
+
for (const character of text) {
|
|
572
|
+
if (seen++ === MAX_SCRUBBED_TEXT)
|
|
573
|
+
return `${scrubCredentialTextUnbounded(kept, true)}…`;
|
|
574
|
+
kept += character;
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return scrubCredentialTextUnbounded(text, cut);
|
|
578
|
+
}
|
|
579
|
+
/** The most of a text `scrubCredentialText` reads, in code points, as of a provider's error text:
|
|
580
|
+
* a longer one is cut there and `…` marks the cut, so no caller hands a regular expression engine
|
|
581
|
+
* a text long enough to exhaust it. */
|
|
582
|
+
const MAX_SCRUBBED_TEXT = 16_384;
|
|
583
|
+
/** `scrubCredentialText` without its bound on the text's length, for tests that time long texts;
|
|
584
|
+
* callers use `scrubCredentialText`. */
|
|
585
|
+
export function scrubCredentialTextUnbounded(text, cut = false) {
|
|
586
|
+
const changes = [];
|
|
587
|
+
const kept = [];
|
|
588
|
+
// The text's pairs, schemes and prefixed tokens before its URLs are replaced: a URL can take in
|
|
589
|
+
// a key or scheme whose value follows it (`…&password=\"…`), one rewritten around a credential
|
|
590
|
+
// is replaced whole, and the parser can join a token in a host to the letter before it.
|
|
591
|
+
const credentials = text.includes("://")
|
|
592
|
+
? [...pairSpans(text), ...authorizationSpans(text), ...matchSpans(text, prefixedToken)].sort(([a], [b]) => a - b)
|
|
593
|
+
: [];
|
|
594
|
+
// What the text hides with its URLs read as they were before escaped quotes were: a URL read
|
|
595
|
+
// now can end sooner or later, and neither may export what that reading hid. A query name
|
|
596
|
+
// overlaps a query value or fragment that reading hid only in a URL starting inside it, whose
|
|
597
|
+
// host or path it then reaches, so those close a URL but are not checked against names.
|
|
598
|
+
const plain = text.includes("://")
|
|
599
|
+
? plainUrlSpans(text)
|
|
600
|
+
: { hidden: [], userinfo: [], values: [] };
|
|
601
|
+
const hiding = [...credentials, ...plain.hidden];
|
|
602
|
+
const closing = [...hiding, ...plain.values].sort(([a], [b]) => a - b);
|
|
603
|
+
const whole = [];
|
|
604
|
+
const scrubbed = scrubTextUrls(text, { changed: false }, cut, changes, kept, closing, whole, plain.userinfo,
|
|
605
|
+
// A credential starting in a URL's own userinfo, which the rewrite drops with its key, hides
|
|
606
|
+
// neither the names after it nor, below, the text after the URL.
|
|
607
|
+
[...outsideUserinfo(credentials, plain.userinfo), ...plain.hidden, ...plain.userinfo].sort(([a], [b]) => a - b));
|
|
608
|
+
const net = [...plain.hidden, ...plain.userinfo, ...plain.values].sort(([a], [b]) => a - b);
|
|
284
609
|
// Every rule reads the same text and their matches are replaced together, so no rule's
|
|
285
|
-
// replacement can hide text another rule would have matched.
|
|
610
|
+
// replacement can hide text another rule would have matched. The pairs and schemes of the text
|
|
611
|
+
// with each URL replaced whole rewritten instead are read too, so what a value that runs out of
|
|
612
|
+
// such a URL reaches is replaced as it was when the URL was rewritten.
|
|
286
613
|
return redactSpans(scrubbed, [
|
|
287
614
|
...pairSpans(scrubbed),
|
|
288
|
-
...matchSpans(scrubbed, prefixedToken),
|
|
615
|
+
...matchSpans(scrubbed, cut ? cutPrefixedToken : prefixedToken),
|
|
289
616
|
...authorizationSpans(scrubbed),
|
|
617
|
+
...nestedUserinfoSpans(scrubbed, kept),
|
|
618
|
+
...(changes.length || net.length
|
|
619
|
+
? spansBesideUrls([...outsideUserinfo(credentials, plain.userinfo), ...outsideUrls(text, net, changes)], changes)
|
|
620
|
+
: []),
|
|
621
|
+
...(whole.length ? rewrittenSpans(scrubbed, whole) : []),
|
|
622
|
+
...(cut ? cutSchemeSpans(scrubbed) : []),
|
|
290
623
|
]);
|
|
291
624
|
}
|
|
625
|
+
/** Where a cut text ends inside a scheme or its `://`, the rest of the URL cut off: the scheme
|
|
626
|
+
* characters it ends in, read back as a scheme is, with a `:` or `:/` after them. */
|
|
627
|
+
function cutSchemeSpans(text) {
|
|
628
|
+
const index = text.length - (text.endsWith(":/") ? 2 : text.endsWith(":") ? 1 : 0);
|
|
629
|
+
let start = index;
|
|
630
|
+
while (start > 0 && index - start < 64 && isSchemeCharacter(text.charCodeAt(start - 1)))
|
|
631
|
+
start--;
|
|
632
|
+
while (start < index && !isSchemeLetter(text.charCodeAt(start)))
|
|
633
|
+
start++;
|
|
634
|
+
return start < index ? [[start, text.length]] : [];
|
|
635
|
+
}
|
|
636
|
+
/** One piece of a URL as read before escaped quotes were: a run of characters that are not
|
|
637
|
+
* whitespace, a quote or `<>`, a closed quoted value right after `=`, or a quote after `=`. */
|
|
638
|
+
const plainUrlPiece = /[^\s"'<>`]+|(?<==)"[^"<>`\r\n]*"|(?<==)'[^'<>`\r\n]*'|(?<==)["']/y;
|
|
639
|
+
/** What the text hides with each URL read as before escaped quotes were, where it lies in the
|
|
640
|
+
* text, each list sorted by start. `hidden` is all of a URL with another scheme holding `@`, `?`
|
|
641
|
+
* or `#`, or of one the parser refuses, and each pair, scheme or prefixed-token credential of the
|
|
642
|
+
* text with its URLs replaced as that reading replaces them, one starting or ending in a URL
|
|
643
|
+
* starting or ending with it. `userinfo` is each
|
|
644
|
+
* other URL's userinfo, after any `/` or `\` that follow its `://`, as the parser skips them, and
|
|
645
|
+
* `values` its query values and fragment. */
|
|
646
|
+
function plainUrlSpans(text) {
|
|
647
|
+
const spans = [];
|
|
648
|
+
const userinfo = [];
|
|
649
|
+
const values = [];
|
|
650
|
+
const changes = [];
|
|
651
|
+
let scrubbed = "";
|
|
652
|
+
let copied = 0;
|
|
653
|
+
for (let index = text.indexOf("://"); index !== -1; index = text.indexOf("://", index + 1)) {
|
|
654
|
+
if (index < copied)
|
|
655
|
+
continue;
|
|
656
|
+
let start = index;
|
|
657
|
+
while (start > copied && index - start < 64 && isSchemeCharacter(text.charCodeAt(start - 1)))
|
|
658
|
+
start--;
|
|
659
|
+
while (start < index && !isSchemeLetter(text.charCodeAt(start)))
|
|
660
|
+
start++;
|
|
661
|
+
if (start === index)
|
|
662
|
+
continue;
|
|
663
|
+
let end = index + 3;
|
|
664
|
+
plainUrlPiece.lastIndex = end;
|
|
665
|
+
while (plainUrlPiece.test(text))
|
|
666
|
+
end = plainUrlPiece.lastIndex;
|
|
667
|
+
if (end === index + 3)
|
|
668
|
+
continue;
|
|
669
|
+
const url = text.slice(start, end);
|
|
670
|
+
const replaced = specialScheme.test(url)
|
|
671
|
+
? scrubUrl(url, { changed: false })
|
|
672
|
+
: /[@?#]/.test(url)
|
|
673
|
+
? REDACTED
|
|
674
|
+
: url;
|
|
675
|
+
scrubbed += text.slice(copied, start) + replaced;
|
|
676
|
+
copied = end;
|
|
677
|
+
if (replaced !== url)
|
|
678
|
+
changes.push([start, end, replaced.length - url.length]);
|
|
679
|
+
if (replaced === REDACTED) {
|
|
680
|
+
spans.push([start, end]);
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
const [authority, at] = userinfoSpan(text, index, end);
|
|
684
|
+
if (at > authority)
|
|
685
|
+
userinfo.push([authority, at]);
|
|
686
|
+
const hash = url.indexOf("#");
|
|
687
|
+
const tail = hash === -1 ? url.length : hash;
|
|
688
|
+
const question = url.indexOf("?");
|
|
689
|
+
if (question !== -1 && question < tail) {
|
|
690
|
+
let part = start + question + 1;
|
|
691
|
+
for (const piece of url.slice(question + 1, tail).split("&")) {
|
|
692
|
+
const equals = piece.indexOf("=");
|
|
693
|
+
if (equals !== -1 && equals + 1 < piece.length)
|
|
694
|
+
values.push([part + equals + 1, part + piece.length]);
|
|
695
|
+
part += piece.length + 1;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
if (hash !== -1)
|
|
699
|
+
values.push([start + hash, end]);
|
|
700
|
+
}
|
|
701
|
+
if (!changes.length)
|
|
702
|
+
return { hidden: spans, userinfo, values };
|
|
703
|
+
scrubbed += text.slice(copied);
|
|
704
|
+
const hidden = [
|
|
705
|
+
...spans,
|
|
706
|
+
...spansBeforeUrls([
|
|
707
|
+
...pairSpans(scrubbed),
|
|
708
|
+
...authorizationSpans(scrubbed),
|
|
709
|
+
...matchSpans(scrubbed, prefixedToken),
|
|
710
|
+
], changes),
|
|
711
|
+
].sort(([a], [b]) => a - b);
|
|
712
|
+
return { hidden, userinfo, values };
|
|
713
|
+
}
|
|
714
|
+
/** The spans of `spans` (sorted by start) that do not start inside a span of `userinfo` (sorted,
|
|
715
|
+
* disjoint). */
|
|
716
|
+
function outsideUserinfo(spans, userinfo) {
|
|
717
|
+
let next = 0;
|
|
718
|
+
return spans.filter(([start]) => {
|
|
719
|
+
while (next < userinfo.length && userinfo[next][1] <= start)
|
|
720
|
+
next++;
|
|
721
|
+
return !(next < userinfo.length && userinfo[next][0] <= start);
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
/** The spans of a text whose URLs `changes` replaced, moved back to where their text lies before:
|
|
725
|
+
* one starting or ending inside a replacement starts or ends with its URL. */
|
|
726
|
+
function spansBeforeUrls(spans, changes) {
|
|
727
|
+
// Where each replacement starts and ends in the text, and how much longer the text is after it.
|
|
728
|
+
const starts = [];
|
|
729
|
+
const ends = [];
|
|
730
|
+
const growth = [];
|
|
731
|
+
let total = 0;
|
|
732
|
+
for (const [start, end, grown] of changes) {
|
|
733
|
+
starts.push(start + total);
|
|
734
|
+
growth.push((total += grown));
|
|
735
|
+
ends.push(end + total);
|
|
736
|
+
}
|
|
737
|
+
// How many replacements start before `position`, or at it when `at` holds.
|
|
738
|
+
const count = (position, at) => {
|
|
739
|
+
let low = 0;
|
|
740
|
+
let high = starts.length;
|
|
741
|
+
while (low < high) {
|
|
742
|
+
const middle = (low + high) >> 1;
|
|
743
|
+
if (starts[middle] < position || (at && starts[middle] === position))
|
|
744
|
+
low = middle + 1;
|
|
745
|
+
else
|
|
746
|
+
high = middle;
|
|
747
|
+
}
|
|
748
|
+
return low;
|
|
749
|
+
};
|
|
750
|
+
return spans.map(([start, end]) => {
|
|
751
|
+
const first = count(start, true) - 1;
|
|
752
|
+
const final = count(end, false) - 1;
|
|
753
|
+
const from = first === -1 ? start : start < ends[first] ? changes[first][0] : start - growth[first];
|
|
754
|
+
const to = final === -1 ? end : end <= ends[final] ? changes[final][1] : end - growth[final];
|
|
755
|
+
return [from, to];
|
|
756
|
+
});
|
|
757
|
+
}
|
|
758
|
+
/** The parts of `spans` outside every URL `found` replaced (both sorted by start) that hold a
|
|
759
|
+
* letter or digit. */
|
|
760
|
+
function outsideUrls(text, spans, found) {
|
|
761
|
+
const parts = [];
|
|
762
|
+
let first = 0;
|
|
763
|
+
for (const [from, to] of spans) {
|
|
764
|
+
while (first < found.length && found[first][1] <= from)
|
|
765
|
+
first++;
|
|
766
|
+
let cursor = from;
|
|
767
|
+
for (let next = first; next < found.length && found[next][0] < to; next++) {
|
|
768
|
+
if (found[next][0] > cursor)
|
|
769
|
+
parts.push([cursor, found[next][0]]);
|
|
770
|
+
cursor = Math.max(cursor, found[next][1]);
|
|
771
|
+
}
|
|
772
|
+
if (cursor < to)
|
|
773
|
+
parts.push([cursor, to]);
|
|
774
|
+
}
|
|
775
|
+
return parts.filter(([start, end]) => /[a-z0-9]/i.test(text.slice(start, end)));
|
|
776
|
+
}
|
|
777
|
+
/** The pair and scheme spans of the text with each URL replaced whole rewritten instead, cut to
|
|
778
|
+
* the text outside those URLs and moved to where it is in `text`. */
|
|
779
|
+
function rewrittenSpans(text, whole) {
|
|
780
|
+
let rewritten = "";
|
|
781
|
+
let copied = 0;
|
|
782
|
+
const changes = [];
|
|
783
|
+
for (const [at, url] of whole) {
|
|
784
|
+
rewritten += text.slice(copied, at);
|
|
785
|
+
changes.push([rewritten.length, rewritten.length + url.length, REDACTED.length - url.length]);
|
|
786
|
+
rewritten += url;
|
|
787
|
+
copied = at + REDACTED.length;
|
|
788
|
+
}
|
|
789
|
+
rewritten += text.slice(copied);
|
|
790
|
+
return spansBesideUrls([...pairSpans(rewritten), ...authorizationSpans(rewritten)], changes);
|
|
791
|
+
}
|
|
792
|
+
/** A URL nested in another's path (`…/p&mongodb://u:…@…`): its `://` (or `:\\`) and any more `/`
|
|
793
|
+
* or `\`, then its userinfo to the last `@` before a `/`, `\`, `?`, `#` or whitespace, as a URL's
|
|
794
|
+
* own userinfo is read. */
|
|
795
|
+
const nestedUserinfo = /:[/\\]{2}[/\\]*[^\s/\\?#]*@/g;
|
|
796
|
+
/** The userinfo of each URL nested in the path of a URL not replaced whole, and the whole of one
|
|
797
|
+
* with another scheme that holds an `@`: the path is kept and the nested URL is not read on its
|
|
798
|
+
* own. */
|
|
799
|
+
function nestedUserinfoSpans(text, urls) {
|
|
800
|
+
const spans = [];
|
|
801
|
+
for (const [start, end] of urls) {
|
|
802
|
+
const url = text.slice(start, end);
|
|
803
|
+
const authority = url.indexOf("://") + 3;
|
|
804
|
+
const boundary = url.slice(authority).search(/[?#]/);
|
|
805
|
+
const pathEnd = boundary === -1 ? url.length : authority + boundary;
|
|
806
|
+
const slash = url.slice(authority, pathEnd).search(/[/\\]/);
|
|
807
|
+
if (slash === -1)
|
|
808
|
+
continue;
|
|
809
|
+
const pathStart = authority + slash;
|
|
810
|
+
for (const match of url.slice(pathStart, pathEnd).matchAll(nestedUserinfo)) {
|
|
811
|
+
const from = start + pathStart + match.index + 3;
|
|
812
|
+
const to = start + pathStart + match.index + match[0].length - 1;
|
|
813
|
+
if (from < to)
|
|
814
|
+
spans.push([from, to]);
|
|
815
|
+
}
|
|
816
|
+
// A nested URL with another scheme and an `@` is replaced whole, to the path's end, as it is
|
|
817
|
+
// when read on its own.
|
|
818
|
+
const lastAt = url.lastIndexOf("@", pathEnd - 1);
|
|
819
|
+
for (const match of url.slice(pathStart, pathEnd).matchAll(/:[/\\]{2}/g)) {
|
|
820
|
+
const colon = pathStart + match.index;
|
|
821
|
+
if (lastAt < colon)
|
|
822
|
+
break;
|
|
823
|
+
let schemeStart = colon;
|
|
824
|
+
while (schemeStart > pathStart &&
|
|
825
|
+
colon - schemeStart < 64 &&
|
|
826
|
+
isSchemeCharacter(url.charCodeAt(schemeStart - 1)))
|
|
827
|
+
schemeStart--;
|
|
828
|
+
while (schemeStart < colon && !isSchemeLetter(url.charCodeAt(schemeStart)))
|
|
829
|
+
schemeStart++;
|
|
830
|
+
if (schemeStart === colon || specialScheme.test(`${url.slice(schemeStart, colon)}:`))
|
|
831
|
+
continue;
|
|
832
|
+
spans.push([start + schemeStart, start + pathEnd]);
|
|
833
|
+
break;
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
return spans;
|
|
837
|
+
}
|
|
838
|
+
/** The spans moved to where their text is once the URLs are replaced: one starting inside a
|
|
839
|
+
* replaced URL starts after it, and one ending inside one ends before it. */
|
|
840
|
+
function spansBesideUrls(spans, changes) {
|
|
841
|
+
// How much longer the text is after each replacement.
|
|
842
|
+
const growth = [];
|
|
843
|
+
let total = 0;
|
|
844
|
+
for (const change of changes)
|
|
845
|
+
growth.push((total += change[2]));
|
|
846
|
+
// The last replaced URL starting before `position`, or -1.
|
|
847
|
+
const last = (position) => {
|
|
848
|
+
let low = 0;
|
|
849
|
+
let high = changes.length;
|
|
850
|
+
while (low < high) {
|
|
851
|
+
const middle = (low + high) >> 1;
|
|
852
|
+
if (changes[middle][0] < position)
|
|
853
|
+
low = middle + 1;
|
|
854
|
+
else
|
|
855
|
+
high = middle;
|
|
856
|
+
}
|
|
857
|
+
return low - 1;
|
|
858
|
+
};
|
|
859
|
+
const moved = [];
|
|
860
|
+
for (const [start, end] of spans) {
|
|
861
|
+
const first = last(start + 1);
|
|
862
|
+
const from = first === -1 ? start : Math.max(start, changes[first][1]) + growth[first];
|
|
863
|
+
const final = last(end);
|
|
864
|
+
const to = final === -1
|
|
865
|
+
? end
|
|
866
|
+
: end < changes[final][1]
|
|
867
|
+
? changes[final][0] + (growth[final - 1] ?? 0)
|
|
868
|
+
: end + growth[final];
|
|
869
|
+
if (from < to)
|
|
870
|
+
moved.push([from, to]);
|
|
871
|
+
}
|
|
872
|
+
return moved;
|
|
873
|
+
}
|
|
292
874
|
/** OpenInference records each tool as `llm.tools.{index}.tool.json_schema`. */
|
|
293
875
|
const openInferenceTool = /^llm\.tools\.(\d+)\.tool\.json_schema$/;
|
|
294
876
|
/**
|