@sema-agent/primitives 7.66.1 → 7.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -1
- package/README.md +7 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/secret-scan.d.ts +24 -3
- package/dist/secret-scan.js +62 -42
- package/dist/secret-vocabulary.d.ts +255 -10
- package/dist/secret-vocabulary.js +172 -89
- package/package.json +2 -2
package/LICENSE
CHANGED
|
@@ -5,7 +5,8 @@ Parameters
|
|
|
5
5
|
Licensor: clay (github.com/clayboby)
|
|
6
6
|
Licensed Work: @sema-agent/core (sema-core) and the satellite packages
|
|
7
7
|
published from this repository (currently
|
|
8
|
-
@sema-agent/mods-kernel, @sema-agent/primitives
|
|
8
|
+
@sema-agent/mods-kernel, @sema-agent/primitives,
|
|
9
|
+
@sema-agent/core-stores-sql and @sema-agent/wiki-gate)
|
|
9
10
|
(formerly published as @sema-ai/core)
|
|
10
11
|
The Licensed Work is (c) 2026 clay.
|
|
11
12
|
Additional Use Grant: You may make production use of the Licensed Work for
|
package/README.md
CHANGED
|
@@ -119,6 +119,7 @@ interface SecretNotation {
|
|
|
119
119
|
| --- | --- |
|
|
120
120
|
| `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved` (the percent layers ran out, under `onUnresolvedReading: "secret"`), `mapping-failed`; a hit in a derived text is reported over the written units it was read from |
|
|
121
121
|
| `isSecretShaped(text, policy)` | `scanSecrets` found a span or answered the whole text |
|
|
122
|
+
| `userinfoHolds(userinfo)` | a URL userinfo (no trailing `@`) holds a user or a password — an empty one and the bare separator `:` hold neither (the reading the `keep-username` rendering withholds by) |
|
|
122
123
|
| `renderSecrets(text, policy, notation, { passes?, report? })` | `text` with every hit replaced by its mark; `passes` are the caller's own rules, run after the table's in the same pipeline; `report.findings` receives one finding per hit (`report.preexistingMarkers` is the caller's to set) |
|
|
123
124
|
| `isSensitiveRecordKey(key)` / `isSensitiveHeaderName(name)` | a record member / an HTTP header name names a credential |
|
|
124
125
|
| `envNameView()` | the readings of environment variable names (exact names, credential words, a namespace's names `anySegmentUnder(namespace)`) |
|
|
@@ -133,6 +134,12 @@ notation can tell. `URL_OF_ANY_SCHEME` is the URL grammar (a global regex: read
|
|
|
133
134
|
layer that withholds whole URLs. `CREDENTIAL_MIN_CHARS` is the fewest characters a string can have and still be read as
|
|
134
135
|
a credential.
|
|
135
136
|
|
|
137
|
+
Every count in the table follows one character or one character class and is closed — "`n` or more" is written as `n`
|
|
138
|
+
of them and then a plain loop, and a group's count is written out — so a reading keeps no regex backtrack state per
|
|
139
|
+
character of one value run, and a run of millions of characters is scanned and rendered to its end (pinned at 8 million
|
|
140
|
+
code units) instead of throwing `RangeError`. A credential header value made of millions of `=` / `,` / quoted-parameter
|
|
141
|
+
pieces is still read one group pass per piece and throws from about two million code units on.
|
|
142
|
+
|
|
136
143
|
The table and its rows are frozen; its regexes are not (`replace` writes a global regex's `lastIndex`), and the functions
|
|
137
144
|
here never run them: the package runs its own copies, made as it loads, so a regex a caller recompiles or moves on the
|
|
138
145
|
exported table changes only that caller's reading.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/** The package entry of `@sema-agent/primitives`; core imports the package by name, never these modules. */
|
|
2
2
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, type CredentialRef, type CredentialRefKind, type CredentialRefRequestAdmission, type CredentialRefProblem, type CredentialRefProblemReason, type CredentialRefReading, } from "./credential-ref.js";
|
|
3
|
-
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type CredentialNameRow, type CredentialNamePlace, type EnvNameView, type EnvValueView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
|
|
3
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, SCHEME_FOLLOWER_WORDS, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialFamilyId, type CredentialWordRow, type CredentialWordPosition, type CredentialNameRow, type CredentialNamePlace, type CredentialNameStrength, type EnvNameView, type EnvValueView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
|
|
4
4
|
export { SECRET_READING_KINDS, secretReadingViews, type SecretReading, type SecretReadingKind, type SecretReadingView, type SecretReadingViews, } from "./secret-readings.js";
|
|
5
|
-
export { scanSecrets, isSecretShaped, renderSecrets, type SecretGrade, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
|
|
5
|
+
export { scanSecrets, isSecretShaped, renderSecrets, userinfoHolds, type SecretGrade, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
|
package/dist/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, } from "./credential-ref.js";
|
|
2
|
-
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
|
|
2
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, SCHEME_FOLLOWER_WORDS, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
|
|
3
3
|
export { SECRET_READING_KINDS, secretReadingViews, } from "./secret-readings.js";
|
|
4
|
-
export { scanSecrets, isSecretShaped, renderSecrets, } from "./secret-scan.js";
|
|
4
|
+
export { scanSecrets, isSecretShaped, renderSecrets, userinfoHolds, } from "./secret-scan.js";
|
package/dist/secret-scan.d.ts
CHANGED
|
@@ -12,11 +12,12 @@
|
|
|
12
12
|
* The renderer is one SEQUENTIAL multi-pass engine: each pass scans the text the earlier passes left, so a hit that runs
|
|
13
13
|
* into an earlier marker reads the marker the caller chose. A row whose wide form captures a label keeps it and replaces
|
|
14
14
|
* only the value (`GITHUB_TOKEN=ghp_xxx` keeps `GITHUB_TOKEN=`), and a second run over already-rendered text is a no-op.
|
|
15
|
+
* Hits that touch write one mark, whichever pass made each.
|
|
15
16
|
* Every pass declares what it detects (kind), how sure it is (confidence) and the marker it writes; a report collector,
|
|
16
17
|
* when given, receives one finding per hit in the ORIGINAL input's coordinates, mapped back through the earlier passes'
|
|
17
18
|
* edits (best-effort where redactions overlap — a span landing inside an earlier replacement anchors to that edit's start).
|
|
18
19
|
*/
|
|
19
|
-
import { type RedactionConfidence, type RedactionSource, type ValueFamilyRow } from "./secret-vocabulary.js";
|
|
20
|
+
import { type RedactionConfidence, type RedactionSource, type ValueFamilyRow, type CredentialFamilyId } from "./secret-vocabulary.js";
|
|
20
21
|
import { type ReadingSet, type SecretReading } from "./secret-readings.js";
|
|
21
22
|
/** One redaction hit: what was detected, where (ORIGINAL-input coordinates), how confidently, the marker
|
|
22
23
|
* literal written in its place, which rule table minted it, and — present when a vocabulary row minted it — the
|
|
@@ -60,6 +61,23 @@ export interface RedactionPass {
|
|
|
60
61
|
* matches at the hit's offset in the pass input. Absent = the pass is no vocabulary row. */
|
|
61
62
|
family?: string | ((input: string, at: number) => string);
|
|
62
63
|
}
|
|
64
|
+
/** The `format-free` reading of a text that holds a format character: the text with every `\p{Cf}` removed (the view),
|
|
65
|
+
* where the format characters stood (`cuts`: the view offset of every view character that has format characters right
|
|
66
|
+
* before it, the first character excepted), and the way back to the text's own coordinates. */
|
|
67
|
+
export interface FormatFreeReading {
|
|
68
|
+
readonly view: string;
|
|
69
|
+
readonly cuts: readonly number[];
|
|
70
|
+
/** A view offset (a view character's start) in the text's coordinates; any other offset is the text's end. */
|
|
71
|
+
readonly toOriginal: (pos: number) => number;
|
|
72
|
+
/** A view span in the text's coordinates. Its end is the end of its LAST view character (not the start of the next one),
|
|
73
|
+
* so the format characters right after the span stay outside it. */
|
|
74
|
+
readonly toOriginalSpan: (viewStart: number, viewEnd: number) => {
|
|
75
|
+
start: number;
|
|
76
|
+
end: number;
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** The reading of `input` (a text that holds a format character). Not on the package barrel; the module's own pins read it. */
|
|
80
|
+
export declare function readFormatFree(input: string): FormatFreeReading;
|
|
63
81
|
/** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings
|
|
64
82
|
* (the policy's, as a set), what a text the percent layers cannot read to its end answers (the policy's
|
|
65
83
|
* `onUnresolvedReading`; read only when the readings hold `percent-layers`), and the optional report. */
|
|
@@ -107,7 +125,7 @@ export interface SecretScanPolicy {
|
|
|
107
125
|
export interface SecretNotation {
|
|
108
126
|
readonly value: string;
|
|
109
127
|
readonly byKind?: Readonly<Record<string, string>>;
|
|
110
|
-
readonly byFamily?: Readonly<Record<
|
|
128
|
+
readonly byFamily?: Readonly<Partial<Record<CredentialFamilyId, string>>>;
|
|
111
129
|
readonly overStrip: string;
|
|
112
130
|
}
|
|
113
131
|
/** Where the vocabulary sees a credential: the withheld part of one hit (a kept label or username outside it), in the
|
|
@@ -116,7 +134,7 @@ export interface SecretSpan {
|
|
|
116
134
|
readonly start: number;
|
|
117
135
|
readonly end: number;
|
|
118
136
|
/** The row's id. */
|
|
119
|
-
readonly family:
|
|
137
|
+
readonly family: CredentialFamilyId;
|
|
120
138
|
/** The row's kind. */
|
|
121
139
|
readonly kind: string;
|
|
122
140
|
}
|
|
@@ -137,6 +155,9 @@ export interface RenderOptions {
|
|
|
137
155
|
readonly passes?: readonly RedactionPass[] | undefined;
|
|
138
156
|
readonly report?: RedactionReport | undefined;
|
|
139
157
|
}
|
|
158
|
+
/** Does a URL userinfo (no trailing `@`) hold a user (before its first `:`) or a password (after it)? An empty interval holds
|
|
159
|
+
* nothing — `scheme://@host` and `scheme://:@host` hold neither — as {@link userinfoWithheld} reads under `keep-username`. */
|
|
160
|
+
export declare function userinfoHolds(userinfo: string): boolean;
|
|
140
161
|
/**
|
|
141
162
|
* The wide forms rendered as passes, in the vocabulary's order: one pass per group (a group of several rows is ONE
|
|
142
163
|
* leftmost alternation, its hits attributed to the row that matches at the hit's offset). A row whose wide form
|
package/dist/secret-scan.js
CHANGED
|
@@ -159,7 +159,7 @@ function renderRemovals(input, removals, overStrip) {
|
|
|
159
159
|
const r = sorted[i];
|
|
160
160
|
if (r.end > end)
|
|
161
161
|
end = r.end;
|
|
162
|
-
if (r.marker !== "" && (widest === undefined || r.end - r.start > widest.end - widest.start))
|
|
162
|
+
if (r.marker !== "" && (widest === undefined || r.end - r.start > widest.end - widest.start || (r.start === widest.start && r.end === widest.end && r.pass > widest.pass)))
|
|
163
163
|
widest = r;
|
|
164
164
|
}
|
|
165
165
|
const marker = widest?.marker ?? overStrip;
|
|
@@ -194,6 +194,20 @@ function runPasses(input, passes, findings, toOriginal) {
|
|
|
194
194
|
});
|
|
195
195
|
return { out: cur, batches };
|
|
196
196
|
}
|
|
197
|
+
function touching(removals) {
|
|
198
|
+
let reached = -1;
|
|
199
|
+
let meets = false;
|
|
200
|
+
if (removals.some((x) => x.end <= x.start && x.marker !== ""))
|
|
201
|
+
return false;
|
|
202
|
+
for (const r of removals.filter((x) => x.end > x.start).sort((x, y) => x.start - y.start || y.end - x.end)) {
|
|
203
|
+
if (r.start < reached)
|
|
204
|
+
return false;
|
|
205
|
+
if (r.start === reached)
|
|
206
|
+
meets = true;
|
|
207
|
+
reached = Math.max(reached, r.end);
|
|
208
|
+
}
|
|
209
|
+
return meets;
|
|
210
|
+
}
|
|
197
211
|
function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = false) {
|
|
198
212
|
const found = report === undefined ? undefined : [];
|
|
199
213
|
const plain = runPasses(input, passes, found, toOriginal);
|
|
@@ -201,7 +215,7 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
|
|
|
201
215
|
const sequential = solo.length === 0 ? [] : sequentialRemovals(plain.batches);
|
|
202
216
|
const outer = unremovedHits(solo, sequential);
|
|
203
217
|
const mappable = !disjoint || replacedSpans(plain.batches) !== undefined;
|
|
204
|
-
const whole = outer.length === 0 && mappable ? undefined : renderRemovals(input, [...sequential, ...outer], overStrip);
|
|
218
|
+
const whole = outer.length === 0 && mappable && !touching(sequential) ? undefined : renderRemovals(input, [...sequential, ...outer], overStrip);
|
|
205
219
|
if (whole === undefined || (whole.out === plain.out && mappable)) {
|
|
206
220
|
if (report !== undefined)
|
|
207
221
|
appendAll(report.findings, found.map((x) => x.f));
|
|
@@ -240,51 +254,49 @@ function replacedSpans(batches) {
|
|
|
240
254
|
return undefined;
|
|
241
255
|
return spans;
|
|
242
256
|
}
|
|
243
|
-
function readFormatFree(input) {
|
|
244
|
-
const
|
|
245
|
-
const
|
|
246
|
-
const
|
|
247
|
-
const
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
viewOff
|
|
263
|
-
|
|
264
|
-
|
|
257
|
+
export function readFormatFree(input) {
|
|
258
|
+
const originOf = new Int32Array(input.length + 1);
|
|
259
|
+
const startingAt = new Int32Array(input.length + 1).fill(-1);
|
|
260
|
+
const kept = [];
|
|
261
|
+
const cuts = [];
|
|
262
|
+
let count = 0;
|
|
263
|
+
let viewOff = 0;
|
|
264
|
+
let orig = 0;
|
|
265
|
+
let keptFrom = 0;
|
|
266
|
+
let formatBefore = false;
|
|
267
|
+
for (const ch of input) {
|
|
268
|
+
if (FORMAT_CHAR_RE.test(ch)) {
|
|
269
|
+
if (orig > keptFrom)
|
|
270
|
+
kept.push(input.slice(keptFrom, orig));
|
|
271
|
+
keptFrom = orig + ch.length;
|
|
272
|
+
formatBefore = true;
|
|
273
|
+
}
|
|
274
|
+
else {
|
|
275
|
+
if (formatBefore && count > 0)
|
|
276
|
+
cuts.push(viewOff);
|
|
277
|
+
formatBefore = false;
|
|
278
|
+
originOf[count] = orig;
|
|
279
|
+
startingAt[viewOff] = count;
|
|
280
|
+
count++;
|
|
281
|
+
viewOff += ch.length;
|
|
265
282
|
}
|
|
266
|
-
|
|
267
|
-
viewToOriginal.push(input.length);
|
|
268
|
-
charStartOffsets.push(viewOff);
|
|
283
|
+
orig += ch.length;
|
|
269
284
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
const
|
|
274
|
-
|
|
275
|
-
|
|
285
|
+
if (input.length > keptFrom)
|
|
286
|
+
kept.push(input.slice(keptFrom));
|
|
287
|
+
originOf[count] = input.length;
|
|
288
|
+
const view = kept.join("");
|
|
289
|
+
const charIndexAt = (pos) => {
|
|
290
|
+
const idx = startingAt[pos];
|
|
291
|
+
return idx === undefined || idx < 0 ? count : idx;
|
|
276
292
|
};
|
|
293
|
+
const toOriginal = (pos) => originOf[charIndexAt(pos)];
|
|
277
294
|
const toOriginalSpan = (viewStart, viewEnd) => {
|
|
278
|
-
const startIdx =
|
|
279
|
-
const
|
|
280
|
-
const
|
|
281
|
-
|
|
282
|
-
return { start: viewToOriginal[startIdx], end: endOrig };
|
|
295
|
+
const startIdx = charIndexAt(viewStart);
|
|
296
|
+
const lastIdx = charIndexAt(viewEnd) - 1;
|
|
297
|
+
const endOrig = lastIdx >= startIdx && lastIdx < count ? originOf[lastIdx] + (input.codePointAt(originOf[lastIdx]) > 0xffff ? 2 : 1) : originOf[startIdx];
|
|
298
|
+
return { start: originOf[startIdx], end: endOrig };
|
|
283
299
|
};
|
|
284
|
-
const cuts = [];
|
|
285
|
-
for (let k = 1; k < viewChars.length; k++)
|
|
286
|
-
if (formatAtSlot[k] !== "")
|
|
287
|
-
cuts.push(charStartOffsets[k]);
|
|
288
300
|
return { view, cuts, toOriginal, toOriginalSpan };
|
|
289
301
|
}
|
|
290
302
|
export function runRedactionPasses(input, passes, run) {
|
|
@@ -473,7 +485,15 @@ const readsDecodedAsCredential = (username) => {
|
|
|
473
485
|
const decoded = percentDecodedOnce(username);
|
|
474
486
|
return decoded !== username && DECODED_USERNAME_FORMS.some((re) => re.test(decoded));
|
|
475
487
|
};
|
|
488
|
+
export function userinfoHolds(userinfo) {
|
|
489
|
+
const colon = userinfo.indexOf(":");
|
|
490
|
+
return colon === -1 ? userinfo.length > 0 : colon > 0 || colon < userinfo.length - 1;
|
|
491
|
+
}
|
|
476
492
|
function userinfoWithheld(userinfo, urlUserinfo) {
|
|
493
|
+
const cut = userinfoCut(userinfo, urlUserinfo);
|
|
494
|
+
return cut !== undefined && cut[1] > cut[0] ? cut : undefined;
|
|
495
|
+
}
|
|
496
|
+
function userinfoCut(userinfo, urlUserinfo) {
|
|
477
497
|
if (urlUserinfo === "whole" || userinfo.includes("@"))
|
|
478
498
|
return [0, userinfo.length];
|
|
479
499
|
const colonIdx = userinfo.indexOf(":");
|
|
@@ -30,10 +30,11 @@ export type RedactionConfidence = "high" | "medium" | "low";
|
|
|
30
30
|
/** Which rule table a finding names as its minter. A closed set: a report's entries can only have been written by
|
|
31
31
|
* the engine's own tables, so "this finding says X" is an engine statement. */
|
|
32
32
|
export type RedactionSource = "arg-summary" | "untrusted-egress" | "secret-env";
|
|
33
|
-
/** One value family: "this run of characters is itself a credential".
|
|
34
|
-
|
|
33
|
+
/** One value family: "this run of characters is itself a credential". `Id` is the id's type: the vocabulary's rows are
|
|
34
|
+
* checked against the open `string` form, and every row read from it carries a {@link CredentialFamilyId}. */
|
|
35
|
+
export interface ValueFamilyRow<Id extends string = CredentialFamilyId> {
|
|
35
36
|
/** Stable family name, `^[a-z0-9-]+$`; a finding's `family` and a span's `family`. */
|
|
36
|
-
readonly id:
|
|
37
|
+
readonly id: Id;
|
|
37
38
|
/** The finding `kind` this family reports (the coarse class a consumer groups by). */
|
|
38
39
|
readonly kind: string;
|
|
39
40
|
readonly source: RedactionSource;
|
|
@@ -61,14 +62,32 @@ export interface ValueFamilyRow {
|
|
|
61
62
|
* are not judged by their head there. */
|
|
62
63
|
readonly envValueHead?: RegExp;
|
|
63
64
|
}
|
|
64
|
-
/** Where a credential NAME is read beside its value:
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
|
|
65
|
+
/** Where a credential NAME is read beside its value:
|
|
66
|
+
* - `assignment` — a keyword ahead of `=` / `:` in free text, where a name's words start, camel humps included (the
|
|
67
|
+
* keyword-value row);
|
|
68
|
+
* - `structured-key` — the name of a field inside a value, read as the tail of the field name behind up to 64 field-name
|
|
69
|
+
* characters (`[a-z0-9_.-]`), any case (the keyword-value row's env value form);
|
|
70
|
+
* - `url-param` — the name of a URL query or fragment parameter, any case, as written (the signed-url-param row).
|
|
71
|
+
* In the first two places an `_` inside a name stands for `_`, `-` or nothing (`private_key`, `private-key`,
|
|
72
|
+
* `privateKey`); a name written with a leading `_` is read only as the tail of a longer name, behind one of `_` `.` `-`
|
|
73
|
+
* (`npm_auth`, never `oauth`). */
|
|
74
|
+
export type CredentialNamePlace = "assignment" | "structured-key" | "url-param";
|
|
75
|
+
/** How firmly a name says the value beside it is a credential, as the `assignment` place reads the value (the other places
|
|
76
|
+
* read every non-empty value the same way):
|
|
77
|
+
* - `strong` — the name holds nothing else (`password`, `secret`): a value is read from one character in a declaration
|
|
78
|
+
* form (a quote around the name or the value, or a separator with nothing on either side: `password=x`,
|
|
79
|
+
* `"secret": x`), from four otherwise; a flag or placeholder word is read too (`secret: false`);
|
|
80
|
+
* - `weak` — the name also names things that are no secret (`token`, `key`): a value is read from eight characters with
|
|
81
|
+
* two upper-case letters or digits among them, or from twenty, and never where it is, whole, a flag or placeholder word
|
|
82
|
+
* ({@link CredentialVocabulary.flagWords}, {@link CredentialVocabulary.placeholderWords}) or an environment reference
|
|
83
|
+
* (`${NAME}`, `$NAME`). */
|
|
84
|
+
export type CredentialNameStrength = "strong" | "weak";
|
|
85
|
+
/** One credential name: "the value this name stands beside is a credential", in each place it lists, as firmly as its
|
|
86
|
+
* strength says. */
|
|
69
87
|
export interface CredentialNameRow {
|
|
70
88
|
readonly name: string;
|
|
71
89
|
readonly places: readonly CredentialNamePlace[];
|
|
90
|
+
readonly strength: CredentialNameStrength;
|
|
72
91
|
}
|
|
73
92
|
/** Where a credential word counts in a name (env names are upper-cased words joined by `_`):
|
|
74
93
|
* - `leading` — at the start of the name or right after `_`;
|
|
@@ -113,11 +132,16 @@ export interface CredentialVocabulary {
|
|
|
113
132
|
* credential words are looked for, in every name view alike. */
|
|
114
133
|
readonly nonCredentialPrefix: RegExp;
|
|
115
134
|
/** The credential names a value stands beside, one list for every place a name is read: the keyword-value row reads the
|
|
116
|
-
* `assignment` names (in this order), the
|
|
135
|
+
* `assignment` names (in this order) and, in its env value form, the `structured-key` names; the signed-url-param row
|
|
136
|
+
* reads the `url-param` names. */
|
|
117
137
|
readonly names: readonly CredentialNameRow[];
|
|
118
138
|
/** The words a setting's value is when it is a switch, not a secret (lower case, read in any case): the env value view's
|
|
119
|
-
* assignment form leaves a value spelled as one of them. */
|
|
139
|
+
* assignment form leaves a value spelled as one of them, and so does the keyword-value row beside a weak name. */
|
|
120
140
|
readonly flagWords: readonly string[];
|
|
141
|
+
/** The words a credential setting holds when it holds none — a local model gateway's placeholder keys (lower case, read
|
|
142
|
+
* in any case): the keyword-value row leaves a value spelled as one of them beside a weak name, and the env value check
|
|
143
|
+
* takes one for no credential. */
|
|
144
|
+
readonly placeholderWords: readonly string[];
|
|
121
145
|
}
|
|
122
146
|
/**
|
|
123
147
|
* A URL of any scheme ({@link URI_SCHEME_SOURCE}, then `://`), to the first white space, quote or backtick, any case. URL
|
|
@@ -131,6 +155,221 @@ export declare const URL_OF_ANY_SCHEME: RegExp;
|
|
|
131
155
|
* and of a bearer / authorization credential run in this table, and of the value rules of the env scrub. A shorter
|
|
132
156
|
* string names no secret, whoever holds it. */
|
|
133
157
|
export declare const CREDENTIAL_MIN_CHARS = 8;
|
|
158
|
+
/** The words that, opening the run glued to a scheme word, make the scheme word part of a name (`bearerTokenProvider`,
|
|
159
|
+
* `BasicAuthenticationFilter`, `BEARER_TOKEN_ENV`): lower case, matched against the run's first word read in any of its
|
|
160
|
+
* two spellings (`Token`, `TOKEN`). */
|
|
161
|
+
export declare const SCHEME_FOLLOWER_WORDS: readonly string[];
|
|
162
|
+
/** The value families, in egress pass order: {@link CREDENTIAL_VOCABULARY}`.values`, held here so their ids are one
|
|
163
|
+
* closed set ({@link CredentialFamilyId}). */
|
|
164
|
+
declare const VALUE_FAMILY_ROWS: readonly [{
|
|
165
|
+
readonly id: "github-token";
|
|
166
|
+
readonly kind: "prefixed-token";
|
|
167
|
+
readonly source: "arg-summary";
|
|
168
|
+
readonly confidence: "high";
|
|
169
|
+
readonly pass: "vendor-prefixed";
|
|
170
|
+
readonly match: RegExp;
|
|
171
|
+
readonly strict: RegExp;
|
|
172
|
+
readonly fragment: RegExp;
|
|
173
|
+
readonly envValueHead: RegExp;
|
|
174
|
+
}, {
|
|
175
|
+
readonly id: "github-fine-grained-pat";
|
|
176
|
+
readonly kind: "prefixed-token";
|
|
177
|
+
readonly source: "arg-summary";
|
|
178
|
+
readonly confidence: "high";
|
|
179
|
+
readonly pass: "vendor-prefixed";
|
|
180
|
+
readonly match: RegExp;
|
|
181
|
+
readonly strict: RegExp;
|
|
182
|
+
readonly fragment: RegExp;
|
|
183
|
+
readonly envValueHead: RegExp;
|
|
184
|
+
}, {
|
|
185
|
+
readonly id: "openai-api-key";
|
|
186
|
+
readonly kind: "prefixed-token";
|
|
187
|
+
readonly source: "arg-summary";
|
|
188
|
+
readonly confidence: "high";
|
|
189
|
+
readonly pass: "vendor-prefixed";
|
|
190
|
+
readonly match: RegExp;
|
|
191
|
+
readonly strict: RegExp;
|
|
192
|
+
readonly fragment: RegExp;
|
|
193
|
+
readonly envValueHead: RegExp;
|
|
194
|
+
}, {
|
|
195
|
+
readonly id: "stripe-secret-key";
|
|
196
|
+
readonly kind: "prefixed-token";
|
|
197
|
+
readonly source: "arg-summary";
|
|
198
|
+
readonly confidence: "high";
|
|
199
|
+
readonly pass: "vendor-prefixed";
|
|
200
|
+
readonly match: RegExp;
|
|
201
|
+
readonly strict: RegExp;
|
|
202
|
+
readonly fragment: RegExp;
|
|
203
|
+
}, {
|
|
204
|
+
readonly id: "stripe-publishable-key";
|
|
205
|
+
readonly kind: "prefixed-token";
|
|
206
|
+
readonly source: "arg-summary";
|
|
207
|
+
readonly confidence: "high";
|
|
208
|
+
readonly pass: "vendor-prefixed";
|
|
209
|
+
readonly match: RegExp;
|
|
210
|
+
readonly envValueHead: RegExp;
|
|
211
|
+
}, {
|
|
212
|
+
readonly id: "gitlab-pat";
|
|
213
|
+
readonly kind: "prefixed-token";
|
|
214
|
+
readonly source: "arg-summary";
|
|
215
|
+
readonly confidence: "high";
|
|
216
|
+
readonly pass: "vendor-prefixed";
|
|
217
|
+
readonly match: RegExp;
|
|
218
|
+
readonly strict: RegExp;
|
|
219
|
+
readonly fragment: RegExp;
|
|
220
|
+
readonly envValueHead: RegExp;
|
|
221
|
+
}, {
|
|
222
|
+
readonly id: "npm-access-token";
|
|
223
|
+
readonly kind: "prefixed-token";
|
|
224
|
+
readonly source: "arg-summary";
|
|
225
|
+
readonly confidence: "high";
|
|
226
|
+
readonly pass: "vendor-prefixed";
|
|
227
|
+
readonly match: RegExp;
|
|
228
|
+
readonly envValueHead: RegExp;
|
|
229
|
+
}, {
|
|
230
|
+
readonly id: "google-oauth-access-token";
|
|
231
|
+
readonly kind: "prefixed-token";
|
|
232
|
+
readonly source: "arg-summary";
|
|
233
|
+
readonly confidence: "high";
|
|
234
|
+
readonly pass: "vendor-prefixed";
|
|
235
|
+
readonly match: RegExp;
|
|
236
|
+
readonly envValueHead: RegExp;
|
|
237
|
+
}, {
|
|
238
|
+
readonly id: "pypi-token";
|
|
239
|
+
readonly kind: "prefixed-token";
|
|
240
|
+
readonly source: "arg-summary";
|
|
241
|
+
readonly confidence: "high";
|
|
242
|
+
readonly pass: "vendor-prefixed";
|
|
243
|
+
readonly match: RegExp;
|
|
244
|
+
readonly strict: RegExp;
|
|
245
|
+
readonly fragment: RegExp;
|
|
246
|
+
}, {
|
|
247
|
+
readonly id: "google-api-key";
|
|
248
|
+
readonly kind: "prefixed-token";
|
|
249
|
+
readonly source: "arg-summary";
|
|
250
|
+
readonly confidence: "high";
|
|
251
|
+
readonly match: RegExp;
|
|
252
|
+
readonly strict: RegExp;
|
|
253
|
+
}, {
|
|
254
|
+
readonly id: "slack-token";
|
|
255
|
+
readonly kind: "prefixed-token";
|
|
256
|
+
readonly source: "arg-summary";
|
|
257
|
+
readonly confidence: "high";
|
|
258
|
+
readonly match: RegExp;
|
|
259
|
+
readonly strict: RegExp;
|
|
260
|
+
readonly fragment: RegExp;
|
|
261
|
+
readonly envValueHead: RegExp;
|
|
262
|
+
}, {
|
|
263
|
+
readonly id: "slack-app-token";
|
|
264
|
+
readonly kind: "prefixed-token";
|
|
265
|
+
readonly source: "arg-summary";
|
|
266
|
+
readonly confidence: "high";
|
|
267
|
+
readonly match: RegExp;
|
|
268
|
+
readonly strict: RegExp;
|
|
269
|
+
readonly fragment: RegExp;
|
|
270
|
+
}, {
|
|
271
|
+
readonly id: "aws-access-key-id";
|
|
272
|
+
readonly kind: "prefixed-token";
|
|
273
|
+
readonly source: "arg-summary";
|
|
274
|
+
readonly confidence: "high";
|
|
275
|
+
readonly match: RegExp;
|
|
276
|
+
readonly strict: RegExp;
|
|
277
|
+
readonly fragment: RegExp;
|
|
278
|
+
readonly envValueHead: RegExp;
|
|
279
|
+
}, {
|
|
280
|
+
readonly id: "huggingface-token";
|
|
281
|
+
readonly kind: "prefixed-token";
|
|
282
|
+
readonly source: "arg-summary";
|
|
283
|
+
readonly confidence: "high";
|
|
284
|
+
readonly match: RegExp;
|
|
285
|
+
readonly strict: RegExp;
|
|
286
|
+
}, {
|
|
287
|
+
readonly id: "jwt";
|
|
288
|
+
readonly kind: "jwt";
|
|
289
|
+
readonly source: "arg-summary";
|
|
290
|
+
readonly confidence: "high";
|
|
291
|
+
readonly match: RegExp;
|
|
292
|
+
readonly strict: RegExp;
|
|
293
|
+
readonly envValueHead: RegExp;
|
|
294
|
+
}, {
|
|
295
|
+
readonly id: "private-key";
|
|
296
|
+
readonly kind: "private-key-block";
|
|
297
|
+
readonly source: "arg-summary";
|
|
298
|
+
readonly confidence: "high";
|
|
299
|
+
readonly match: RegExp;
|
|
300
|
+
readonly strict: RegExp;
|
|
301
|
+
readonly envValue: RegExp;
|
|
302
|
+
}, {
|
|
303
|
+
readonly id: "bearer-token";
|
|
304
|
+
readonly kind: "bearer-token";
|
|
305
|
+
readonly source: "arg-summary";
|
|
306
|
+
readonly confidence: "high";
|
|
307
|
+
readonly match: RegExp;
|
|
308
|
+
readonly strict: RegExp;
|
|
309
|
+
readonly envValue: RegExp;
|
|
310
|
+
}, {
|
|
311
|
+
readonly id: "authorization-header";
|
|
312
|
+
readonly kind: "authorization-header";
|
|
313
|
+
readonly source: "arg-summary";
|
|
314
|
+
readonly confidence: "high";
|
|
315
|
+
readonly match: RegExp;
|
|
316
|
+
readonly envValue: RegExp;
|
|
317
|
+
}, {
|
|
318
|
+
readonly id: "basic-credential";
|
|
319
|
+
readonly kind: "scheme-credential";
|
|
320
|
+
readonly source: "arg-summary";
|
|
321
|
+
readonly confidence: "high";
|
|
322
|
+
readonly match: RegExp;
|
|
323
|
+
}, {
|
|
324
|
+
readonly id: "token-scheme";
|
|
325
|
+
readonly kind: "scheme-credential";
|
|
326
|
+
readonly source: "arg-summary";
|
|
327
|
+
readonly confidence: "high";
|
|
328
|
+
readonly match: RegExp;
|
|
329
|
+
readonly envValue: RegExp;
|
|
330
|
+
}, {
|
|
331
|
+
readonly id: "azure-storage-account-key";
|
|
332
|
+
readonly kind: "prefixed-token";
|
|
333
|
+
readonly source: "arg-summary";
|
|
334
|
+
readonly confidence: "high";
|
|
335
|
+
readonly match: RegExp;
|
|
336
|
+
readonly strict: RegExp;
|
|
337
|
+
}, {
|
|
338
|
+
readonly id: "keyword-value";
|
|
339
|
+
readonly kind: "keyword-value";
|
|
340
|
+
readonly source: "arg-summary";
|
|
341
|
+
readonly confidence: "medium";
|
|
342
|
+
readonly match: RegExp;
|
|
343
|
+
readonly envValue: RegExp;
|
|
344
|
+
}, {
|
|
345
|
+
readonly id: "sonar-login";
|
|
346
|
+
readonly kind: "keyword-value";
|
|
347
|
+
readonly source: "arg-summary";
|
|
348
|
+
readonly confidence: "high";
|
|
349
|
+
readonly match: RegExp;
|
|
350
|
+
readonly envValue: RegExp;
|
|
351
|
+
}, {
|
|
352
|
+
readonly id: "url-userinfo";
|
|
353
|
+
readonly kind: "url-credentials";
|
|
354
|
+
readonly source: "untrusted-egress";
|
|
355
|
+
readonly confidence: "medium";
|
|
356
|
+
readonly match: RegExp;
|
|
357
|
+
}, {
|
|
358
|
+
readonly id: "webhook-url";
|
|
359
|
+
readonly kind: "webhook-url";
|
|
360
|
+
readonly source: "untrusted-egress";
|
|
361
|
+
readonly confidence: "high";
|
|
362
|
+
readonly match: RegExp;
|
|
363
|
+
readonly envValue: RegExp;
|
|
364
|
+
}, {
|
|
365
|
+
readonly id: "signed-url-param";
|
|
366
|
+
readonly kind: "signed-url-param";
|
|
367
|
+
readonly source: "untrusted-egress";
|
|
368
|
+
readonly confidence: "high";
|
|
369
|
+
readonly match: RegExp;
|
|
370
|
+
}];
|
|
371
|
+
/** A value family's id: the closed set of the vocabulary's rows, so a table keyed by family is checked against it. */
|
|
372
|
+
export type CredentialFamilyId = (typeof VALUE_FAMILY_ROWS)[number]["id"];
|
|
134
373
|
export declare const CREDENTIAL_VOCABULARY: CredentialVocabulary;
|
|
135
374
|
/** The table the package's own readers run (the matcher, the renderer, the record-key and header-name readings, the
|
|
136
375
|
* env-name view's default): {@link CREDENTIAL_VOCABULARY} with every regex its own object, copied as this module is
|
|
@@ -170,6 +409,11 @@ export interface EnvValueView {
|
|
|
170
409
|
readonly tokenHead: RegExp;
|
|
171
410
|
/** The words a setting's value is when it is a switch, lower case (compare a value lower-cased). */
|
|
172
411
|
readonly flagWords: ReadonlySet<string>;
|
|
412
|
+
/** The placeholder keys that hold no credential, lower case ({@link CredentialVocabulary.placeholderWords}). */
|
|
413
|
+
readonly placeholderWords: ReadonlySet<string>;
|
|
414
|
+
/** The `url-userinfo` row's form, global: group 1 what stands in front of the userinfo, group 2 the userinfo and its
|
|
415
|
+
* `@`. A vocabulary with no such row reads no userinfo. */
|
|
416
|
+
readonly urlUserinfo: RegExp;
|
|
173
417
|
}
|
|
174
418
|
/** The env value view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
|
|
175
419
|
* Every regex of a view is the view's own object. */
|
|
@@ -207,3 +451,4 @@ export declare function isSensitiveRecordKey(key: string, view?: RecordKeyView):
|
|
|
207
451
|
* that mistakes such a header for a credential drops the key that should have gone out. `view` is the vocabulary's
|
|
208
452
|
* record-key view unless a caller holds another. */
|
|
209
453
|
export declare function isSensitiveHeaderName(name: string, view?: RecordKeyView): boolean;
|
|
454
|
+
export {};
|
|
@@ -1,8 +1,11 @@
|
|
|
1
|
-
const
|
|
2
|
-
const
|
|
1
|
+
const SCHEME_CHAR = String.raw `[A-Za-z0-9+.\-_]`;
|
|
2
|
+
const SCHEME_LEAD = String.raw `[0-9+.\-_]`;
|
|
3
|
+
const URI_SCHEME_SOURCE = `(?<!${SCHEME_CHAR})${SCHEME_LEAD}*[A-Za-z]${SCHEME_CHAR}*`;
|
|
3
4
|
export const URL_OF_ANY_SCHEME = new RegExp(String.raw `${URI_SCHEME_SOURCE}:\/\/[^\s"'\`]+`, "gi");
|
|
4
5
|
export const CREDENTIAL_MIN_CHARS = 8;
|
|
5
|
-
const
|
|
6
|
+
const atLeast = (cls, n) => `${cls}{${n}}${cls}*`;
|
|
7
|
+
const repeated = (piece, min, max) => piece.repeat(min) + `(?:${piece}`.repeat(max - min) + ")?".repeat(max - min);
|
|
8
|
+
const VENDOR_BODY = atLeast("[A-Za-z0-9_-]", CREDENTIAL_MIN_CHARS);
|
|
6
9
|
const VALUE_END = String.raw `\s"';|&`;
|
|
7
10
|
const VALUE_CHAR = `[^${VALUE_END}]`;
|
|
8
11
|
const ESCAPE_DEPTH = 63;
|
|
@@ -11,9 +14,20 @@ const PLAIN_QUOTES = [`"`, `'`, `"`, `'`, `"`, `'`, `%22`, `%2
|
|
|
11
14
|
const ESCAPED_QUOTES = [`"`, `'`, `x22`, `x27`, `u0022`, `u0027`, `042`, `047`];
|
|
12
15
|
const BACKSLASHES = String.raw `\\{1,${ESCAPE_DEPTH}}`;
|
|
13
16
|
const QUOTE = `(?:${PLAIN_QUOTES.join("|")}|${BACKSLASHES}(?:${ESCAPED_QUOTES.join("|")}))`;
|
|
14
|
-
const
|
|
15
|
-
const
|
|
17
|
+
const ESCAPE_TAIL_CODES = String.raw `\\(?:[bfnrt]|u[0-9A-Fa-f]{4})|%[0-9A-Fa-f]{2}`;
|
|
18
|
+
const TERMINAL_CONTROL_TAIL = String.raw `(?:\x1b|\\u001[bB])(?:\[[0-?]*[ -/\\]*[@-\[\]-~]|[ -/\\]*[0-\[\]-~])|(?:\x9b|\\u009[bB])[0-?]*[ -/\\]*[@-\[\]-~]`;
|
|
19
|
+
const ESCAPE_TAIL = `(?:${ESCAPE_TAIL_CODES}|${TERMINAL_CONTROL_TAIL})`;
|
|
20
|
+
const BEARER_WORD = "[Bb][Ee][Aa][Rr][Ee][Rr]";
|
|
21
|
+
const BASIC_WORD = "[Bb][Aa][Ss][Ii][Cc]";
|
|
22
|
+
const GLUED_LEFT = String.raw `(?:(?<![A-Za-z0-9_/\\.@])|(?<=${ESCAPE_TAIL}))`;
|
|
23
|
+
const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE})|(?<=${ESCAPE_TAIL})|(?<=${GLUED_LEFT}(?:${BEARER_WORD}|${BASIC_WORD})))`;
|
|
16
24
|
const AFTER_CLOSING_QUOTE = String.raw `(?:[\s,}\]:]|["']|$)`;
|
|
25
|
+
const USERINFO_QUOTES_MAX = 8;
|
|
26
|
+
const URL_USERINFO_SOURCE = String.raw `[^/\s"]*` + repeated(String.raw `"(?!${AFTER_CLOSING_QUOTE}|>)[^/\s"]*`, 0, USERINFO_QUOTES_MAX) + "@";
|
|
27
|
+
const USERINFO_WORD_START = String.raw `(?<![^\s"])`;
|
|
28
|
+
const USERINFO_FRONT = String.raw `${URI_SCHEME_SOURCE}:\/\/|(?<!${SCHEME_CHAR})(?:https?|wss?|ftp):\/?(?!\/)|${USERINFO_WORD_START}\/\/`;
|
|
29
|
+
const FRONTLESS_HOST = String.raw `(?:\[[0-9A-Fa-f:.]+\]|[^\s/@:"'\`]+)(?::[0-9]{1,5}(?![0-9A-Za-z_])|\/)`;
|
|
30
|
+
const FRONTLESS_USERINFO = String.raw `(?!(?:mailto|sips?|xmpp|im|acct):)[^\s/@:"]+:(?![\\/"'])${URL_USERINFO_SOURCE}(?<=[^:]@)(?=${FRONTLESS_HOST})`;
|
|
17
31
|
const VALUE_RUN_END = `(?:(?<!\\\\)|(?!\\\\{0,${ESCAPE_DEPTH}}["']))`;
|
|
18
32
|
const GLUED_RUN = `${VALUE_CHAR}*${VALUE_RUN_END}`;
|
|
19
33
|
const pairedValue = (open, close) => `${open}(?!${AFTER_CLOSING_QUOTE})[^\\n]{0,${QUOTED_VALUE_MAX}}?${close}`;
|
|
@@ -23,109 +37,174 @@ const QUOTED_VALUE = [
|
|
|
23
37
|
`${QUOTE}(?!${AFTER_CLOSING_QUOTE})${GLUED_RUN}`,
|
|
24
38
|
].join("|");
|
|
25
39
|
const BEARER_TAIL = `(?:(?!${QUOTE})\\S${GLUED_RUN}|(?<==)(?:${QUOTED_VALUE}))`;
|
|
26
|
-
const
|
|
27
|
-
const
|
|
40
|
+
const TOKEN68_CHAR = "[A-Za-z0-9._~+/-]";
|
|
41
|
+
const SCHEME_SEPARATOR = String.raw `[\s\\"'\`“”|]`;
|
|
42
|
+
export const SCHEME_FOLLOWER_WORDS = Object.freeze([
|
|
43
|
+
"token", "tokens", "auth", "authentication", "authorization", "header", "headers", "scheme", "schemes", "realm", "challenge",
|
|
44
|
+
"access", "login", "prefix", "format", "type", "mode", "method", "flow", "grant", "usage", "example", "setup", "plan", "tier",
|
|
45
|
+
"info", "information", "support", "level", "string", "config", "configuration", "settings", "version", "check", "validation",
|
|
46
|
+
"error", "errors", "failed", "failure", "credentials", "user", "username",
|
|
47
|
+
]);
|
|
48
|
+
const SCHEME_GLUE_GATE = (() => {
|
|
49
|
+
const t = TOKEN68_CHAR;
|
|
50
|
+
const address = `(?=${SCHEME_LEAD}{0,95}[A-Za-z]${SCHEME_CHAR}{0,95}:\\/\\/)`;
|
|
51
|
+
const long = `(?:(?=${t}{0,95}[A-Z0-9])|(?=${t}{20}))`;
|
|
52
|
+
const signature = `(?=${t}{0,95}[A-Z])(?=${t}{0,95}[a-z])(?=${t}{0,95}[0-9])`;
|
|
53
|
+
const follower = `(?=[-_~+]?(?:${SCHEME_FOLLOWER_WORDS.map((w) => `[${w[0].toUpperCase()}${w[0]}]${w.slice(1)}(?![a-z])`).join("|")}|(?:${SCHEME_FOLLOWER_WORDS.map((w) => w.toUpperCase()).join("|")})(?=[A-Z][a-z]|[^A-Za-z]|$)))`;
|
|
54
|
+
const named = `(?:(?=[-_~+])|(?=[A-Z][a-z])(?![A-Za-z]{0,95}[A-Z](?![a-z]))(?=[A-Za-z]{1,96}(?![A-Za-z0-9])))`;
|
|
55
|
+
return `(?=${t})(?:${address}|(?![./])${long}(?:${signature}|(?!${follower})(?!${named})))`;
|
|
56
|
+
})();
|
|
57
|
+
const BASE64_COLON = `${repeated("[A-Za-z0-9+/]{4}", 0, 255)}(?:O[g-v][A-Za-z0-9+/]{2}|[A-Za-z0-9+/][DTjz][o-r][A-Za-z0-9+/])`;
|
|
58
|
+
const backReference = (n) => `\\${n}`;
|
|
59
|
+
const headerParam = (g) => {
|
|
60
|
+
const partner = String.raw `(?<!\\)${backReference(g)}`;
|
|
61
|
+
return String.raw `=[ \t]*(\\{0,${ESCAPE_DEPTH}}["'])(?=([^\r\n]*?(?=${partner}|[\r\n]|$)))${backReference(g + 1)}(?:${partner}(?=[\s,;]|\\{0,${ESCAPE_DEPTH}}["']|$)|(?=[\r\n]|$))`;
|
|
62
|
+
};
|
|
63
|
+
const HEADER_PLAIN = String.raw `[^\s"',=]`;
|
|
64
|
+
const headerPiece = (g) => String.raw `(?:${headerParam(g)}|=(?!=)|=(?==)=*(?![ \t]*\\{0,${ESCAPE_DEPTH}}["'])|[ \t]*,[ \t]*(?=[^\s"',])|[ \t]+(?==)|(?<==)[ \t]+(?=[^\s"',]))`;
|
|
65
|
+
const HEADER_VALUE = String.raw `(?:[A-Za-z][A-Za-z0-9!#$%&*+.^_|~-]*[ \t]+)?(?:${HEADER_PLAIN}|${headerPiece(2)})(?:${HEADER_PLAIN}*${headerPiece(4)})*${HEADER_PLAIN}*${VALUE_RUN_END}`;
|
|
28
66
|
const headed = (body, flags = "g") => new RegExp(`${WORD_START}${body}`, flags);
|
|
67
|
+
const unheaded = (body, flags = "g") => new RegExp(body, flags);
|
|
29
68
|
const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
69
|
+
const inAnyCase = (word) => word.replace(/[A-Za-z]/g, (c) => `[${c.toUpperCase()}${c.toLowerCase()}]`);
|
|
70
|
+
const PEM_LABEL = "[ A-Za-z0-9_-]{0,100}?";
|
|
71
|
+
const PEM_HEAD = `-----${inAnyCase("BEGIN")}${PEM_LABEL}${inAnyCase("PRIVATE KEY")}(?: ${inAnyCase("BLOCK")})?(?:-----|(?=\\s*$|[ \\t]*\\r?\\n[A-Za-z0-9+/]{${CREDENTIAL_MIN_CHARS}}))`;
|
|
72
|
+
const PEM_FOOT = `-----${inAnyCase("END")}${PEM_LABEL}${inAnyCase("PRIVATE KEY")}(?: ${inAnyCase("BLOCK")})?-----`;
|
|
30
73
|
const HUMP_BEFORE = "[a-z0-9]";
|
|
31
74
|
const CREDENTIAL_NAMES = [
|
|
32
|
-
|
|
33
|
-
{ name: "
|
|
75
|
+
{ name: "token", places: ["assignment", "structured-key", "url-param"], strength: "weak" },
|
|
76
|
+
{ name: "secret", places: ["assignment", "structured-key", "url-param"], strength: "strong" },
|
|
77
|
+
{ name: "key", places: ["assignment", "url-param"], strength: "weak" },
|
|
78
|
+
{ name: "password", places: ["assignment", "structured-key", "url-param"], strength: "strong" },
|
|
79
|
+
{ name: "passwd", places: ["assignment", "structured-key", "url-param"], strength: "strong" },
|
|
80
|
+
{ name: "credential", places: ["assignment", "structured-key"], strength: "strong" },
|
|
81
|
+
{ name: "private_key", places: ["assignment", "structured-key", "url-param"], strength: "strong" },
|
|
82
|
+
{ name: "pwd", places: ["structured-key", "url-param"], strength: "strong" },
|
|
83
|
+
{ name: "client_secret", places: ["url-param"], strength: "strong" },
|
|
84
|
+
{ name: "api_key", places: ["structured-key", "url-param"], strength: "weak" },
|
|
85
|
+
...["account_key", "access_key", "subscription_key", "_auth"].map((name) => ({ name, places: ["structured-key"], strength: "weak" })),
|
|
86
|
+
...["signature", "sig"].map((name) => ({ name, places: ["structured-key", "url-param"], strength: "weak" })),
|
|
34
87
|
...[
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
].map((name) => ({ name, places: ["url-param"] })),
|
|
88
|
+
"apikey", "api-key", "access_token", "refresh_token", "id_token", "auth", "authorization", "proxy-authorization",
|
|
89
|
+
"x-amz-signature", "x-amz-security-token", "x-amz-credential", "sas", "code",
|
|
90
|
+
].map((name) => ({ name, places: ["url-param"], strength: "weak" })),
|
|
38
91
|
];
|
|
39
|
-
const namesAt = (place) => CREDENTIAL_NAMES.filter((n) => n.places.includes(place)).map((n) => n.name);
|
|
92
|
+
const namesAt = (place, strength) => CREDENTIAL_NAMES.filter((n) => n.places.includes(place) && (strength === undefined || n.strength === strength)).map((n) => n.name);
|
|
40
93
|
const KEYWORD_VALUE_WORDS = namesAt("assignment");
|
|
41
94
|
const FLAG_WORDS = ["true", "false", "none", "null", "yes", "no", "on", "off", "enabled", "disabled", "required", "optional"];
|
|
42
|
-
const
|
|
95
|
+
const PLACEHOLDER_WORDS = ["undefined", "placeholder", "lm-studio", "not-needed", "sk-no-key-required"];
|
|
96
|
+
const residue = (head, cls, min) => String.raw `${head}(?=${cls}{0,96}[0-9A-Z]${cls}{0,96}[0-9A-Z])${atLeast(cls, min)}`;
|
|
43
97
|
const GITLAB_HEAD = "gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-";
|
|
44
|
-
const anyCase = (word) => [...word].map((c) => `[${c.toUpperCase()}${c.toLowerCase()}]`).join("");
|
|
98
|
+
const anyCase = (word) => [...word].map((c) => (c === "_" ? "[-_]?" : `[${c.toUpperCase()}${c.toLowerCase()}]`)).join("");
|
|
45
99
|
const AZURE_ACCOUNT_KEY = { front: "Account", keyword: "Key", separator: "=", body: "A-Za-z0-9+/" };
|
|
46
100
|
const AZURE_FIELD = `${AZURE_ACCOUNT_KEY.front}${AZURE_ACCOUNT_KEY.keyword}${AZURE_ACCOUNT_KEY.separator}`;
|
|
47
|
-
const AZURE_BODY = `[${AZURE_ACCOUNT_KEY.body}]
|
|
101
|
+
const AZURE_BODY = `${atLeast(`[${AZURE_ACCOUNT_KEY.body}]`, 40)}={0,2}`;
|
|
48
102
|
const VALUE_ROW_FIELDS = [
|
|
49
103
|
[AZURE_ACCOUNT_KEY.front, AZURE_ACCOUNT_KEY.keyword, AZURE_ACCOUNT_KEY.separator, `[${AZURE_ACCOUNT_KEY.body}<]`],
|
|
50
104
|
];
|
|
51
105
|
const leftToItsRow = (keyword) => VALUE_ROW_FIELDS.filter(([, k]) => k.toLowerCase() === keyword)
|
|
52
106
|
.map(([front, k, separator, valueStart]) => `(?!(?<=${WORD_START}${front})${k}${separator}${valueStart})`)
|
|
53
107
|
.join("");
|
|
54
|
-
const
|
|
108
|
+
const keywordHead = (words) => `(?:${WORD_START}(?:${words.map(anyCase).join("|")})|(?<=${HUMP_BEFORE})(?:${words.map((w) => leftToItsRow(w) + w[0].toUpperCase() + anyCase(w.slice(1))).join("|")})|(?<=[A-Z])(?:${words.map((w) => w[0].toUpperCase() + w[1] + anyCase(w.slice(2))).join("|")}))`;
|
|
109
|
+
const KEYWORD_HEAD = keywordHead(KEYWORD_VALUE_WORDS);
|
|
110
|
+
const STRONG_KEYWORD_HEAD = keywordHead(namesAt("assignment", "strong"));
|
|
55
111
|
const KEY_QUOTE = `(?:\\\\{0,${ESCAPE_DEPTH}}["'])?`;
|
|
112
|
+
const KEYWORD_LABEL = String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*(?:(\\{0,${ESCAPE_DEPTH}})(["']))?)`;
|
|
113
|
+
const VALUE_PARTNER = String.raw `(?<!\\)\2\3`;
|
|
114
|
+
const STRONG_DECLARATION_BEHIND = String.raw `(?<=${STRONG_KEYWORD_HEAD}(?:\\{0,${ESCAPE_DEPTH}}["']\s*[=:]\s*${KEY_QUOTE}|\s*[=:]\s*\\{0,${ESCAPE_DEPTH}}["']|[=:]))`;
|
|
115
|
+
const STRONG_LABEL_BEHIND = String.raw `(?<=${STRONG_KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})`;
|
|
116
|
+
const ENV_REFERENCE_NAME = `[A-Za-z_][A-Za-z0-9_]{0,127}`;
|
|
117
|
+
const NO_CREDENTIAL_VALUE = `(?:${[...FLAG_WORDS, ...PLACEHOLDER_WORDS].map(anyCase).join("|")}|\\$(?:\\{${ENV_REFERENCE_NAME}\\}|${ENV_REFERENCE_NAME}))`;
|
|
118
|
+
const UNQUOTED_VALUE_END = `(?=[${VALUE_END}]|\\\\{1,${ESCAPE_DEPTH}}["']|$)`;
|
|
119
|
+
const PLAIN_VALUE_CHAR = `[^${VALUE_END}0-9A-Z]`;
|
|
120
|
+
const KEYWORD_VALUE = [
|
|
121
|
+
String.raw `(?<=["'])(?=([^\n]{0,${QUOTED_VALUE_MAX}}?)${VALUE_PARTNER})(?:${STRONG_LABEL_BEHIND}(?!${VALUE_PARTNER})|(?![^\n]{0,7}${VALUE_PARTNER})(?:(?![^0-9A-Z\n]{0,${QUOTED_VALUE_MAX}}(?:[0-9A-Z][^0-9A-Z\n]{0,${QUOTED_VALUE_MAX}})?${VALUE_PARTNER})|(?![^\n]{0,19}${VALUE_PARTNER}))(?!${NO_CREDENTIAL_VALUE}${VALUE_PARTNER}))\4`,
|
|
122
|
+
`(?:${STRONG_DECLARATION_BEHIND}${atLeast(VALUE_CHAR, 1)}|${STRONG_LABEL_BEHIND}${atLeast(VALUE_CHAR, 4)}|(?!${NO_CREDENTIAL_VALUE}${UNQUOTED_VALUE_END})(?:(?=${PLAIN_VALUE_CHAR}{0,18}[0-9A-Z]${PLAIN_VALUE_CHAR}{0,18}[0-9A-Z])${atLeast(VALUE_CHAR, 8)}|${atLeast(VALUE_CHAR, 20)}))${VALUE_RUN_END}`,
|
|
123
|
+
].join("|");
|
|
124
|
+
const STRUCTURED_KEY_NAME = (() => {
|
|
125
|
+
const names = namesAt("structured-key");
|
|
126
|
+
const spelled = (name) => name.replaceAll("_", "[-_]?");
|
|
127
|
+
const tails = names.filter((n) => !n.startsWith("_")).map(spelled);
|
|
128
|
+
const separated = names.filter((n) => n.startsWith("_")).map((n) => spelled(n.slice(1)));
|
|
129
|
+
return `(?:[a-z0-9_.-]{0,64}(?:${tails.join("|")})${separated.length === 0 ? "" : `|[a-z0-9_.-]{0,63}[_.-](?:${separated.join("|")})`})`;
|
|
130
|
+
})();
|
|
131
|
+
const WEBHOOK_HEAD = String.raw `https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/)`;
|
|
132
|
+
const WEBHOOK_PATH = atLeast(String.raw `[\w\/@.~-]`, 16);
|
|
133
|
+
const VALUE_FAMILY_ROWS = [
|
|
134
|
+
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: unheaded(`gh[opsur][-_]${VENDOR_BODY}`), strict: new RegExp(String.raw `(?:ghp|gho|ghu|ghs|ghr)_${atLeast("[A-Za-z0-9]", 36)}\b`), fragment: headed(residue("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8)), envValueHead: /gh[opusr]_/ },
|
|
135
|
+
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: unheaded(`github_pat[-_]${VENDOR_BODY}`), strict: new RegExp(String.raw `github_pat_${atLeast("[A-Za-z0-9_]", 60)}\b`), fragment: new RegExp(`github_pat_${atLeast("[A-Za-z0-9_]", 8)}`, "g"), envValueHead: /github_pat_/ },
|
|
136
|
+
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: new RegExp(String.raw `\bsk-(?:ant-(?:api03-)?|proj-)?${atLeast("[A-Za-z0-9_]", 20)}\b`), fragment: headed(residue("sk-", "[A-Za-z0-9_-]", 6)), envValueHead: /sk-/ },
|
|
137
|
+
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: new RegExp(String.raw `\b[sr]k_(?:live|test|org)_${atLeast("[A-Za-z0-9]", 24)}\b`), fragment: headed(residue("(?:sk|rk)_(?:live|test|org)_", "[A-Za-z0-9_-]", 8)) },
|
|
138
|
+
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`), envValueHead: /pk-/ },
|
|
139
|
+
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: unheaded(`${GITLAB_HEAD}${atLeast("[A-Za-z0-9_-]", 20)}`), strict: new RegExp(String.raw `${GITLAB_HEAD}(?=[A-Za-z0-9_]*[0-9])${atLeast("[A-Za-z0-9_]", 20)}\b`), fragment: headed(residue(GITLAB_HEAD, "[A-Za-z0-9_-]", 8)), envValueHead: /glpat-/ },
|
|
140
|
+
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`npm_${atLeast("[A-Za-z0-9]", 36)}`), envValueHead: /npm_/ },
|
|
141
|
+
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.${atLeast("[A-Za-z0-9_.-]", 20)}`), envValueHead: /ya29\./ },
|
|
142
|
+
{ id: "pypi-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pypi-${atLeast("[A-Za-z0-9_-]", 85)}`), strict: new RegExp(String.raw `\bpypi-(?=[A-Za-z0-9_-]*[0-9])${atLeast("[A-Za-z0-9_-]", 85)}`), fragment: headed(residue("pypi-", "[A-Za-z0-9_-]", 8)) },
|
|
143
|
+
{ id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(`AIza${atLeast("[A-Za-z0-9_-]", 20)}`), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
|
144
|
+
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(`xox[baprse]-${atLeast("[A-Za-z0-9_-]", 10)}`), strict: new RegExp(String.raw `\bxox[baprse]-${repeated("(?:[A-Za-z0-9]+-)", 1, 3)}${atLeast("[A-Za-z0-9]", 16)}\b`), fragment: headed(residue("xox[baprse]-", "[A-Za-z0-9-]", 4)), envValueHead: /xox[abpr]-/ },
|
|
145
|
+
{ id: "slack-app-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(String.raw `xapp-\d-${atLeast("[A-Za-z0-9_-]", 20)}`), strict: new RegExp(String.raw `\bxapp-\d-(?=[A-Za-z0-9-]{20})${repeated("(?:[A-Za-z0-9]+-)", 1, 3)}(?=[A-Za-z0-9]*[0-9])${atLeast("[A-Za-z0-9]", 16)}\b`), fragment: headed(residue(String.raw `xapp-\d-`, "[A-Za-z0-9-]", 4)) },
|
|
146
|
+
{ id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(`(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)${atLeast("[0-9A-Z]", 12)}`), strict: /AKIA[0-9A-Z]{16}\b/, fragment: headed(`(?:${residue("AKIA", "[0-9A-Z]", 6)}|${residue("ASIA", "[0-9A-Z]", 8)})`), envValueHead: /AKIA/ },
|
|
147
|
+
{ id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(`hf_${atLeast("[A-Za-z0-9]", 20)}`), strict: new RegExp(String.raw `\bhf_(?=[A-Za-z0-9]*[0-9])${atLeast("[A-Za-z0-9]", 20)}\b`) },
|
|
148
|
+
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed(`eyJ${atLeast("[A-Za-z0-9._-]", 20)}`), strict: new RegExp(String.raw `\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.${atLeast("[A-Za-z0-9_-]", 16)}\b`), envValueHead: /eyJ/ },
|
|
149
|
+
{
|
|
150
|
+
id: "private-key",
|
|
151
|
+
kind: "private-key-block",
|
|
152
|
+
source: "arg-summary",
|
|
153
|
+
confidence: "high",
|
|
154
|
+
match: new RegExp(`${PEM_HEAD}[\\s\\S]*?(?:${PEM_FOOT}|$)|` +
|
|
155
|
+
String.raw `---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]*?(?:---- END SSH2 ENCRYPTED PRIVATE KEY ----|$)|PuTTY-User-Key-File-\d+:[\s\S]*?(?:Private-MAC:[ \t]*[0-9A-Fa-f]+|$)|Private-Lines:[ \t]*\d+(?:\r?\n[A-Za-z0-9+/=]+(?=\r?\n|$))+`, "g"),
|
|
156
|
+
strict: new RegExp(PEM_HEAD),
|
|
157
|
+
envValue: new RegExp(PEM_HEAD),
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
id: "bearer-token",
|
|
161
|
+
kind: "bearer-token",
|
|
162
|
+
source: "arg-summary",
|
|
163
|
+
confidence: "high",
|
|
164
|
+
match: new RegExp(`()(?:${WORD_START}${BEARER_WORD}${SCHEME_SEPARATOR}+|${GLUED_LEFT}${BEARER_WORD}${SCHEME_GLUE_GATE})${atLeast(TOKEN68_CHAR, CREDENTIAL_MIN_CHARS)}=*(?:${BEARER_TAIL})?`, "g"),
|
|
165
|
+
strict: new RegExp(String.raw `\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])${atLeast("[A-Za-z0-9._~+/-]", 20)}=*`),
|
|
166
|
+
envValue: new RegExp(String.raw `\b(?:Bearer|Basic)\s+(?=[A-Za-z._~+\/=-]*[0-9]|${repeated(String.raw `(?:[A-Za-z0-9._~+\/=-]*?[a-z][A-Z](?![a-z]))`, 2, 2)}|[A-Za-z0-9.-]*[_~+\/=])${atLeast(String.raw `[A-Za-z0-9._~+\/=-]`, CREDENTIAL_MIN_CHARS)}`),
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
id: "authorization-header",
|
|
170
|
+
kind: "authorization-header",
|
|
171
|
+
source: "arg-summary",
|
|
172
|
+
confidence: "high",
|
|
173
|
+
match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi"),
|
|
174
|
+
envValue: new RegExp(String.raw `\bauthorization\s*[:=]\s*["']?[a-z][a-z0-9_-]*\s+${atLeast(String.raw `[A-Za-z0-9._~+\/=-]`, CREDENTIAL_MIN_CHARS)}`, "i"),
|
|
175
|
+
},
|
|
176
|
+
{ id: "basic-credential", kind: "scheme-credential", source: "arg-summary", confidence: "high", match: new RegExp(`(?:${WORD_START}${BASIC_WORD}${SCHEME_SEPARATOR}+|${GLUED_LEFT}${BASIC_WORD}${SCHEME_GLUE_GATE})(?=[A-Za-z+/]*[0-9]|${repeated("(?:[A-Za-z0-9+/]*?[a-z][A-Z](?![a-z]))", 2, 2)}|[A-Za-z0-9]*[+/=]|${BASE64_COLON})${atLeast("[A-Za-z0-9+/]", CREDENTIAL_MIN_CHARS)}={0,2}`, "g") },
|
|
177
|
+
{
|
|
178
|
+
id: "token-scheme",
|
|
179
|
+
kind: "scheme-credential",
|
|
180
|
+
source: "arg-summary",
|
|
181
|
+
confidence: "high",
|
|
182
|
+
match: headed(String.raw `()(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=(${atLeast("[A-Za-z0-9_~+=-]", 20)}))\2(?![\/.])`),
|
|
183
|
+
envValue: new RegExp(String.raw `\b(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=[A-Za-z0-9_~+=-]*[A-Z])(?=(${atLeast("[A-Za-z0-9_~+=-]", 20)}))\1(?![\/.])`),
|
|
184
|
+
},
|
|
185
|
+
{ id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: new RegExp(`(${AZURE_FIELD})${AZURE_BODY}`, "g"), strict: new RegExp(`${AZURE_FIELD}${AZURE_BODY}`) },
|
|
186
|
+
{
|
|
187
|
+
id: "keyword-value",
|
|
188
|
+
kind: "keyword-value",
|
|
189
|
+
source: "arg-summary",
|
|
190
|
+
confidence: "medium",
|
|
191
|
+
match: new RegExp(`${KEYWORD_LABEL}(?:${KEYWORD_VALUE})`, "g"),
|
|
192
|
+
envValue: new RegExp(String.raw `(?:(?:^|[;&?#,{]|\/:)\s*|\s)["']?(?!-*jobserver-auth\s*[=:])${STRUCTURED_KEY_NAME}["']?\s*[=:]\s*["']?(?!(?:${FLAG_WORDS.join("|")})(?:$|[;&,\s"']))[^;&,\s"']+`, "i"),
|
|
193
|
+
},
|
|
194
|
+
{ id: "sonar-login", kind: "keyword-value", source: "arg-summary", confidence: "high", match: new RegExp(String.raw `(${KEY_QUOTE}sonar\.login${KEY_QUOTE}\s*[:=]\s*${KEY_QUOTE})${atLeast(String.raw `[^\s"',}]`, CREDENTIAL_MIN_CHARS)}${VALUE_RUN_END}`, "g"), envValue: new RegExp(String.raw `["']?sonar\.login["']?\s*[:=]\s*["']?${atLeast(String.raw `[^\s"',}]`, CREDENTIAL_MIN_CHARS)}`) },
|
|
195
|
+
{ id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${USERINFO_FRONT}|${USERINFO_WORD_START})((?<=[:/])${URL_USERINFO_SOURCE}|${USERINFO_WORD_START}${FRONTLESS_USERINFO})`, "gi") },
|
|
196
|
+
{
|
|
197
|
+
id: "webhook-url",
|
|
198
|
+
kind: "webhook-url",
|
|
199
|
+
source: "untrusted-egress",
|
|
200
|
+
confidence: "high",
|
|
201
|
+
match: new RegExp(`(${WEBHOOK_HEAD})${WEBHOOK_PATH}`, "gi"),
|
|
202
|
+
envValue: new RegExp(`${WEBHOOK_HEAD}${WEBHOOK_PATH}`, "i"),
|
|
203
|
+
},
|
|
204
|
+
{ id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: new RegExp(String.raw `([?&#](?:${namesAt("url-param").map(literally).join("|")})=)[^&\s"'#]+${VALUE_RUN_END}`, "gi") },
|
|
205
|
+
];
|
|
56
206
|
export const CREDENTIAL_VOCABULARY = {
|
|
57
|
-
values:
|
|
58
|
-
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/, fragment: headed(residue("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8)), envValueHead: /gh[opusr]_/ },
|
|
59
|
-
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/, fragment: /github_pat_[A-Za-z0-9_]{8,}/g, envValueHead: /github_pat_/ },
|
|
60
|
-
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/, fragment: headed(residue("sk-", "[A-Za-z0-9_-]", 6)), envValueHead: /sk-/ },
|
|
61
|
-
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test|org)_[A-Za-z0-9]{24,}\b/, fragment: headed(residue("(?:sk|rk)_(?:live|test|org)_", "[A-Za-z0-9_-]", 8)) },
|
|
62
|
-
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`), envValueHead: /pk-/ },
|
|
63
|
-
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`${GITLAB_HEAD}[A-Za-z0-9_-]{20,}`), strict: new RegExp(String.raw `\b${GITLAB_HEAD}(?=[A-Za-z0-9_]*[0-9])[A-Za-z0-9_]{20,}\b`), fragment: headed(residue(GITLAB_HEAD, "[A-Za-z0-9_-]", 8)), envValueHead: /glpat-/ },
|
|
64
|
-
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}"), envValueHead: /npm_/ },
|
|
65
|
-
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`), envValueHead: /ya29\./ },
|
|
66
|
-
{ id: "pypi-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("pypi-[A-Za-z0-9_-]{85,}"), strict: /\bpypi-(?=[A-Za-z0-9_-]*[0-9])[A-Za-z0-9_-]{85,}/, fragment: headed(residue("pypi-", "[A-Za-z0-9_-]", 8)) },
|
|
67
|
-
{ id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("AIza[A-Za-z0-9_-]{20,}"), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
|
68
|
-
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprse]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprse]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/, fragment: headed(residue("xox[baprse]-", "[A-Za-z0-9-]", 4)), envValueHead: /xox[abpr]-/ },
|
|
69
|
-
{ id: "slack-app-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(String.raw `xapp-\d-[A-Za-z0-9-]{20,}`), strict: /\bxapp-\d-(?=[A-Za-z0-9-]{20})(?:[A-Za-z0-9]+-){1,3}(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{16,}\b/, fragment: headed(residue(String.raw `xapp-\d-`, "[A-Za-z0-9-]", 4)) },
|
|
70
|
-
{ id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/, fragment: headed(`(?:${residue("AKIA", "[0-9A-Z]", 6)}|${residue("ASIA", "[0-9A-Z]", 8)})`), envValueHead: /AKIA/ },
|
|
71
|
-
{ id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("hf_[A-Za-z0-9]{20,}"), strict: /\bhf_(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{20,}\b/ },
|
|
72
|
-
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/, envValueHead: /eyJ/ },
|
|
73
|
-
{
|
|
74
|
-
id: "private-key",
|
|
75
|
-
kind: "private-key-block",
|
|
76
|
-
source: "arg-summary",
|
|
77
|
-
confidence: "high",
|
|
78
|
-
match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)|---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]*?(?:---- END SSH2 ENCRYPTED PRIVATE KEY ----|$)|PuTTY-User-Key-File-\d+:[\s\S]*?(?:Private-MAC:[ \t]*[0-9A-Fa-f]+|$)|Private-Lines:[ \t]*\d+(?:\r?\n[A-Za-z0-9+/=]+(?=\r?\n|$))+/g,
|
|
79
|
-
strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/,
|
|
80
|
-
envValue: /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/,
|
|
81
|
-
},
|
|
82
|
-
{
|
|
83
|
-
id: "bearer-token",
|
|
84
|
-
kind: "bearer-token",
|
|
85
|
-
source: "arg-summary",
|
|
86
|
-
confidence: "high",
|
|
87
|
-
match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`),
|
|
88
|
-
strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/,
|
|
89
|
-
envValue: new RegExp(String.raw `\b(?:Bearer|Basic)\s+(?=[A-Za-z._~+\/=-]*[0-9]|(?:[A-Za-z0-9._~+\/=-]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9.-]*[_~+\/=])[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`),
|
|
90
|
-
},
|
|
91
|
-
{
|
|
92
|
-
id: "authorization-header",
|
|
93
|
-
kind: "authorization-header",
|
|
94
|
-
source: "arg-summary",
|
|
95
|
-
confidence: "high",
|
|
96
|
-
match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi"),
|
|
97
|
-
envValue: new RegExp(String.raw `\bauthorization\s*[:=]\s*["']?[a-z][a-z0-9_-]*\s+[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`, "i"),
|
|
98
|
-
},
|
|
99
|
-
{ id: "basic-credential", kind: "scheme-credential", source: "arg-summary", confidence: "high", match: headed(String.raw `[Bb][Aa][Ss][Ii][Cc]\s+(?=[A-Za-z+/]*[0-9]|(?:[A-Za-z0-9+/]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9]*[+/=])[A-Za-z0-9+/]{${CREDENTIAL_MIN_CHARS},}={0,2}`) },
|
|
100
|
-
{
|
|
101
|
-
id: "token-scheme",
|
|
102
|
-
kind: "scheme-credential",
|
|
103
|
-
source: "arg-summary",
|
|
104
|
-
confidence: "high",
|
|
105
|
-
match: headed(String.raw `()(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=([A-Za-z0-9_~+=-]{20,}))\2(?![\/.])`),
|
|
106
|
-
envValue: /\b(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=[A-Za-z0-9_~+=-]*[A-Z])(?=([A-Za-z0-9_~+=-]{20,}))\1(?![\/.])/,
|
|
107
|
-
},
|
|
108
|
-
{ id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: new RegExp(`(${AZURE_FIELD})${AZURE_BODY}`, "g"), strict: new RegExp(`${AZURE_FIELD}${AZURE_BODY}`) },
|
|
109
|
-
{
|
|
110
|
-
id: "keyword-value",
|
|
111
|
-
kind: "keyword-value",
|
|
112
|
-
source: "arg-summary",
|
|
113
|
-
confidence: "medium",
|
|
114
|
-
match: new RegExp(String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "g"),
|
|
115
|
-
envValue: new RegExp(String.raw `(?:(?:^|[;&?#,{]|\/:)\s*|\s)["']?(?!-*jobserver-auth\s*[=:])(?:[a-z0-9_.-]{0,64}(?:password|passwd|pwd|secret|token|(?:account|access|api|private|subscription)[-_]?key|signature|sig|credential)|[a-z0-9_.-]{0,63}[_.-]auth)["']?\s*[=:]\s*["']?(?!(?:${FLAG_WORDS.join("|")})(?:$|[;&,\s"']))[^;&,\s"']+`, "i"),
|
|
116
|
-
},
|
|
117
|
-
{ id: "sonar-login", kind: "keyword-value", source: "arg-summary", confidence: "high", match: new RegExp(String.raw `(${KEY_QUOTE}sonar\.login${KEY_QUOTE}\s*[:=]\s*${KEY_QUOTE})[^\s"',}]{${CREDENTIAL_MIN_CHARS},}${VALUE_RUN_END}`, "g"), envValue: new RegExp(String.raw `["']?sonar\.login["']?\s*[:=]\s*["']?[^\s"',}]{${CREDENTIAL_MIN_CHARS},}`) },
|
|
118
|
-
{ id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${URI_SCHEME_SOURCE}:\/\/)(${URL_USERINFO_SOURCE})`, "gi") },
|
|
119
|
-
{
|
|
120
|
-
id: "webhook-url",
|
|
121
|
-
kind: "webhook-url",
|
|
122
|
-
source: "untrusted-egress",
|
|
123
|
-
confidence: "high",
|
|
124
|
-
match: /(https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/))[\w\/@.~-]{16,}/gi,
|
|
125
|
-
envValue: /https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/)[\w\/@.~-]{16,}/i,
|
|
126
|
-
},
|
|
127
|
-
{ id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: new RegExp(String.raw `([?&#](?:${namesAt("url-param").map(literally).join("|")})=)[^&\s"'#]+${VALUE_RUN_END}`, "gi") },
|
|
128
|
-
],
|
|
207
|
+
values: VALUE_FAMILY_ROWS,
|
|
129
208
|
words: [
|
|
130
209
|
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE", "KEY"].map((word) => ({ word, position: "leading" })),
|
|
131
210
|
{ word: "AUTH", position: "leading", recordKeys: ["AUTHORIZATION", "AUTHENTICATION", "OAUTH", "AUTHN", "AUTHZ"] },
|
|
@@ -173,6 +252,7 @@ export const CREDENTIAL_VOCABULARY = {
|
|
|
173
252
|
nonCredentialPrefix: /^AUTH0_/i,
|
|
174
253
|
names: CREDENTIAL_NAMES,
|
|
175
254
|
flagWords: FLAG_WORDS,
|
|
255
|
+
placeholderWords: PLACEHOLDER_WORDS,
|
|
176
256
|
};
|
|
177
257
|
function freezeTable(node) {
|
|
178
258
|
if (typeof node !== "object" || node === null || node instanceof RegExp || Object.isFrozen(node))
|
|
@@ -229,10 +309,13 @@ export function envNameView(vocabulary = ENGINE_VOCABULARY) {
|
|
|
229
309
|
}
|
|
230
310
|
export function envValueView(vocabulary = ENGINE_VOCABULARY) {
|
|
231
311
|
const heads = vocabulary.values.flatMap((row) => (row.envValueHead === undefined ? [] : [row.envValueHead.source]));
|
|
312
|
+
const userinfo = vocabulary.values.find((row) => row.id === "url-userinfo")?.match;
|
|
232
313
|
return {
|
|
233
314
|
forms: Object.freeze(vocabulary.values.flatMap((row) => (row.envValue === undefined ? [] : [Object.freeze({ family: row.id, re: withOwnRegexes(row.envValue) })]))),
|
|
234
315
|
tokenHead: new RegExp(heads.length === 0 ? "(?!)" : `^(?:${heads.join("|")})`),
|
|
235
316
|
flagWords: new Set(vocabulary.flagWords),
|
|
317
|
+
placeholderWords: new Set(vocabulary.placeholderWords),
|
|
318
|
+
urlUserinfo: userinfo === undefined ? /(?!)/g : new RegExp(userinfo.source, userinfo.flags.includes("g") ? userinfo.flags : `${userinfo.flags}g`),
|
|
236
319
|
};
|
|
237
320
|
}
|
|
238
321
|
export function splitCamelCase(name) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/primitives",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.67.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Shared low-level definitions of the sema engine, importable by any layer: the credential reference (closed set of kinds, one reader) and the secret-shape detection definition (one table, its matcher and its renderer, no layer's policy). Zero dependencies.",
|
|
6
6
|
"type": "module",
|
|
@@ -31,5 +31,5 @@
|
|
|
31
31
|
"scripts": {
|
|
32
32
|
"build": "tsc -p tsconfig.json && tsc -p tsconfig.json --emitDeclarationOnly --removeComments false"
|
|
33
33
|
},
|
|
34
|
-
"gitHead": "
|
|
34
|
+
"gitHead": "d59f845b5daf6d41297dd0433304787c2fa2eb49"
|
|
35
35
|
}
|