@sema-agent/primitives 7.63.0 → 7.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/dist/index.d.ts +3 -2
- package/dist/index.js +3 -2
- package/dist/secret-readings.d.ts +80 -0
- package/dist/secret-readings.js +184 -0
- package/dist/secret-scan.d.ts +23 -34
- package/dist/secret-scan.js +169 -91
- package/dist/secret-vocabulary.d.ts +55 -9
- package/dist/secret-vocabulary.js +102 -24
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -91,15 +91,19 @@ look a kind up in it only after `readCredentialRef` has accepted the value.
|
|
|
91
91
|
## Secret-shape detection
|
|
92
92
|
|
|
93
93
|
`CREDENTIAL_VOCABULARY` is the one table of what a credential looks like: value families (each row with a wide form, and
|
|
94
|
-
for some a strict form
|
|
94
|
+
for some a strict form, a fragment form — what is left of the credential when a text was cut inside its body — and the
|
|
95
|
+
form and token head an environment value is read by), the credential words a name is read by, the names that look like
|
|
96
|
+
credentials but are none, and `names`: the names that mark a credential in an assignment (`places: ["assignment"]`) or as a
|
|
97
|
+
URL query or fragment parameter (`"url-param"`).
|
|
95
98
|
Every layer that looks for a secret reads this table, so a shape added to it is found everywhere at once. The table holds
|
|
96
99
|
no layer's policy: which form is read, how a text is read first, how a URL's username is treated and which mark replaces
|
|
97
100
|
a hit are the caller's, passed in whole on every call. Nothing here has a default.
|
|
98
101
|
|
|
99
102
|
```ts
|
|
100
103
|
interface SecretScanPolicy {
|
|
101
|
-
grade: "strict" | "wide";
|
|
102
|
-
readings: SecretReading[]; // at least one of { kind: "raw" } (as written) / { kind: "format-free" } (format characters removed)
|
|
104
|
+
grade: "strict" | "wide" | "fragment"; // the strict forms (narrow: a hit may refuse a write), the wide ones (broad: a hit is withheld) or the fragment ones (a cut residue)
|
|
105
|
+
readings: SecretReading[]; // at least one of { kind: "raw" } (as written) / { kind: "format-free" } (format characters removed),
|
|
106
|
+
// and any of { kind: "percent-layers", depth: 1..8 } / { kind: "percent-boundary" } / { kind: "form-plus" }
|
|
103
107
|
urlUserinfo: "keep-username" | "whole"; // a URL userinfo: keep the user and withhold the password, or withhold all of it
|
|
104
108
|
onUnresolvedReading: "secret" | "ignore"; // what a reading that cannot read a text to its end answers
|
|
105
109
|
}
|
|
@@ -113,11 +117,13 @@ interface SecretNotation {
|
|
|
113
117
|
|
|
114
118
|
| Function | Answers |
|
|
115
119
|
| --- | --- |
|
|
116
|
-
| `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved
|
|
120
|
+
| `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved` (the percent layers ran out, under `onUnresolvedReading: "secret"`), `mapping-failed`; a hit in a derived text is reported over the written units it was read from |
|
|
117
121
|
| `isSecretShaped(text, policy)` | `scanSecrets` found a span or answered the whole text |
|
|
118
122
|
| `renderSecrets(text, policy, notation, { passes?, report? })` | `text` with every hit replaced by its mark; `passes` are the caller's own rules, run after the table's in the same pipeline; `report.findings` receives one finding per hit (`report.preexistingMarkers` is the caller's to set) |
|
|
119
123
|
| `isSensitiveRecordKey(key)` / `isSensitiveHeaderName(name)` | a record member / an HTTP header name names a credential |
|
|
120
|
-
| `envNameView()` | the readings of environment variable names (exact names, credential words,
|
|
124
|
+
| `envNameView()` | the readings of environment variable names (exact names, credential words, a namespace's names `anySegmentUnder(namespace)`) |
|
|
125
|
+
| `envValueView()` | the readings of environment values, off the rows: `forms` (each row's env value form, in table order), `tokenHead` (each row's token head, one anchored alternation), `flagWords` |
|
|
126
|
+
| `secretReadingViews(text, readings)` | the texts the readings derive — the written text first, each once — each with `toOriginalSpan(start, end)` (the whole written units a span was read from), and `unresolved` (the percent layers ran out while one more would still decode) |
|
|
121
127
|
|
|
122
128
|
A policy or a notation that is not whole (a missing or unknown member, an empty mark, a mark holding `$` before a digit,
|
|
123
129
|
a `byKind` / `byFamily` key no row has, two rows of one pass spelled differently) throws a `TypeError`. Both are read on
|
|
@@ -151,5 +157,5 @@ removed or re-kinded export is named as breaking in the `@sema-agent/core` relea
|
|
|
151
157
|
| --- | --- |
|
|
152
158
|
| Credential reference | `readCredentialRef` `CREDENTIAL_REF_KINDS` `CREDENTIAL_REF_TRUST` `CREDENTIAL_REF_TEXT_MAX` `CREDENTIAL_REF_MAP_MAX` `CREDENTIAL_REF_PROBLEM_REASONS` |
|
|
153
159
|
| Types | `CredentialRef` `CredentialRefKind` `CredentialRefRequestAdmission` `CredentialRefProblem` `CredentialRefProblemReason` `CredentialRefReading` |
|
|
154
|
-
| Secret-shape detection | `CREDENTIAL_VOCABULARY` `CREDENTIAL_MIN_CHARS` `URL_OF_ANY_SCHEME` `SECRET_READING_KINDS` `scanSecrets` `isSecretShaped` `renderSecrets` `isSensitiveRecordKey` `isSensitiveHeaderName` `envNameView` `splitCamelCase` |
|
|
155
|
-
| Types | `CredentialVocabulary` `ValueFamilyRow` `CredentialWordRow` `CredentialWordPosition` `EnvNameView` `RecordKeyView` `RedactionConfidence` `RedactionSource` `SecretGrade` `SecretReading` `SecretScanPolicy` `SecretNotation` `SecretScan` `SecretSpan` `RenderOptions` `RedactionPass` `RedactionFinding` `RedactionReport` `SummarizableFinding` |
|
|
160
|
+
| Secret-shape detection | `CREDENTIAL_VOCABULARY` `CREDENTIAL_MIN_CHARS` `URL_OF_ANY_SCHEME` `SECRET_READING_KINDS` `scanSecrets` `isSecretShaped` `renderSecrets` `isSensitiveRecordKey` `isSensitiveHeaderName` `envNameView` `envValueView` `secretReadingViews` `splitCamelCase` |
|
|
161
|
+
| Types | `CredentialVocabulary` `ValueFamilyRow` `CredentialWordRow` `CredentialWordPosition` `CredentialNameRow` `CredentialNamePlace` `EnvNameView` `EnvValueView` `RecordKeyView` `RedactionConfidence` `RedactionSource` `SecretGrade` `SecretReading` `SecretReadingKind` `SecretReadingView` `SecretReadingViews` `SecretScanPolicy` `SecretNotation` `SecretScan` `SecretSpan` `RenderOptions` `RedactionPass` `RedactionFinding` `RedactionReport` `SummarizableFinding` |
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
/** The package entry of `@sema-agent/primitives`; core imports the package by name, never these modules. */
|
|
2
2
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, type CredentialRef, type CredentialRefKind, type CredentialRefRequestAdmission, type CredentialRefProblem, type CredentialRefProblemReason, type CredentialRefReading, } from "./credential-ref.js";
|
|
3
|
-
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type EnvNameView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
|
|
4
|
-
export { SECRET_READING_KINDS,
|
|
3
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type CredentialNameRow, type CredentialNamePlace, type EnvNameView, type EnvValueView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
|
|
4
|
+
export { SECRET_READING_KINDS, secretReadingViews, type SecretReading, type SecretReadingKind, type SecretReadingView, type SecretReadingViews, } from "./secret-readings.js";
|
|
5
|
+
export { scanSecrets, isSecretShaped, renderSecrets, type SecretGrade, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, } from "./credential-ref.js";
|
|
2
|
-
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
|
|
3
|
-
export { SECRET_READING_KINDS,
|
|
2
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
|
|
3
|
+
export { SECRET_READING_KINDS, secretReadingViews, } from "./secret-readings.js";
|
|
4
|
+
export { scanSecrets, isSecretShaped, renderSecrets, } from "./secret-scan.js";
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The readings a text is read through before the vocabulary's forms run over it — a closed set, each reading named and
|
|
3
|
+
* each a pure function of the text:
|
|
4
|
+
* - CHARACTER readings decide how one text is scanned: `raw` (as written) and `format-free` (every format character
|
|
5
|
+
* `\p{Cf}` removed, its hits mapped back — the engine's own reading, secret-scan.ts);
|
|
6
|
+
* - TEXT readings derive more texts from the one written, each scanned the same way and each hit mapped back to the
|
|
7
|
+
* written text's coordinates: `percent-layers` (up to `depth` tolerant percent-decoded layers), `percent-boundary` (each
|
|
8
|
+
* run of `%HH` escapes read as UTF-8 in one go, so an encoded no-break or zero-width space is the delimiter it stands
|
|
9
|
+
* for) and `form-plus` (`+` read as a space, the form-urlencoded way).
|
|
10
|
+
* A view's characters each map back to the span of the written text they were read from: a decoded escape to its three
|
|
11
|
+
* characters, a code point decoded from a run to all of its escapes. A hit in a view maps back to the whole units it
|
|
12
|
+
* touches, so a mapped-back span never cuts an escape (fail-closed: more is withheld, never less).
|
|
13
|
+
*/
|
|
14
|
+
/** The readings, a closed set ({@link SecretReading}). */
|
|
15
|
+
export declare const SECRET_READING_KINDS: readonly ["raw", "format-free", "percent-layers", "percent-boundary", "form-plus"];
|
|
16
|
+
/** One reading's name. */
|
|
17
|
+
export type SecretReadingKind = (typeof SECRET_READING_KINDS)[number];
|
|
18
|
+
/** One reading, by its kind. `percent-layers` takes `depth`, the most decoded layers it reads (an integer 1 … 8); every
|
|
19
|
+
* other reading takes no parameter. */
|
|
20
|
+
export type SecretReading = {
|
|
21
|
+
readonly kind: "raw";
|
|
22
|
+
} | {
|
|
23
|
+
readonly kind: "format-free";
|
|
24
|
+
} | {
|
|
25
|
+
readonly kind: "percent-layers";
|
|
26
|
+
readonly depth: number;
|
|
27
|
+
} | {
|
|
28
|
+
readonly kind: "percent-boundary";
|
|
29
|
+
} | {
|
|
30
|
+
readonly kind: "form-plus";
|
|
31
|
+
};
|
|
32
|
+
/** The most percent-decoded layers a `percent-layers` reading may read. */
|
|
33
|
+
export declare const PERCENT_LAYERS_MAX_DEPTH = 8;
|
|
34
|
+
/** A readings list read once and checked whole: which character readings scan a text, and which text readings derive
|
|
35
|
+
* the texts scanned. `key` names what it reads as. */
|
|
36
|
+
export interface ReadingSet {
|
|
37
|
+
readonly raw: boolean;
|
|
38
|
+
readonly formatFree: boolean;
|
|
39
|
+
/** The `percent-layers` depth, when the reading is read. */
|
|
40
|
+
readonly percentLayers?: number | undefined;
|
|
41
|
+
readonly percentBoundary?: boolean | undefined;
|
|
42
|
+
readonly formPlus?: boolean | undefined;
|
|
43
|
+
}
|
|
44
|
+
/** `readings` checked whole and read as a set: a non-empty array of {@link SecretReading}, holding `raw` or `format-free`
|
|
45
|
+
* (a text reading derives texts; a character reading scans them), with one `percent-layers` at most (one depth). A
|
|
46
|
+
* parameterless reading listed twice is the same reading. Throws a `TypeError` naming the first fault. */
|
|
47
|
+
export declare function readReadings(readings: unknown, at?: string): ReadingSet;
|
|
48
|
+
/** Does the set read a text reading (one that derives texts beyond the written one)? */
|
|
49
|
+
export declare const readsTextReadings: (set: ReadingSet) => boolean;
|
|
50
|
+
/** A text derived by the text readings, and the way back. */
|
|
51
|
+
export interface SecretReadingView {
|
|
52
|
+
/** The derived text. */
|
|
53
|
+
readonly text: string;
|
|
54
|
+
/** A span of `text` [start, end) as the span of the written text it was read from: the whole units (a decoded escape, a
|
|
55
|
+
* code point decoded from a run) the span touches. An empty span maps to the written position it stands at. */
|
|
56
|
+
readonly toOriginalSpan: (start: number, end: number) => {
|
|
57
|
+
readonly start: number;
|
|
58
|
+
readonly end: number;
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
/** What {@link secretReadingViews} returns: the views, the written text first, each text once (first seen first), and
|
|
62
|
+
* whether the percent layers ran out while one more layer would still decode something. */
|
|
63
|
+
export interface SecretReadingViews {
|
|
64
|
+
readonly views: readonly SecretReadingView[];
|
|
65
|
+
readonly unresolved: boolean;
|
|
66
|
+
}
|
|
67
|
+
/** `text` through one {@link percentDecodeLayer} (the same tolerant layer the `percent-layers` reading reads), without
|
|
68
|
+
* the way back. */
|
|
69
|
+
export declare function percentDecodedOnce(text: string): string;
|
|
70
|
+
/** The views a checked set of readings derives from `text` (the written text first, each text once) — the engine's entry;
|
|
71
|
+
* {@link secretReadingViews} is the checked one. */
|
|
72
|
+
export declare function readingViews(text: string, set: ReadingSet): SecretReadingViews;
|
|
73
|
+
/**
|
|
74
|
+
* The texts `readings` derive from `text` and the way back from each: the written text first, then its percent-decoded
|
|
75
|
+
* layers (`percent-layers`, up to `depth`, stopping at the first layer that decodes nothing), each of those read the
|
|
76
|
+
* boundary way (`percent-boundary`), and each of those with `+` read as a space (`form-plus`) — each text once, first seen
|
|
77
|
+
* first. `unresolved` = the layers ran out while one more would still decode something. The character readings (`raw`,
|
|
78
|
+
* `format-free`) derive no text: they say how each text is scanned. Throws a `TypeError` on readings that are not whole.
|
|
79
|
+
*/
|
|
80
|
+
export declare function secretReadingViews(text: string, readings: readonly SecretReading[]): SecretReadingViews;
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
export const SECRET_READING_KINDS = Object.freeze(["raw", "format-free", "percent-layers", "percent-boundary", "form-plus"]);
|
|
2
|
+
export const PERCENT_LAYERS_MAX_DEPTH = 8;
|
|
3
|
+
function fail(what) {
|
|
4
|
+
throw new TypeError(`secret scan: ${what}`);
|
|
5
|
+
}
|
|
6
|
+
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
7
|
+
export function readReadings(readings, at = "policy.readings") {
|
|
8
|
+
if (!Array.isArray(readings) || readings.length === 0)
|
|
9
|
+
fail(`${at} must be a non-empty array`);
|
|
10
|
+
const kinds = new Set();
|
|
11
|
+
let depth;
|
|
12
|
+
const list = readings;
|
|
13
|
+
for (const entry of list) {
|
|
14
|
+
const kind = isRecord(entry) ? SECRET_READING_KINDS.find((k) => k === entry.kind) : undefined;
|
|
15
|
+
if (kind === undefined || !isRecord(entry))
|
|
16
|
+
fail(`every ${at} entry is { kind } with kind one of ${SECRET_READING_KINDS.join(" / ")}`);
|
|
17
|
+
const members = kind === "percent-layers" ? ["kind", "depth"] : ["kind"];
|
|
18
|
+
if (Object.keys(entry).some((k) => !members.includes(k)))
|
|
19
|
+
fail(kind === "percent-layers" ? `the reading "percent-layers" takes \`depth\` only` : `the reading "${kind}" takes no parameter`);
|
|
20
|
+
if (kind === "percent-layers") {
|
|
21
|
+
if (depth !== undefined)
|
|
22
|
+
fail(`${at} lists "percent-layers" twice (one depth is read)`);
|
|
23
|
+
const d = entry.depth;
|
|
24
|
+
if (typeof d !== "number" || !Number.isInteger(d) || d < 1 || d > PERCENT_LAYERS_MAX_DEPTH)
|
|
25
|
+
fail(`the reading "percent-layers" takes \`depth\`, an integer 1 … ${PERCENT_LAYERS_MAX_DEPTH}`);
|
|
26
|
+
depth = d;
|
|
27
|
+
}
|
|
28
|
+
kinds.add(kind);
|
|
29
|
+
}
|
|
30
|
+
if (!kinds.has("raw") && !kinds.has("format-free"))
|
|
31
|
+
fail(`${at} must hold "raw" or "format-free" (a text reading derives texts; one of those two scans them)`);
|
|
32
|
+
return { raw: kinds.has("raw"), formatFree: kinds.has("format-free"), percentLayers: depth, percentBoundary: kinds.has("percent-boundary"), formPlus: kinds.has("form-plus") };
|
|
33
|
+
}
|
|
34
|
+
export const readsTextReadings = (set) => set.percentLayers !== undefined || set.percentBoundary === true || set.formPlus === true;
|
|
35
|
+
const fromOf = (t, i) => (t.from === undefined ? i : t.from[i]);
|
|
36
|
+
const toOf = (t, i) => (t.to === undefined ? i + 1 : t.to[i]);
|
|
37
|
+
function derive(src, build) {
|
|
38
|
+
let text = "";
|
|
39
|
+
const from = [];
|
|
40
|
+
const to = [];
|
|
41
|
+
build((chars, a, b) => {
|
|
42
|
+
const start = fromOf(src, a);
|
|
43
|
+
const end = toOf(src, b - 1);
|
|
44
|
+
for (let k = 0; k < chars.length; k++) {
|
|
45
|
+
from.push(start);
|
|
46
|
+
to.push(end);
|
|
47
|
+
}
|
|
48
|
+
text += chars;
|
|
49
|
+
});
|
|
50
|
+
return { text, from: Int32Array.from(from), to: Int32Array.from(to) };
|
|
51
|
+
}
|
|
52
|
+
function percentDecodeLayer(src) {
|
|
53
|
+
if (src.text.indexOf("%") === -1)
|
|
54
|
+
return src;
|
|
55
|
+
const re = /%([0-7][0-9A-Fa-f])/g;
|
|
56
|
+
if (!re.test(src.text))
|
|
57
|
+
return src;
|
|
58
|
+
re.lastIndex = 0;
|
|
59
|
+
return derive(src, (push) => {
|
|
60
|
+
let cursor = 0;
|
|
61
|
+
for (let m = re.exec(src.text); m !== null; m = re.exec(src.text)) {
|
|
62
|
+
for (let i = cursor; i < m.index; i++)
|
|
63
|
+
push(src.text[i], i, i + 1);
|
|
64
|
+
push(String.fromCharCode(parseInt(m[1], 16)), m.index, m.index + 3);
|
|
65
|
+
cursor = m.index + 3;
|
|
66
|
+
}
|
|
67
|
+
for (let i = cursor; i < src.text.length; i++)
|
|
68
|
+
push(src.text[i], i, i + 1);
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
export function percentDecodedOnce(text) {
|
|
72
|
+
return percentDecodeLayer({ text }).text;
|
|
73
|
+
}
|
|
74
|
+
function utf8CodePointLengths(bytes) {
|
|
75
|
+
const lengths = [];
|
|
76
|
+
for (let i = 0; i < bytes.length;) {
|
|
77
|
+
const lead = bytes[i];
|
|
78
|
+
if (lead < 0x80) {
|
|
79
|
+
lengths.push(1);
|
|
80
|
+
i += 1;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
const shape = lead >= 0xc2 && lead <= 0xdf ? [1, 0x80, 0xbf]
|
|
84
|
+
: lead === 0xe0 ? [2, 0xa0, 0xbf]
|
|
85
|
+
: (lead >= 0xe1 && lead <= 0xec) || lead === 0xee || lead === 0xef ? [2, 0x80, 0xbf]
|
|
86
|
+
: lead === 0xed ? [2, 0x80, 0x9f]
|
|
87
|
+
: lead === 0xf0 ? [3, 0x90, 0xbf]
|
|
88
|
+
: lead >= 0xf1 && lead <= 0xf3 ? [3, 0x80, 0xbf]
|
|
89
|
+
: lead === 0xf4 ? [3, 0x80, 0x8f]
|
|
90
|
+
: undefined;
|
|
91
|
+
if (shape === undefined)
|
|
92
|
+
return undefined;
|
|
93
|
+
const [need, lo, hi] = shape;
|
|
94
|
+
for (let k = 1; k <= need; k++) {
|
|
95
|
+
const c = bytes[i + k];
|
|
96
|
+
if (c === undefined || c < (k === 1 ? lo : 0x80) || c > (k === 1 ? hi : 0xbf))
|
|
97
|
+
return undefined;
|
|
98
|
+
}
|
|
99
|
+
lengths.push(need + 1);
|
|
100
|
+
i += need + 1;
|
|
101
|
+
}
|
|
102
|
+
return lengths;
|
|
103
|
+
}
|
|
104
|
+
function boundaryReading(src) {
|
|
105
|
+
if (src.text.indexOf("%") === -1)
|
|
106
|
+
return src;
|
|
107
|
+
const re = /(?:%[0-9A-Fa-f]{2})+/g;
|
|
108
|
+
if (!re.test(src.text))
|
|
109
|
+
return src;
|
|
110
|
+
re.lastIndex = 0;
|
|
111
|
+
return derive(src, (push) => {
|
|
112
|
+
let cursor = 0;
|
|
113
|
+
for (let m = re.exec(src.text); m !== null; m = re.exec(src.text)) {
|
|
114
|
+
for (let i = cursor; i < m.index; i++)
|
|
115
|
+
push(src.text[i], i, i + 1);
|
|
116
|
+
const run = m[0];
|
|
117
|
+
const bytes = [];
|
|
118
|
+
for (let i = 0; i < run.length; i += 3)
|
|
119
|
+
bytes.push(parseInt(run.slice(i + 1, i + 3), 16));
|
|
120
|
+
const lengths = utf8CodePointLengths(bytes);
|
|
121
|
+
if (lengths === undefined)
|
|
122
|
+
bytes.forEach((b, k) => push(b < 0x80 ? String.fromCharCode(b) : " ", m.index + 3 * k, m.index + 3 * k + 3));
|
|
123
|
+
else {
|
|
124
|
+
let k = 0;
|
|
125
|
+
for (const n of lengths) {
|
|
126
|
+
const a = m.index + 3 * k;
|
|
127
|
+
push(decodeURIComponent(run.slice(3 * k, 3 * (k + n))), a, a + 3 * n);
|
|
128
|
+
k += n;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
cursor = m.index + run.length;
|
|
132
|
+
}
|
|
133
|
+
for (let i = cursor; i < src.text.length; i++)
|
|
134
|
+
push(src.text[i], i, i + 1);
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
const formPlus = (src) => (src.text.indexOf("+") === -1 ? src : { text: src.text.replace(/\+/g, " "), from: src.from, to: src.to });
|
|
138
|
+
function viewOf(t, writtenLength) {
|
|
139
|
+
return {
|
|
140
|
+
text: t.text,
|
|
141
|
+
toOriginalSpan: (start, end) => {
|
|
142
|
+
if (end <= start) {
|
|
143
|
+
const at = start >= t.text.length ? writtenLength : fromOf(t, start);
|
|
144
|
+
return { start: at, end: at };
|
|
145
|
+
}
|
|
146
|
+
let lo = Infinity;
|
|
147
|
+
let hi = -Infinity;
|
|
148
|
+
for (let i = start; i < end; i++) {
|
|
149
|
+
lo = Math.min(lo, fromOf(t, i));
|
|
150
|
+
hi = Math.max(hi, toOf(t, i));
|
|
151
|
+
}
|
|
152
|
+
return { start: lo, end: hi };
|
|
153
|
+
},
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
export function readingViews(text, set) {
|
|
157
|
+
let texts = [{ text }];
|
|
158
|
+
let unresolved = false;
|
|
159
|
+
if (set.percentLayers !== undefined) {
|
|
160
|
+
let cur = texts[0];
|
|
161
|
+
for (let i = 0; i < set.percentLayers; i++) {
|
|
162
|
+
const next = percentDecodeLayer(cur);
|
|
163
|
+
if (next === cur)
|
|
164
|
+
break;
|
|
165
|
+
texts.push(next);
|
|
166
|
+
cur = next;
|
|
167
|
+
}
|
|
168
|
+
unresolved = texts.length === set.percentLayers + 1 && percentDecodeLayer(cur) !== cur;
|
|
169
|
+
}
|
|
170
|
+
const once = (list) => {
|
|
171
|
+
const seen = new Set();
|
|
172
|
+
return list.filter((t) => (seen.has(t.text) ? false : (seen.add(t.text), true)));
|
|
173
|
+
};
|
|
174
|
+
if (set.percentBoundary === true)
|
|
175
|
+
texts = once([...texts, ...texts.map(boundaryReading)]);
|
|
176
|
+
if (set.formPlus === true)
|
|
177
|
+
texts = once([...texts, ...texts.map(formPlus)]);
|
|
178
|
+
return { views: once(texts).map((t) => viewOf(t, text.length)), unresolved };
|
|
179
|
+
}
|
|
180
|
+
export function secretReadingViews(text, readings) {
|
|
181
|
+
if (typeof text !== "string")
|
|
182
|
+
fail("the text is not a string");
|
|
183
|
+
return readingViews(text, readReadings(readings, "readings"));
|
|
184
|
+
}
|
package/dist/secret-scan.d.ts
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
* edits (best-effort where redactions overlap — a span landing inside an earlier replacement anchors to that edit's start).
|
|
18
18
|
*/
|
|
19
19
|
import { type RedactionConfidence, type RedactionSource, type ValueFamilyRow } from "./secret-vocabulary.js";
|
|
20
|
+
import { type ReadingSet, type SecretReading } from "./secret-readings.js";
|
|
20
21
|
/** One redaction hit: what was detected, where (ORIGINAL-input coordinates), how confidently, the marker
|
|
21
22
|
* literal written in its place, which rule table minted it, and — present when a vocabulary row minted it — the
|
|
22
23
|
* value family (the row's stable id; a host-leak hit such as a URL or a path names none). */
|
|
@@ -59,53 +60,40 @@ export interface RedactionPass {
|
|
|
59
60
|
* matches at the hit's offset in the pass input. Absent = the pass is no vocabulary row. */
|
|
60
61
|
family?: string | ((input: string, at: number) => string);
|
|
61
62
|
}
|
|
62
|
-
/**
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
readonly formatFree: boolean;
|
|
66
|
-
}
|
|
67
|
-
/** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings,
|
|
68
|
-
* and the optional report. */
|
|
63
|
+
/** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings
|
|
64
|
+
* (the policy's, as a set), what a text the percent layers cannot read to its end answers (the policy's
|
|
65
|
+
* `onUnresolvedReading`; read only when the readings hold `percent-layers`), and the optional report. */
|
|
69
66
|
export interface EngineRun {
|
|
70
67
|
readonly overStrip: string;
|
|
71
68
|
readonly readings: ReadingSet;
|
|
69
|
+
readonly onUnresolvedReading?: SecretScanPolicy["onUnresolvedReading"] | undefined;
|
|
72
70
|
readonly report?: RedactionReport | undefined;
|
|
73
71
|
}
|
|
74
72
|
/**
|
|
75
73
|
* Run the declared passes in order, collecting findings (original-input spans) when a report is given.
|
|
76
74
|
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
* later pass that matched across an earlier pass's inserted text is rendered from the removals instead), so the spans
|
|
84
|
-
* always map back exactly and no scan is skipped.
|
|
75
|
+
* TEXT READINGS (`percent-layers` / `percent-boundary` / `form-plus`): the written text is rendered as below, and every
|
|
76
|
+
* text the readings derive from it ({@link readingViews}) is read by the same passes through the same character readings;
|
|
77
|
+
* each place a derived text withholds maps back to the written text (the whole escapes it touches) and joins the written
|
|
78
|
+
* text's own removals. A derived text that withholds nothing the written one did not leaves the output — findings and
|
|
79
|
+
* all — the written text's. A text the percent layers cannot read to its end is replaced whole when the run says
|
|
80
|
+
* `onUnresolvedReading: "secret"`.
|
|
85
81
|
*/
|
|
86
82
|
export declare function runRedactionPasses(input: string, passes: readonly RedactionPass[], run: EngineRun): string;
|
|
87
|
-
/** Which form of a row a reader reads: `strict` (refuse-write — narrow, a hit fail-closed refuses)
|
|
88
|
-
* broad, a hit is withheld)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
* format character (`\p{Cf}`) removed, its hits mapped back, plus a scan from every place a format character stood (so a
|
|
92
|
-
* format character neither splits a credential nor hides its start). */
|
|
93
|
-
export declare const SECRET_READING_KINDS: readonly ["raw", "format-free"];
|
|
94
|
-
/** One reading, by its kind. A reading that takes a parameter carries it as a member beside `kind`. */
|
|
95
|
-
export type SecretReading = {
|
|
96
|
-
readonly kind: "raw";
|
|
97
|
-
} | {
|
|
98
|
-
readonly kind: "format-free";
|
|
99
|
-
};
|
|
83
|
+
/** Which form of a row a reader reads: `strict` (refuse-write — narrow, a hit fail-closed refuses), `wide` (egress —
|
|
84
|
+
* broad, a hit is withheld) or `fragment` (what is left of a credential a cut shortened — a vendor prefix and a residue
|
|
85
|
+
* that still holds two digits or capitals; the rows that carry one). */
|
|
86
|
+
export type SecretGrade = "strict" | "wide" | "fragment";
|
|
100
87
|
/** What a caller reads the vocabulary for — every member required, none has a default here:
|
|
101
88
|
* - `grade` — which form of each row is read ({@link SecretGrade});
|
|
102
|
-
* - `readings` — the readings a text is read through
|
|
103
|
-
* reading are joined;
|
|
89
|
+
* - `readings` — the readings a text is read through (secret-readings.ts): `raw` or `format-free` at least, which scan a
|
|
90
|
+
* text, and any of the text readings, which derive more texts to scan; the hits of every reading are joined;
|
|
104
91
|
* - `urlUserinfo` — a URL userinfo's rendering: `keep-username` keeps the user and withholds the password (a bare user is
|
|
105
92
|
* kept below 20 characters and withheld at or past it; a userinfo holding a second `@` is withheld whole), `whole`
|
|
106
93
|
* withholds the whole userinfo;
|
|
107
94
|
* - `onUnresolvedReading` — what a reading that cannot read a text to its end answers: `secret` (the whole text) or
|
|
108
|
-
* `ignore` (what it did read).
|
|
95
|
+
* `ignore` (what it did read). Only `percent-layers` can run out (its depth reached while one more layer would still
|
|
96
|
+
* decode something); every other reading reads to the end. */
|
|
109
97
|
export interface SecretScanPolicy {
|
|
110
98
|
readonly grade: SecretGrade;
|
|
111
99
|
readonly readings: readonly SecretReading[];
|
|
@@ -165,12 +153,13 @@ export declare function egressPasses(groups: ReadonlyArray<readonly ValueFamilyR
|
|
|
165
153
|
export declare function renderSecrets(text: string, policy: SecretScanPolicy, notation: SecretNotation, options?: RenderOptions): string;
|
|
166
154
|
/**
|
|
167
155
|
* Where the vocabulary sees a credential in `text` under `policy`: every hit of the grade's form of every row, on every
|
|
168
|
-
* reading of the policy
|
|
169
|
-
*
|
|
156
|
+
* reading of the policy — each text the text readings derive, scanned by the character readings — as the hit's withheld
|
|
157
|
+
* part in `text`'s own coordinates (the whole written units a derived hit was read from; deduplicated, sorted by start) —
|
|
158
|
+
* or the whole text, when a reading cannot be afforded (more than 64 places where format characters stood: `scan-budget`)
|
|
159
|
+
* or the percent layers ran out before the text did under `onUnresolvedReading: "secret"` (`reading-unresolved`). The
|
|
170
160
|
* matcher answers where a credential is; the renderer additionally leaves a hit that already reads as its own rendering
|
|
171
161
|
* (a value that is already a mark), which only a notation can tell. Throws a `TypeError` on a policy that is not whole.
|
|
172
162
|
*/
|
|
173
163
|
export declare function scanSecrets(text: string, policy: SecretScanPolicy): SecretScan;
|
|
174
164
|
/** Does the vocabulary see a credential in `text` under `policy`? {@link scanSecrets} answers the whole text or a span. */
|
|
175
165
|
export declare function isSecretShaped(text: string, policy: SecretScanPolicy): boolean;
|
|
176
|
-
export {};
|
package/dist/secret-scan.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ENGINE_VOCABULARY, egressPassGroups } from "./secret-vocabulary.js";
|
|
2
|
+
import { percentDecodedOnce, readReadings, readingViews, readsTextReadings } from "./secret-readings.js";
|
|
2
3
|
function passZeroMapper(batches) {
|
|
3
4
|
const cursors = batches.map(() => ({ i: 0, delta: 0 }));
|
|
4
5
|
return (pos) => {
|
|
@@ -70,6 +71,10 @@ function removedRegion(pass, match, groups, inserted) {
|
|
|
70
71
|
tail++;
|
|
71
72
|
return [head, match.length - tail];
|
|
72
73
|
}
|
|
74
|
+
function appendAll(into, items) {
|
|
75
|
+
for (const item of items)
|
|
76
|
+
into.push(item);
|
|
77
|
+
}
|
|
73
78
|
const SOLO_SCAN_WORK_PER_CHAR = 8;
|
|
74
79
|
const SOLO_SCAN_WORK_FLOOR = 1 << 17;
|
|
75
80
|
function soloHits(input, passes, overStrip) {
|
|
@@ -135,7 +140,8 @@ function sequentialRemovals(batches) {
|
|
|
135
140
|
for (const e of batches[j]) {
|
|
136
141
|
const start = toPassZero(e.at + e.head);
|
|
137
142
|
const end = toPassZero(e.at + e.removedLen - e.tail);
|
|
138
|
-
|
|
143
|
+
const marker = e.inserted.slice(e.head, e.insertedLen - e.tail);
|
|
144
|
+
removals.push({ start, end: Math.max(start, end), marker, pass: e.pass, family: e.family, matchStart: toPassZero(e.at), matchEnd: toPassZero(e.at + e.removedLen) });
|
|
139
145
|
}
|
|
140
146
|
}
|
|
141
147
|
return removals;
|
|
@@ -149,7 +155,7 @@ function renderRemovals(input, removals, overStrip) {
|
|
|
149
155
|
const start = sorted[i].start;
|
|
150
156
|
let end = sorted[i].end;
|
|
151
157
|
let widest;
|
|
152
|
-
for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].marker === ""
|
|
158
|
+
for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].start === end && (sorted[i].marker === "" || sorted[i].marker === widest?.marker))); i++) {
|
|
153
159
|
const r = sorted[i];
|
|
154
160
|
if (r.end > end)
|
|
155
161
|
end = r.end;
|
|
@@ -198,7 +204,7 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
|
|
|
198
204
|
const whole = outer.length === 0 && mappable ? undefined : renderRemovals(input, [...sequential, ...outer], overStrip);
|
|
199
205
|
if (whole === undefined || (whole.out === plain.out && mappable)) {
|
|
200
206
|
if (report !== undefined)
|
|
201
|
-
report.findings
|
|
207
|
+
appendAll(report.findings, found.map((x) => x.f));
|
|
202
208
|
return plain;
|
|
203
209
|
}
|
|
204
210
|
if (report !== undefined) {
|
|
@@ -212,7 +218,7 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
|
|
|
212
218
|
found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family) });
|
|
213
219
|
}
|
|
214
220
|
found.sort((x, y) => x.pass - y.pass || x.f.span[0] - y.f.span[0]);
|
|
215
|
-
report.findings
|
|
221
|
+
appendAll(report.findings, found.map((x) => x.f));
|
|
216
222
|
}
|
|
217
223
|
return { out: whole.out, batches: [whole.edits] };
|
|
218
224
|
}
|
|
@@ -283,75 +289,131 @@ function readFormatFree(input) {
|
|
|
283
289
|
}
|
|
284
290
|
export function runRedactionPasses(input, passes, run) {
|
|
285
291
|
const { overStrip, readings, report } = run;
|
|
286
|
-
if (!readings
|
|
287
|
-
return
|
|
288
|
-
const
|
|
289
|
-
|
|
290
|
-
const { batches } = applyPasses(view, passes, overStrip, report, reading.toOriginal, true);
|
|
291
|
-
const viewSpans = replacedSpans(batches);
|
|
292
|
-
if (viewSpans === undefined)
|
|
292
|
+
if (!readsTextReadings(readings))
|
|
293
|
+
return renderOneText(input, passes, overStrip, readings, report).out;
|
|
294
|
+
const { views, unresolved } = readingViews(input, readings);
|
|
295
|
+
if (unresolved && run.onUnresolvedReading === "secret")
|
|
293
296
|
return overStrip;
|
|
294
|
-
const
|
|
295
|
-
if (
|
|
297
|
+
const base = renderOneText(input, passes, overStrip, readings, report);
|
|
298
|
+
if (views.length === 1)
|
|
299
|
+
return base.out;
|
|
300
|
+
const removals = base.removals();
|
|
301
|
+
if (removals === undefined)
|
|
296
302
|
return overStrip;
|
|
297
303
|
const extra = [];
|
|
298
|
-
|
|
299
|
-
const
|
|
300
|
-
if (
|
|
301
|
-
|
|
304
|
+
for (const view of views.slice(1)) {
|
|
305
|
+
const found = textRegions(view, passes, overStrip, readings);
|
|
306
|
+
if (found === undefined)
|
|
307
|
+
return overStrip;
|
|
308
|
+
appendAll(extra, found);
|
|
302
309
|
}
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
+
const seen = new Set();
|
|
311
|
+
const added = coveredNot(extra, removals).filter((r) => {
|
|
312
|
+
const key = `${r.pass}:${r.start}:${r.end}`;
|
|
313
|
+
return seen.has(key) ? false : (seen.add(key), true);
|
|
314
|
+
});
|
|
315
|
+
if (added.length === 0)
|
|
316
|
+
return base.out;
|
|
317
|
+
if (report !== undefined)
|
|
318
|
+
for (const r of added)
|
|
319
|
+
report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
|
|
320
|
+
return renderRemovals(input, [...removals, ...added], overStrip).out;
|
|
321
|
+
}
|
|
322
|
+
function textRegions(view, passes, overStrip, readings) {
|
|
323
|
+
const regions = [];
|
|
324
|
+
const read = (text, back) => {
|
|
325
|
+
const run = applyPasses(text, passes, overStrip, undefined, (pos) => pos);
|
|
326
|
+
for (const r of [...sequentialRemovals(run.batches), ...soloHits(text, passes, overStrip)]) {
|
|
327
|
+
if (r.end <= r.start)
|
|
328
|
+
continue;
|
|
329
|
+
const inView = back(r.start, r.end);
|
|
330
|
+
const written = view.toOriginalSpan(inView.start, inView.end);
|
|
331
|
+
regions.push({ start: written.start, end: written.end, marker: r.marker, pass: r.pass, family: r.family });
|
|
310
332
|
}
|
|
333
|
+
};
|
|
334
|
+
const asWritten = (start, end) => ({ start, end });
|
|
335
|
+
const text = view.text;
|
|
336
|
+
if (readings.raw || !FORMAT_CHAR_RE.test(text))
|
|
337
|
+
read(text, asWritten);
|
|
338
|
+
if (readings.formatFree && FORMAT_CHAR_RE.test(text)) {
|
|
339
|
+
const reading = readFormatFree(text);
|
|
340
|
+
if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
|
|
341
|
+
return undefined;
|
|
342
|
+
read(reading.view, reading.toOriginalSpan);
|
|
343
|
+
for (const cut of reading.cuts)
|
|
344
|
+
read(reading.view.slice(cut), (start, end) => reading.toOriginalSpan(start + cut, end + cut));
|
|
311
345
|
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
346
|
+
return regions;
|
|
347
|
+
}
|
|
348
|
+
function coveredNot(regions, removals) {
|
|
349
|
+
return unremovedHits(regions, removals);
|
|
350
|
+
}
|
|
351
|
+
function renderOneText(input, passes, overStrip, readings, report) {
|
|
352
|
+
const whole = { out: overStrip, removals: () => undefined };
|
|
353
|
+
if (!readings.formatFree || !FORMAT_CHAR_RE.test(input)) {
|
|
354
|
+
const plain = applyPasses(input, passes, overStrip, report, (pos) => pos);
|
|
355
|
+
return { out: plain.out, removals: () => sequentialRemovals(plain.batches) };
|
|
356
|
+
}
|
|
357
|
+
const reading = readFormatFree(input);
|
|
358
|
+
const { view } = reading;
|
|
359
|
+
const { batches } = applyPasses(view, passes, overStrip, report, reading.toOriginal, true);
|
|
360
|
+
if (replacedSpans(batches) === undefined)
|
|
361
|
+
return whole;
|
|
362
|
+
const back = (r, shift) => {
|
|
363
|
+
const kept = reading.toOriginalSpan(r.start + shift, r.end + shift);
|
|
364
|
+
const match = reading.toOriginalSpan(r.matchStart + shift, r.matchEnd + shift);
|
|
365
|
+
return { ...r, start: kept.start, end: kept.end, matchStart: match.start, matchEnd: match.end };
|
|
324
366
|
};
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
367
|
+
const removals = sequentialRemovals(batches).map((r) => back(r, 0));
|
|
368
|
+
if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
|
|
369
|
+
return whole;
|
|
370
|
+
const extra = [];
|
|
371
|
+
if (readings.raw)
|
|
372
|
+
appendAll(extra, sequentialRemovals(applyPasses(input, passes, overStrip, undefined, (pos) => pos, true).batches));
|
|
373
|
+
for (const cut of reading.cuts)
|
|
374
|
+
appendAll(extra, sequentialRemovals(applyPasses(view.slice(cut), passes, overStrip, undefined, (pos) => pos, true).batches).map((r) => back(r, cut)));
|
|
375
|
+
if (report !== undefined) {
|
|
376
|
+
const gone = [];
|
|
377
|
+
const firstEndingAfter = (pos) => {
|
|
378
|
+
let lo = 0;
|
|
379
|
+
let hi = gone.length;
|
|
380
|
+
while (lo < hi) {
|
|
381
|
+
const mid = (lo + hi) >> 1;
|
|
382
|
+
if (gone[mid][1] > pos)
|
|
383
|
+
hi = mid;
|
|
384
|
+
else
|
|
385
|
+
lo = mid + 1;
|
|
386
|
+
}
|
|
387
|
+
return lo;
|
|
388
|
+
};
|
|
389
|
+
const join = (r) => {
|
|
390
|
+
if (r.end <= r.start)
|
|
391
|
+
return false;
|
|
392
|
+
const at = firstEndingAfter(r.start);
|
|
393
|
+
const hit = gone[at];
|
|
394
|
+
if (hit === undefined || hit[0] >= r.end) {
|
|
395
|
+
gone.splice(at, 0, [r.start, r.end]);
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
hit[0] = Math.min(hit[0], r.start);
|
|
399
|
+
hit[1] = Math.max(hit[1], r.end);
|
|
333
400
|
let next = at + 1;
|
|
334
|
-
while (next <
|
|
335
|
-
hit
|
|
401
|
+
while (next < gone.length && gone[next][0] < hit[1])
|
|
402
|
+
hit[1] = Math.max(hit[1], gone[next++][1]);
|
|
336
403
|
if (next > at + 1)
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
let cursor = 0;
|
|
346
|
-
for (const sp of spans) {
|
|
347
|
-
result += input.slice(cursor, sp.start) + sp.inserted;
|
|
348
|
-
cursor = sp.end;
|
|
404
|
+
gone.splice(at + 1, next - at - 1);
|
|
405
|
+
return false;
|
|
406
|
+
};
|
|
407
|
+
for (const r of removals)
|
|
408
|
+
join(r);
|
|
409
|
+
for (const r of [...extra].sort((a, b) => a.start - b.start))
|
|
410
|
+
if (join(r))
|
|
411
|
+
report.findings.push(finding(passes[r.pass], [r.matchStart, r.matchEnd], r.family));
|
|
349
412
|
}
|
|
350
|
-
|
|
351
|
-
return
|
|
413
|
+
const all = [...removals, ...extra];
|
|
414
|
+
return { out: renderRemovals(input, all, overStrip).out, removals: () => all };
|
|
352
415
|
}
|
|
353
|
-
const SECRET_GRADES = ["strict", "wide"];
|
|
354
|
-
export const SECRET_READING_KINDS = Object.freeze(["raw", "format-free"]);
|
|
416
|
+
const SECRET_GRADES = ["strict", "wide", "fragment"];
|
|
355
417
|
function fail(what) {
|
|
356
418
|
throw new TypeError(`secret scan: ${what}`);
|
|
357
419
|
}
|
|
@@ -362,24 +424,12 @@ function readPolicy(policy) {
|
|
|
362
424
|
const { grade, readings, urlUserinfo, onUnresolvedReading } = policy;
|
|
363
425
|
if (!SECRET_GRADES.includes(grade))
|
|
364
426
|
fail(`policy.grade must be one of ${SECRET_GRADES.join(" / ")}`);
|
|
365
|
-
|
|
366
|
-
fail("policy.readings must be a non-empty array");
|
|
367
|
-
const kinds = new Set();
|
|
368
|
-
for (const r of readings) {
|
|
369
|
-
const named = isRecord(r) ? r.kind : undefined;
|
|
370
|
-
const kind = SECRET_READING_KINDS.find((k) => k === named);
|
|
371
|
-
if (kind === undefined)
|
|
372
|
-
fail(`every policy.readings entry is { kind } with kind one of ${SECRET_READING_KINDS.join(" / ")}`);
|
|
373
|
-
if (Object.keys(r).some((k) => k !== "kind"))
|
|
374
|
-
fail(`the reading "${kind}" takes no parameter`);
|
|
375
|
-
kinds.add(kind);
|
|
376
|
-
}
|
|
427
|
+
const set = readReadings(readings);
|
|
377
428
|
if (urlUserinfo !== "keep-username" && urlUserinfo !== "whole")
|
|
378
429
|
fail("policy.urlUserinfo must be keep-username / whole");
|
|
379
430
|
if (onUnresolvedReading !== "secret" && onUnresolvedReading !== "ignore")
|
|
380
431
|
fail("policy.onUnresolvedReading must be secret / ignore");
|
|
381
|
-
|
|
382
|
-
return { grade, readings: set, urlUserinfo, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
|
|
432
|
+
return { grade, readings: set, urlUserinfo, onUnresolvedReading, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
|
|
383
433
|
}
|
|
384
434
|
const ROW_IDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.id));
|
|
385
435
|
const ROW_KINDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.kind));
|
|
@@ -416,10 +466,19 @@ const USERINFO_FAMILY = "url-userinfo";
|
|
|
416
466
|
if (!ROW_IDS.has(USERINFO_FAMILY))
|
|
417
467
|
throw new Error(`secret scan: the vocabulary has no "${USERINFO_FAMILY}" row`);
|
|
418
468
|
const BARE_USERNAME_MAX_LEN = 20;
|
|
469
|
+
const DECODED_USERNAME_FORMS = ENGINE_VOCABULARY.values.map((row) => new RegExp(row.match.source, row.match.flags.replace("g", "")));
|
|
470
|
+
const readsDecodedAsCredential = (username) => {
|
|
471
|
+
if (!username.includes("%"))
|
|
472
|
+
return false;
|
|
473
|
+
const decoded = percentDecodedOnce(username);
|
|
474
|
+
return decoded !== username && DECODED_USERNAME_FORMS.some((re) => re.test(decoded));
|
|
475
|
+
};
|
|
419
476
|
function userinfoWithheld(userinfo, urlUserinfo) {
|
|
420
477
|
if (urlUserinfo === "whole" || userinfo.includes("@"))
|
|
421
478
|
return [0, userinfo.length];
|
|
422
479
|
const colonIdx = userinfo.indexOf(":");
|
|
480
|
+
if (readsDecodedAsCredential(colonIdx === -1 ? userinfo : userinfo.slice(0, colonIdx)))
|
|
481
|
+
return [0, userinfo.length];
|
|
423
482
|
if (colonIdx === -1)
|
|
424
483
|
return userinfo.length >= BARE_USERNAME_MAX_LEN ? [0, userinfo.length] : undefined;
|
|
425
484
|
return [colonIdx + 1, userinfo.length];
|
|
@@ -455,13 +514,21 @@ function strictPasses(markOf) {
|
|
|
455
514
|
return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: new RegExp(row.strict.source, `${row.strict.flags}g`), replace: marker, family: row.id }];
|
|
456
515
|
});
|
|
457
516
|
}
|
|
517
|
+
function fragmentPasses(markOf) {
|
|
518
|
+
return ENGINE_VOCABULARY.values.flatMap((row) => {
|
|
519
|
+
if (row.fragment === undefined)
|
|
520
|
+
return [];
|
|
521
|
+
const marker = markOf(row);
|
|
522
|
+
return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: row.fragment, replace: capturesLabel(row.fragment) ? `$1${marker}` : marker, family: row.id }];
|
|
523
|
+
});
|
|
524
|
+
}
|
|
458
525
|
const COMPILED_PASSES = new Map();
|
|
459
526
|
const COMPILED_PASSES_MAX = 64;
|
|
460
527
|
function tablePasses(read, marks) {
|
|
461
528
|
const key = `${read.key}\u0000${marks.key}`;
|
|
462
529
|
let passes = COMPILED_PASSES.get(key);
|
|
463
530
|
if (passes === undefined) {
|
|
464
|
-
passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), marks.markOf, read.urlUserinfo) : strictPasses(marks.markOf));
|
|
531
|
+
passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), marks.markOf, read.urlUserinfo) : read.grade === "strict" ? strictPasses(marks.markOf) : fragmentPasses(marks.markOf));
|
|
465
532
|
if (COMPILED_PASSES.size >= COMPILED_PASSES_MAX)
|
|
466
533
|
COMPILED_PASSES.clear();
|
|
467
534
|
COMPILED_PASSES.set(key, passes);
|
|
@@ -473,15 +540,16 @@ export function renderSecrets(text, policy, notation, options = {}) {
|
|
|
473
540
|
const marks = readNotation(notation);
|
|
474
541
|
const { passes: extra, report } = options;
|
|
475
542
|
const own = tablePasses(read, marks);
|
|
476
|
-
return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, report });
|
|
543
|
+
return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, onUnresolvedReading: read.onUnresolvedReading, report });
|
|
477
544
|
}
|
|
478
545
|
const FORMS = {
|
|
479
546
|
strict: ENGINE_VOCABULARY.values.flatMap((row) => (row.strict === undefined ? [] : [{ row, re: new RegExp(row.strict.source, `${row.strict.flags}g`), cut: "whole" }])),
|
|
480
547
|
wide: ENGINE_VOCABULARY.values.map((row) => ({ row, re: new RegExp(row.match.source, row.match.flags), cut: row.id === USERINFO_FAMILY ? "userinfo" : capturesLabel(row.match) ? "label" : "whole" })),
|
|
548
|
+
fragment: ENGINE_VOCABULARY.values.flatMap((row) => (row.fragment === undefined ? [] : [{ row, re: new RegExp(row.fragment.source, row.fragment.flags), cut: capturesLabel(row.fragment) ? "label" : "whole" }])),
|
|
481
549
|
};
|
|
482
550
|
function scannerFor(policy) {
|
|
483
551
|
const read = readPolicy(policy);
|
|
484
|
-
return { forms: FORMS[read.grade], readings: read.readings, urlUserinfo: read.urlUserinfo };
|
|
552
|
+
return { forms: FORMS[read.grade], readings: read.readings, urlUserinfo: read.urlUserinfo, onUnresolvedReading: read.onUnresolvedReading };
|
|
485
553
|
}
|
|
486
554
|
function formHits(input, scan) {
|
|
487
555
|
const hits = [];
|
|
@@ -515,21 +583,31 @@ function formHits(input, scan) {
|
|
|
515
583
|
}
|
|
516
584
|
return hits;
|
|
517
585
|
}
|
|
586
|
+
function characterHits(text, scan) {
|
|
587
|
+
if (!scan.readings.formatFree || !FORMAT_CHAR_RE.test(text))
|
|
588
|
+
return formHits(text, scan);
|
|
589
|
+
const reading = readFormatFree(text);
|
|
590
|
+
if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
|
|
591
|
+
return undefined;
|
|
592
|
+
const back = (h, shift) => ({ ...h, ...reading.toOriginalSpan(h.start + shift, h.end + shift) });
|
|
593
|
+
const spans = formHits(reading.view, scan).map((h) => back(h, 0));
|
|
594
|
+
if (scan.readings.raw)
|
|
595
|
+
appendAll(spans, formHits(text, scan));
|
|
596
|
+
for (const cut of reading.cuts)
|
|
597
|
+
appendAll(spans, formHits(reading.view.slice(cut), scan).map((h) => back(h, cut)));
|
|
598
|
+
return spans;
|
|
599
|
+
}
|
|
518
600
|
export function scanSecrets(text, policy) {
|
|
519
601
|
const scan = scannerFor(policy);
|
|
520
|
-
|
|
521
|
-
if (
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
602
|
+
const derived = readsTextReadings(scan.readings) ? readingViews(text, scan.readings) : undefined;
|
|
603
|
+
if (derived?.unresolved === true && scan.onUnresolvedReading === "secret")
|
|
604
|
+
return { outcome: "whole", reason: "reading-unresolved" };
|
|
605
|
+
const spans = [];
|
|
606
|
+
for (const view of derived?.views ?? [undefined]) {
|
|
607
|
+
const read = characterHits(view?.text ?? text, scan);
|
|
608
|
+
if (read === undefined)
|
|
526
609
|
return { outcome: "whole", reason: "scan-budget" };
|
|
527
|
-
|
|
528
|
-
spans = formHits(reading.view, scan).map((h) => back(h, 0));
|
|
529
|
-
if (scan.readings.raw)
|
|
530
|
-
spans.push(...formHits(text, scan));
|
|
531
|
-
for (const cut of reading.cuts)
|
|
532
|
-
spans.push(...formHits(reading.view.slice(cut), scan).map((h) => back(h, cut)));
|
|
610
|
+
appendAll(spans, view === undefined || view.text === text ? read : read.map((h) => ({ ...h, ...view.toOriginalSpan(h.start, h.end) })));
|
|
533
611
|
}
|
|
534
612
|
const seen = new Set();
|
|
535
613
|
const unique = spans.filter((s) => {
|
|
@@ -6,14 +6,20 @@
|
|
|
6
6
|
* - egress ({@link egressPassGroups}): every row's wide `match`, in table order, grouped into passes. A hit replaces
|
|
7
7
|
* the value in text that is persisted and forwarded, so a wide form is broad: a missed credential is that face's
|
|
8
8
|
* head risk and over-redaction is cheap.
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
9
|
+
* - the fragment grade (`scanSecrets` with `grade: "fragment"`): the rows that carry a `fragment` form — what the residue
|
|
10
|
+
* of a credential cut inside its body still reads as (a lower floor than the wide form, a word start, and two
|
|
11
|
+
* upper-case letters or digits in the body where the prefix is short).
|
|
12
|
+
* - env names ({@link envNameView}): the exact credential names, the credential words by position and the
|
|
13
|
+
* connection-string name shape a spawned child's environment is judged by.
|
|
14
|
+
* - env values ({@link envValueView}): every row's `envValue` form and the vendor token heads (`envValueHead`) a
|
|
15
|
+
* child's environment VALUE is judged by.
|
|
11
16
|
* - record keys ({@link recordKeyView}): the same credential words, read as the name of a member of a structured record
|
|
12
17
|
* (a header name, a config member) rather than of an environment variable.
|
|
13
18
|
*
|
|
14
|
-
* A family's strict and
|
|
15
|
-
* too (machine-judged over the fixture file by `scripts/verify-credential-vocabulary.mjs`): the two faces may
|
|
16
|
-
* floors, never in vocabulary. Detection is shared; each layer renders its own marker.
|
|
19
|
+
* A family's strict, wide, fragment and env forms live on the SAME row, and every text the strict form matches the wide
|
|
20
|
+
* form matches too (machine-judged over the fixture file by `scripts/verify-credential-vocabulary.mjs`): the two faces may
|
|
21
|
+
* differ in floors, never in vocabulary. Detection is shared; each layer renders its own marker. The credential NAMES a
|
|
22
|
+
* value stands beside (a keyword ahead of `=`, a URL parameter) are one list too ({@link CredentialVocabulary.names}).
|
|
17
23
|
*
|
|
18
24
|
* The table holds no layer's policy: which form a reader reads, how a text is read first, which marker replaces a hit
|
|
19
25
|
* and whether a URL's username is kept are the caller's, passed in whole (`SecretScanPolicy`, `SecretNotation` in
|
|
@@ -43,6 +49,26 @@ export interface ValueFamilyRow {
|
|
|
43
49
|
* alternation, so a token is taken at its leftmost start whichever of them it is (a nested shape is never cut by
|
|
44
50
|
* an earlier family's pass). Absent = a pass of its own. */
|
|
45
51
|
readonly pass?: string;
|
|
52
|
+
/** The fragment form, global: the residue of this family's credential when a text was cut inside its body. Present =
|
|
53
|
+
* the fragment grade reads this family; absent = it does not (a family with no distinctive head has no residue that
|
|
54
|
+
* still reads as one). */
|
|
55
|
+
readonly fragment?: RegExp;
|
|
56
|
+
/** The env value form, neither global nor sticky: a value it matches carries this family's credential, read by the env
|
|
57
|
+
* value view ({@link envValueView}) on a spawned child's environment. */
|
|
58
|
+
readonly envValue?: RegExp;
|
|
59
|
+
/** The head a token of this family opens with, unanchored, no flags: present = a value or a password-less URL user
|
|
60
|
+
* opening with it is token-shaped to the env value view ({@link EnvValueView.tokenHead}); absent = the family's tokens
|
|
61
|
+
* are not judged by their head there. */
|
|
62
|
+
readonly envValueHead?: RegExp;
|
|
63
|
+
}
|
|
64
|
+
/** Where a credential NAME is read beside its value: `assignment` — a keyword ahead of `=` / `:` in free text (the
|
|
65
|
+
* keyword-value row, camel humps included); `url-param` — the name of a URL query or fragment parameter, any case (the
|
|
66
|
+
* signed-url-param row). */
|
|
67
|
+
export type CredentialNamePlace = "assignment" | "url-param";
|
|
68
|
+
/** One credential name: "the value this name stands beside is a credential", in each place it lists. */
|
|
69
|
+
export interface CredentialNameRow {
|
|
70
|
+
readonly name: string;
|
|
71
|
+
readonly places: readonly CredentialNamePlace[];
|
|
46
72
|
}
|
|
47
73
|
/** Where a credential word counts in a name (env names are upper-cased words joined by `_`):
|
|
48
74
|
* - `leading` — at the start of the name or right after `_`;
|
|
@@ -86,8 +112,12 @@ export interface CredentialVocabulary {
|
|
|
86
112
|
/** A leading vendor prefix that spells a credential word without being one (`AUTH0_`): dropped from a name before the
|
|
87
113
|
* credential words are looked for, in every name view alike. */
|
|
88
114
|
readonly nonCredentialPrefix: RegExp;
|
|
89
|
-
/**
|
|
90
|
-
|
|
115
|
+
/** The credential names a value stands beside, one list for every place a name is read: the keyword-value row reads the
|
|
116
|
+
* `assignment` names (in this order), the signed-url-param row the `url-param` names. */
|
|
117
|
+
readonly names: readonly CredentialNameRow[];
|
|
118
|
+
/** The words a setting's value is when it is a switch, not a secret (lower case, read in any case): the env value view's
|
|
119
|
+
* assignment form leaves a value spelled as one of them. */
|
|
120
|
+
readonly flagWords: readonly string[];
|
|
91
121
|
}
|
|
92
122
|
/**
|
|
93
123
|
* A URL of any scheme ({@link URI_SCHEME_SOURCE}, then `://`), to the first white space, quote or backtick, any case. URL
|
|
@@ -110,7 +140,7 @@ export declare const CREDENTIAL_VOCABULARY: CredentialVocabulary;
|
|
|
110
140
|
export declare const ENGINE_VOCABULARY: CredentialVocabulary;
|
|
111
141
|
/** The egress view: the rows in table order, each run of adjacent rows naming the same `pass` as one group. */
|
|
112
142
|
export declare function egressPassGroups(vocabulary: CredentialVocabulary): ReadonlyArray<readonly ValueFamilyRow[]>;
|
|
113
|
-
/** The env-name view: what {@link CREDENTIAL_VOCABULARY} says about variable names
|
|
143
|
+
/** The env-name view: what {@link CREDENTIAL_VOCABULARY} says about variable names. */
|
|
114
144
|
export interface EnvNameView {
|
|
115
145
|
/** Upper-cased, `-` read as `_`; each name also under `INPUT_`. */
|
|
116
146
|
readonly exactNames: ReadonlySet<string>;
|
|
@@ -123,11 +153,27 @@ export interface EnvNameView {
|
|
|
123
153
|
/** A name opening with `namespace` (as written) that holds a credential word as ANY whole `_` segment after it —
|
|
124
154
|
* every position but `embedded`; a counted plural no count word touches — any case, `.` crossing line breaks. */
|
|
125
155
|
readonly anySegmentUnder: (namespace: string) => RegExp;
|
|
126
|
-
readonly tokenPrefix: RegExp;
|
|
127
156
|
}
|
|
128
157
|
/** The env-name view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
|
|
129
158
|
* Every regex of a view is the view's own object, so a reader that recompiles one changes only its own view. */
|
|
130
159
|
export declare function envNameView(vocabulary?: CredentialVocabulary): EnvNameView;
|
|
160
|
+
/** The env value view: what {@link CREDENTIAL_VOCABULARY} says about a spawned child's environment VALUES, every member
|
|
161
|
+
* derived from the rows. */
|
|
162
|
+
export interface EnvValueView {
|
|
163
|
+
/** Every row's env value form, in table order: a value one of them matches carries a credential. */
|
|
164
|
+
readonly forms: ReadonlyArray<{
|
|
165
|
+
readonly family: string;
|
|
166
|
+
readonly re: RegExp;
|
|
167
|
+
}>;
|
|
168
|
+
/** A value (or a password-less URL user) opening with a vendor token head: every row's `envValueHead`, in table order,
|
|
169
|
+
* as one anchored alternation. */
|
|
170
|
+
readonly tokenHead: RegExp;
|
|
171
|
+
/** The words a setting's value is when it is a switch, lower case (compare a value lower-cased). */
|
|
172
|
+
readonly flagWords: ReadonlySet<string>;
|
|
173
|
+
}
|
|
174
|
+
/** The env value view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
|
|
175
|
+
* Every regex of a view is the view's own object. */
|
|
176
|
+
export declare function envValueView(vocabulary?: CredentialVocabulary): EnvValueView;
|
|
131
177
|
/** `camelCase` / `PascalCase` read as `_`-separated words, `OAuth` and `NextAuth` kept whole. */
|
|
132
178
|
export declare function splitCamelCase(name: string): string;
|
|
133
179
|
/** The record-key view: what {@link CREDENTIAL_VOCABULARY} says about the names a structured record carries — the member
|
|
@@ -11,7 +11,8 @@ const PLAIN_QUOTES = [`"`, `'`, `"`, `'`, `"`, `'`, `%22`, `%2
|
|
|
11
11
|
const ESCAPED_QUOTES = [`"`, `'`, `x22`, `x27`, `u0022`, `u0027`, `042`, `047`];
|
|
12
12
|
const BACKSLASHES = String.raw `\\{1,${ESCAPE_DEPTH}}`;
|
|
13
13
|
const QUOTE = `(?:${PLAIN_QUOTES.join("|")}|${BACKSLASHES}(?:${ESCAPED_QUOTES.join("|")}))`;
|
|
14
|
-
const
|
|
14
|
+
const ESCAPE_TAIL = String.raw `(?:\\(?:[bfnrt]|u[0-9A-Fa-f]{4})|%[0-9A-Fa-f]{2})`;
|
|
15
|
+
const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE})|(?<=${ESCAPE_TAIL}))`;
|
|
15
16
|
const AFTER_CLOSING_QUOTE = String.raw `(?:[\s,}\]:]|["']|$)`;
|
|
16
17
|
const VALUE_RUN_END = `(?:(?<!\\\\)|(?!\\\\{0,${ESCAPE_DEPTH}}["']))`;
|
|
17
18
|
const GLUED_RUN = `${VALUE_CHAR}*${VALUE_RUN_END}`;
|
|
@@ -25,35 +26,105 @@ const BEARER_TAIL = `(?:(?!${QUOTE})\\S${GLUED_RUN}|(?<==)(?:${QUOTED_VALUE}))`;
|
|
|
25
26
|
const HEADER_PARAM = String.raw `=[ \t]*(\\{0,${ESCAPE_DEPTH}})(["'])(?:(?!(?<!\\)\2\3)[^\r\n])*(?:(?<!\\)\2\3(?=[\s,;]|\\{0,${ESCAPE_DEPTH}}["']|$)|(?=[\r\n]|$))`;
|
|
26
27
|
const HEADER_VALUE = String.raw `(?:[A-Za-z][A-Za-z0-9!#$%&*+.^_|~-]*[ \t]+)?(?:${HEADER_PARAM}|[^\s"',]|[ \t]*,[ \t]*(?=[^\s"',])|[ \t]+(?==)|(?<==)[ \t]+(?=[^\s"',]))+${VALUE_RUN_END}`;
|
|
27
28
|
const headed = (body, flags = "g") => new RegExp(`${WORD_START}${body}`, flags);
|
|
29
|
+
const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
28
30
|
const HUMP_BEFORE = "[a-z0-9]";
|
|
29
|
-
const
|
|
31
|
+
const CREDENTIAL_NAMES = [
|
|
32
|
+
...["token", "secret", "key", "password", "passwd"].map((name) => ({ name, places: ["assignment", "url-param"] })),
|
|
33
|
+
{ name: "credential", places: ["assignment"] },
|
|
34
|
+
...[
|
|
35
|
+
"api_key", "apikey", "api-key", "access_token", "refresh_token", "id_token", "auth", "authorization", "proxy-authorization",
|
|
36
|
+
"pwd", "client_secret", "private_key", "signature", "sig", "x-amz-signature", "x-amz-security-token", "x-amz-credential", "sas", "code",
|
|
37
|
+
].map((name) => ({ name, places: ["url-param"] })),
|
|
38
|
+
];
|
|
39
|
+
const namesAt = (place) => CREDENTIAL_NAMES.filter((n) => n.places.includes(place)).map((n) => n.name);
|
|
40
|
+
const KEYWORD_VALUE_WORDS = namesAt("assignment");
|
|
41
|
+
const FLAG_WORDS = ["true", "false", "none", "null", "yes", "no", "on", "off", "enabled", "disabled", "required", "optional"];
|
|
42
|
+
const residue = (head, cls, min) => String.raw `${head}(?=${cls}{0,96}[0-9A-Z]${cls}{0,96}[0-9A-Z])${cls}{${min},}`;
|
|
43
|
+
const GITLAB_HEAD = "gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-";
|
|
30
44
|
const anyCase = (word) => [...word].map((c) => `[${c.toUpperCase()}${c.toLowerCase()}]`).join("");
|
|
31
|
-
const
|
|
32
|
-
const
|
|
45
|
+
const AZURE_ACCOUNT_KEY = { front: "Account", keyword: "Key", separator: "=", body: "A-Za-z0-9+/" };
|
|
46
|
+
const AZURE_FIELD = `${AZURE_ACCOUNT_KEY.front}${AZURE_ACCOUNT_KEY.keyword}${AZURE_ACCOUNT_KEY.separator}`;
|
|
47
|
+
const AZURE_BODY = `[${AZURE_ACCOUNT_KEY.body}]{40,}={0,2}`;
|
|
48
|
+
const VALUE_ROW_FIELDS = [
|
|
49
|
+
[AZURE_ACCOUNT_KEY.front, AZURE_ACCOUNT_KEY.keyword, AZURE_ACCOUNT_KEY.separator, `[${AZURE_ACCOUNT_KEY.body}<]`],
|
|
50
|
+
];
|
|
51
|
+
const leftToItsRow = (keyword) => VALUE_ROW_FIELDS.filter(([, k]) => k.toLowerCase() === keyword)
|
|
52
|
+
.map(([front, k, separator, valueStart]) => `(?!(?<=${WORD_START}${front})${k}${separator}${valueStart})`)
|
|
53
|
+
.join("");
|
|
33
54
|
const KEYWORD_HEAD = `(?:${WORD_START}(?:${KEYWORD_VALUE_WORDS.map(anyCase).join("|")})|(?<=${HUMP_BEFORE})(?:${KEYWORD_VALUE_WORDS.map((w) => leftToItsRow(w) + w[0].toUpperCase() + anyCase(w.slice(1))).join("|")})|(?<=[A-Z])(?:${KEYWORD_VALUE_WORDS.map((w) => w[0].toUpperCase() + w[1] + anyCase(w.slice(2))).join("|")}))`;
|
|
34
55
|
const KEY_QUOTE = `(?:\\\\{0,${ESCAPE_DEPTH}}["'])?`;
|
|
35
56
|
export const CREDENTIAL_VOCABULARY = {
|
|
36
57
|
values: [
|
|
37
|
-
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/ },
|
|
38
|
-
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/ },
|
|
39
|
-
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/ },
|
|
40
|
-
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test)_[A-Za-z0-9]{24,}\b
|
|
41
|
-
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`) },
|
|
42
|
-
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(
|
|
43
|
-
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}") },
|
|
44
|
-
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`) },
|
|
58
|
+
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/, fragment: headed(residue("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8)), envValueHead: /gh[opusr]_/ },
|
|
59
|
+
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/, fragment: /github_pat_[A-Za-z0-9_]{8,}/g, envValueHead: /github_pat_/ },
|
|
60
|
+
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/, fragment: headed(residue("sk-", "[A-Za-z0-9_-]", 6)), envValueHead: /sk-/ },
|
|
61
|
+
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test|org)_[A-Za-z0-9]{24,}\b/, fragment: headed(residue("(?:sk|rk)_(?:live|test|org)_", "[A-Za-z0-9_-]", 8)) },
|
|
62
|
+
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`), envValueHead: /pk-/ },
|
|
63
|
+
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`${GITLAB_HEAD}[A-Za-z0-9_-]{20,}`), strict: new RegExp(String.raw `\b${GITLAB_HEAD}(?=[A-Za-z0-9_]*[0-9])[A-Za-z0-9_]{20,}\b`), fragment: headed(residue(GITLAB_HEAD, "[A-Za-z0-9_-]", 8)), envValueHead: /glpat-/ },
|
|
64
|
+
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}"), envValueHead: /npm_/ },
|
|
65
|
+
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`), envValueHead: /ya29\./ },
|
|
66
|
+
{ id: "pypi-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("pypi-[A-Za-z0-9_-]{85,}"), strict: /\bpypi-(?=[A-Za-z0-9_-]*[0-9])[A-Za-z0-9_-]{85,}/, fragment: headed(residue("pypi-", "[A-Za-z0-9_-]", 8)) },
|
|
45
67
|
{ id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("AIza[A-Za-z0-9_-]{20,}"), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
|
46
|
-
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[
|
|
47
|
-
{ id: "
|
|
68
|
+
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprse]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprse]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/, fragment: headed(residue("xox[baprse]-", "[A-Za-z0-9-]", 4)), envValueHead: /xox[abpr]-/ },
|
|
69
|
+
{ id: "slack-app-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(String.raw `xapp-\d-[A-Za-z0-9-]{20,}`), strict: /\bxapp-\d-(?=[A-Za-z0-9-]{20})(?:[A-Za-z0-9]+-){1,3}(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{16,}\b/, fragment: headed(residue(String.raw `xapp-\d-`, "[A-Za-z0-9-]", 4)) },
|
|
70
|
+
{ id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/, fragment: headed(`(?:${residue("AKIA", "[0-9A-Z]", 6)}|${residue("ASIA", "[0-9A-Z]", 8)})`), envValueHead: /AKIA/ },
|
|
48
71
|
{ id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("hf_[A-Za-z0-9]{20,}"), strict: /\bhf_(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{20,}\b/ },
|
|
49
|
-
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/ },
|
|
50
|
-
{
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
72
|
+
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/, envValueHead: /eyJ/ },
|
|
73
|
+
{
|
|
74
|
+
id: "private-key",
|
|
75
|
+
kind: "private-key-block",
|
|
76
|
+
source: "arg-summary",
|
|
77
|
+
confidence: "high",
|
|
78
|
+
match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)|---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]*?(?:---- END SSH2 ENCRYPTED PRIVATE KEY ----|$)|PuTTY-User-Key-File-\d+:[\s\S]*?(?:Private-MAC:[ \t]*[0-9A-Fa-f]+|$)|Private-Lines:[ \t]*\d+(?:\r?\n[A-Za-z0-9+/=]+(?=\r?\n|$))+/g,
|
|
79
|
+
strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/,
|
|
80
|
+
envValue: /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/,
|
|
81
|
+
},
|
|
82
|
+
{
|
|
83
|
+
id: "bearer-token",
|
|
84
|
+
kind: "bearer-token",
|
|
85
|
+
source: "arg-summary",
|
|
86
|
+
confidence: "high",
|
|
87
|
+
match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`),
|
|
88
|
+
strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/,
|
|
89
|
+
envValue: new RegExp(String.raw `\b(?:Bearer|Basic)\s+(?=[A-Za-z._~+\/=-]*[0-9]|(?:[A-Za-z0-9._~+\/=-]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9.-]*[_~+\/=])[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`),
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
id: "authorization-header",
|
|
93
|
+
kind: "authorization-header",
|
|
94
|
+
source: "arg-summary",
|
|
95
|
+
confidence: "high",
|
|
96
|
+
match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi"),
|
|
97
|
+
envValue: new RegExp(String.raw `\bauthorization\s*[:=]\s*["']?[a-z][a-z0-9_-]*\s+[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`, "i"),
|
|
98
|
+
},
|
|
99
|
+
{ id: "basic-credential", kind: "scheme-credential", source: "arg-summary", confidence: "high", match: headed(String.raw `[Bb][Aa][Ss][Ii][Cc]\s+(?=[A-Za-z+/]*[0-9]|(?:[A-Za-z0-9+/]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9]*[+/=])[A-Za-z0-9+/]{${CREDENTIAL_MIN_CHARS},}={0,2}`) },
|
|
100
|
+
{
|
|
101
|
+
id: "token-scheme",
|
|
102
|
+
kind: "scheme-credential",
|
|
103
|
+
source: "arg-summary",
|
|
104
|
+
confidence: "high",
|
|
105
|
+
match: headed(String.raw `()(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=([A-Za-z0-9_~+=-]{20,}))\2(?![\/.])`),
|
|
106
|
+
envValue: /\b(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=[A-Za-z0-9_~+=-]*[A-Z])(?=([A-Za-z0-9_~+=-]{20,}))\1(?![\/.])/,
|
|
107
|
+
},
|
|
108
|
+
{ id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: new RegExp(`(${AZURE_FIELD})${AZURE_BODY}`, "g"), strict: new RegExp(`${AZURE_FIELD}${AZURE_BODY}`) },
|
|
109
|
+
{
|
|
110
|
+
id: "keyword-value",
|
|
111
|
+
kind: "keyword-value",
|
|
112
|
+
source: "arg-summary",
|
|
113
|
+
confidence: "medium",
|
|
114
|
+
match: new RegExp(String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "g"),
|
|
115
|
+
envValue: new RegExp(String.raw `(?:(?:^|[;&?#,{]|\/:)\s*|\s)["']?(?!-*jobserver-auth\s*[=:])(?:[a-z0-9_.-]{0,64}(?:password|passwd|pwd|secret|token|(?:account|access|api|private|subscription)[-_]?key|signature|sig|credential)|[a-z0-9_.-]{0,63}[_.-]auth)["']?\s*[=:]\s*["']?(?!(?:${FLAG_WORDS.join("|")})(?:$|[;&,\s"']))[^;&,\s"']+`, "i"),
|
|
116
|
+
},
|
|
117
|
+
{ id: "sonar-login", kind: "keyword-value", source: "arg-summary", confidence: "high", match: new RegExp(String.raw `(${KEY_QUOTE}sonar\.login${KEY_QUOTE}\s*[:=]\s*${KEY_QUOTE})[^\s"',}]{${CREDENTIAL_MIN_CHARS},}${VALUE_RUN_END}`, "g"), envValue: new RegExp(String.raw `["']?sonar\.login["']?\s*[:=]\s*["']?[^\s"',}]{${CREDENTIAL_MIN_CHARS},}`) },
|
|
55
118
|
{ id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${URI_SCHEME_SOURCE}:\/\/)(${URL_USERINFO_SOURCE})`, "gi") },
|
|
56
|
-
{
|
|
119
|
+
{
|
|
120
|
+
id: "webhook-url",
|
|
121
|
+
kind: "webhook-url",
|
|
122
|
+
source: "untrusted-egress",
|
|
123
|
+
confidence: "high",
|
|
124
|
+
match: /(https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/))[\w\/@.~-]{16,}/gi,
|
|
125
|
+
envValue: /https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/)[\w\/@.~-]{16,}/i,
|
|
126
|
+
},
|
|
127
|
+
{ id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: new RegExp(String.raw `([?&#](?:${namesAt("url-param").map(literally).join("|")})=)[^&\s"'#]+${VALUE_RUN_END}`, "gi") },
|
|
57
128
|
],
|
|
58
129
|
words: [
|
|
59
130
|
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE", "KEY"].map((word) => ({ word, position: "leading" })),
|
|
@@ -100,7 +171,8 @@ export const CREDENTIAL_VOCABULARY = {
|
|
|
100
171
|
],
|
|
101
172
|
nonSecretNameShape: /^GIT_CONFIG_KEY_[0-9][A-Za-z0-9_]*$/,
|
|
102
173
|
nonCredentialPrefix: /^AUTH0_/i,
|
|
103
|
-
|
|
174
|
+
names: CREDENTIAL_NAMES,
|
|
175
|
+
flagWords: FLAG_WORDS,
|
|
104
176
|
};
|
|
105
177
|
function freezeTable(node) {
|
|
106
178
|
if (typeof node !== "object" || node === null || node instanceof RegExp || Object.isFrozen(node))
|
|
@@ -131,7 +203,6 @@ export function egressPassGroups(vocabulary) {
|
|
|
131
203
|
}
|
|
132
204
|
return groups;
|
|
133
205
|
}
|
|
134
|
-
const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
135
206
|
function credentialWordPattern(countWords, rows) {
|
|
136
207
|
const at = (position) => rows.filter((r) => r.position === position).map((r) => r.word);
|
|
137
208
|
const countedPlural = `(?<!(?:^|_)(?:${countWords.join("|")})_)(?:${at("counted-plural").join("|")})S`;
|
|
@@ -154,7 +225,14 @@ export function envNameView(vocabulary = ENGINE_VOCABULARY) {
|
|
|
154
225
|
credentialWord,
|
|
155
226
|
connectionStringName: withOwnRegexes(vocabulary.connectionStringName),
|
|
156
227
|
anySegmentUnder: (namespace) => new RegExp(`^${literally(namespace)}(?:.*_)?${segmentWord}(?:_|$)`, "is"),
|
|
157
|
-
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
export function envValueView(vocabulary = ENGINE_VOCABULARY) {
|
|
231
|
+
const heads = vocabulary.values.flatMap((row) => (row.envValueHead === undefined ? [] : [row.envValueHead.source]));
|
|
232
|
+
return {
|
|
233
|
+
forms: Object.freeze(vocabulary.values.flatMap((row) => (row.envValue === undefined ? [] : [Object.freeze({ family: row.id, re: withOwnRegexes(row.envValue) })]))),
|
|
234
|
+
tokenHead: new RegExp(heads.length === 0 ? "(?!)" : `^(?:${heads.join("|")})`),
|
|
235
|
+
flagWords: new Set(vocabulary.flagWords),
|
|
158
236
|
};
|
|
159
237
|
}
|
|
160
238
|
export function splitCamelCase(name) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/primitives",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.65.0",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Shared low-level definitions of the sema engine, importable by any layer: the credential reference (closed set of kinds, one reader) and the secret-shape detection definition (one table, its matcher and its renderer, no layer's policy). Zero dependencies.",
|
|
6
6
|
"type": "module",
|
|
@@ -31,5 +31,5 @@
|
|
|
31
31
|
"scripts": {
|
|
32
32
|
"build": "tsc -p tsconfig.json && tsc -p tsconfig.json --emitDeclarationOnly --removeComments false"
|
|
33
33
|
},
|
|
34
|
-
"gitHead": "
|
|
34
|
+
"gitHead": "5ab3f8045ee0d9965d58344531544df295422b69"
|
|
35
35
|
}
|