@sema-agent/primitives 7.62.0 → 7.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +60 -4
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/secret-scan.d.ts +176 -0
- package/dist/secret-scan.js +545 -0
- package/dist/secret-vocabulary.d.ts +163 -0
- package/dist/secret-vocabulary.js +205 -0
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# @sema-agent/primitives
|
|
2
2
|
|
|
3
3
|
Low-level definitions of the sema engine that every layer may import: no policy of any layer, no state, no
|
|
4
|
-
dependencies.
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
own types.
|
|
4
|
+
dependencies. It holds two members: the credential reference, and the secret-shape detection definition (the one table
|
|
5
|
+
of credential shapes, its matcher and its renderer). The engine depends on this package and reads it by name, so in a
|
|
6
|
+
project that installs both there is one copy: an acceptance gate, a settings validator or an admin form reads the same
|
|
7
|
+
closed set through the same function object as the engine does. Runs in Node and in browsers; ships its own types.
|
|
8
8
|
|
|
9
9
|
## Install
|
|
10
10
|
|
|
@@ -88,6 +88,60 @@ whether a reference may reach a destination. Those are the policy of the layer t
|
|
|
88
88
|
`CREDENTIAL_REF_TRUST` has a row for each kind and for nothing else (it has no prototype, so no inherited member answers);
|
|
89
89
|
look a kind up in it only after `readCredentialRef` has accepted the value.
|
|
90
90
|
|
|
91
|
+
## Secret-shape detection
|
|
92
|
+
|
|
93
|
+
`CREDENTIAL_VOCABULARY` is the one table of what a credential looks like: value families (each row with a wide form, and
|
|
94
|
+
for some a strict form), the credential words a name is read by, and the names that look like credentials but are none.
|
|
95
|
+
Every layer that looks for a secret reads this table, so a shape added to it is found everywhere at once. The table holds
|
|
96
|
+
no layer's policy: which form is read, how a text is read first, how a URL's username is treated and which mark replaces
|
|
97
|
+
a hit are the caller's, passed in whole on every call. Nothing here has a default.
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
interface SecretScanPolicy {
|
|
101
|
+
grade: "strict" | "wide"; // the strict forms (narrow: a hit may refuse a write) or the wide ones (broad: a hit is withheld)
|
|
102
|
+
readings: SecretReading[]; // at least one of { kind: "raw" } (as written) / { kind: "format-free" } (format characters removed)
|
|
103
|
+
urlUserinfo: "keep-username" | "whole"; // a URL userinfo: keep the user and withhold the password, or withhold all of it
|
|
104
|
+
onUnresolvedReading: "secret" | "ignore"; // what a reading that cannot read a text to its end answers
|
|
105
|
+
}
|
|
106
|
+
interface SecretNotation {
|
|
107
|
+
value: string; // the mark a hit is written as
|
|
108
|
+
byKind?: Record<string, string>; // ...or, for a row of this kind, this mark
|
|
109
|
+
byFamily?: Record<string, string>; // ...or, for this row, this mark
|
|
110
|
+
overStrip: string; // the whole text, when no reading can afford it
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
| Function | Answers |
|
|
115
|
+
| --- | --- |
|
|
116
|
+
| `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved`, `mapping-failed` |
|
|
117
|
+
| `isSecretShaped(text, policy)` | `scanSecrets` found a span or answered the whole text |
|
|
118
|
+
| `renderSecrets(text, policy, notation, { passes?, report? })` | `text` with every hit replaced by its mark; `passes` are the caller's own rules, run after the table's in the same pipeline; `report.findings` receives one finding per hit (`report.preexistingMarkers` is the caller's to set) |
|
|
119
|
+
| `isSensitiveRecordKey(key)` / `isSensitiveHeaderName(name)` | a record member / an HTTP header name names a credential |
|
|
120
|
+
| `envNameView()` | the readings of environment variable names (exact names, credential words, the token-prefix value predicate, a namespace's names `anySegmentUnder(namespace)`) |
|
|
121
|
+
|
|
122
|
+
A policy or a notation that is not whole (a missing or unknown member, an empty mark, a mark holding `$` before a digit,
|
|
123
|
+
a `byKind` / `byFamily` key no row has, two rows of one pass spelled differently) throws a `TypeError`. Both are read on
|
|
124
|
+
every call; what they read as picks a compilation kept across calls. A mark is written as spelled. `scanSecrets`
|
|
125
|
+
answers where a credential is; `renderSecrets` also leaves a hit that already reads as its own mark, which only a
|
|
126
|
+
notation can tell. `URL_OF_ANY_SCHEME` is the URL grammar (a global regex: read it with `replace` / `matchAll`), for a
|
|
127
|
+
layer that withholds whole URLs. `CREDENTIAL_MIN_CHARS` is the fewest characters a string can have and still be read as
|
|
128
|
+
a credential.
|
|
129
|
+
|
|
130
|
+
The table and its rows are frozen; its regexes are not (`replace` writes a global regex's `lastIndex`), and the functions
|
|
131
|
+
here never run them: the package runs its own copies, made as it loads, so a regex a caller recompiles or moves on the
|
|
132
|
+
exported table changes only that caller's reading.
|
|
133
|
+
|
|
134
|
+
```js
|
|
135
|
+
import { renderSecrets, scanSecrets } from "@sema-agent/primitives";
|
|
136
|
+
|
|
137
|
+
const policy = Object.freeze({ grade: "wide", readings: Object.freeze([Object.freeze({ kind: "raw" })]), urlUserinfo: "keep-username", onUnresolvedReading: "secret" });
|
|
138
|
+
const notation = Object.freeze({ value: "[hidden]", overStrip: "[hidden]" });
|
|
139
|
+
renderSecrets("clone https://me:pa55word@git.example/r", policy, notation); // "clone https://me:[hidden]@git.example/r"
|
|
140
|
+
scanSecrets("clone https://me:pa55word@git.example/r", policy); // { outcome: "spans", spans: [{ start: 17, end: 25, family: "url-userinfo", kind: "url-credentials" }] }
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
Detection is best effort over known shapes: it never proves a text holds no credential.
|
|
144
|
+
|
|
91
145
|
## Exports
|
|
92
146
|
|
|
93
147
|
Each name below is a runtime value or a type exported from the package root. The root's declarations are the contract; a
|
|
@@ -97,3 +151,5 @@ removed or re-kinded export is named as breaking in the `@sema-agent/core` relea
|
|
|
97
151
|
| --- | --- |
|
|
98
152
|
| Credential reference | `readCredentialRef` `CREDENTIAL_REF_KINDS` `CREDENTIAL_REF_TRUST` `CREDENTIAL_REF_TEXT_MAX` `CREDENTIAL_REF_MAP_MAX` `CREDENTIAL_REF_PROBLEM_REASONS` |
|
|
99
153
|
| Types | `CredentialRef` `CredentialRefKind` `CredentialRefRequestAdmission` `CredentialRefProblem` `CredentialRefProblemReason` `CredentialRefReading` |
|
|
154
|
+
| Secret-shape detection | `CREDENTIAL_VOCABULARY` `CREDENTIAL_MIN_CHARS` `URL_OF_ANY_SCHEME` `SECRET_READING_KINDS` `scanSecrets` `isSecretShaped` `renderSecrets` `isSensitiveRecordKey` `isSensitiveHeaderName` `envNameView` `splitCamelCase` |
|
|
155
|
+
| Types | `CredentialVocabulary` `ValueFamilyRow` `CredentialWordRow` `CredentialWordPosition` `EnvNameView` `RecordKeyView` `RedactionConfidence` `RedactionSource` `SecretGrade` `SecretReading` `SecretScanPolicy` `SecretNotation` `SecretScan` `SecretSpan` `RenderOptions` `RedactionPass` `RedactionFinding` `RedactionReport` `SummarizableFinding` |
|
package/dist/index.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
1
|
/** The package entry of `@sema-agent/primitives`; core imports the package by name, never these modules. */
|
|
2
2
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, type CredentialRef, type CredentialRefKind, type CredentialRefRequestAdmission, type CredentialRefProblem, type CredentialRefProblemReason, type CredentialRefReading, } from "./credential-ref.js";
|
|
3
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type EnvNameView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
|
|
4
|
+
export { SECRET_READING_KINDS, scanSecrets, isSecretShaped, renderSecrets, type SecretGrade, type SecretReading, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
|
package/dist/index.js
CHANGED
|
@@ -1 +1,3 @@
|
|
|
1
1
|
export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, } from "./credential-ref.js";
|
|
2
|
+
export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
|
|
3
|
+
export { SECRET_READING_KINDS, scanSecrets, isSecretShaped, renderSecrets, } from "./secret-scan.js";
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The matcher and the renderer of the credential vocabulary (secret-vocabulary.ts), each reading the caller's policy
|
|
3
|
+
* whole — the grade of form, the readings a text is read through, the URL userinfo rendering — and the renderer the
|
|
4
|
+
* caller's notation (which marker a hit is written as):
|
|
5
|
+
* - {@link scanSecrets} answers WHERE the vocabulary sees a credential (original-text spans, or "the whole text" when a
|
|
6
|
+
* reading cannot be afforded), and {@link isSecretShaped} whether it sees one at all;
|
|
7
|
+
* - {@link renderSecrets} answers what the text reads as with every credential withheld, in the caller's notation. A caller
|
|
8
|
+
* that also withholds what is not a credential (a layer's own URL or path rule) appends its passes, which run in the
|
|
9
|
+
* same pipeline, through the same readings.
|
|
10
|
+
* No layer's policy or marker lives here: every one is an argument, every argument is required.
|
|
11
|
+
*
|
|
12
|
+
* The renderer is one SEQUENTIAL multi-pass engine: each pass scans the text the earlier passes left, so a hit that runs
|
|
13
|
+
* into an earlier marker reads the marker the caller chose. A row whose wide form captures a label keeps it and replaces
|
|
14
|
+
* only the value (`GITHUB_TOKEN=ghp_xxx` keeps `GITHUB_TOKEN=`), and a second run over already-rendered text is a no-op.
|
|
15
|
+
* Every pass declares what it detects (kind), how sure it is (confidence) and the marker it writes; a report collector,
|
|
16
|
+
* when given, receives one finding per hit in the ORIGINAL input's coordinates, mapped back through the earlier passes'
|
|
17
|
+
* edits (best-effort where redactions overlap — a span landing inside an earlier replacement anchors to that edit's start).
|
|
18
|
+
*/
|
|
19
|
+
import { type RedactionConfidence, type RedactionSource, type ValueFamilyRow } from "./secret-vocabulary.js";
|
|
20
|
+
/** One redaction hit: what was detected, where (ORIGINAL-input coordinates), how confidently, the marker
|
|
21
|
+
* literal written in its place, which rule table minted it, and — present when a vocabulary row minted it — the
|
|
22
|
+
* value family (the row's stable id; a host-leak hit such as a URL or a path names none). */
|
|
23
|
+
export interface RedactionFinding {
|
|
24
|
+
kind: string;
|
|
25
|
+
span: readonly [number, number];
|
|
26
|
+
confidence: RedactionConfidence;
|
|
27
|
+
marker: string;
|
|
28
|
+
source: RedactionSource;
|
|
29
|
+
family?: string;
|
|
30
|
+
}
|
|
31
|
+
/** The minimum a finding must declare to be summarizable: a text finding ({@link RedactionFinding}) and an entry finding
|
|
32
|
+
* of a layer that deletes whole entries both satisfy it, so a layer's one summary line reads either. */
|
|
33
|
+
export interface SummarizableFinding {
|
|
34
|
+
kind: string;
|
|
35
|
+
confidence: RedactionConfidence;
|
|
36
|
+
}
|
|
37
|
+
/** The optional collector the renderer accepts. `findings` receives one entry per hit. `preexistingMarkers` counts the
|
|
38
|
+
* marks ALREADY present in the input before any pass ran — a nonzero value means the text arrived pre-redacted or carries
|
|
39
|
+
* caller-declared marks, either way "a mark is present" is not proof the pipeline ran. Which strings are marks is the
|
|
40
|
+
* caller's notation, so the caller sets it; the renderer never writes it. Caller-owned: pass `{ findings: [] }` and read
|
|
41
|
+
* it back after the call. */
|
|
42
|
+
export interface RedactionReport {
|
|
43
|
+
findings: RedactionFinding[];
|
|
44
|
+
preexistingMarkers?: number;
|
|
45
|
+
}
|
|
46
|
+
/** A declared redaction pass — the unit the renderer executes and reports on: one per vocabulary row group, and each pass
|
|
47
|
+
* a caller appends ({@link RenderOptions.passes}). */
|
|
48
|
+
export interface RedactionPass {
|
|
49
|
+
kind: string;
|
|
50
|
+
confidence: RedactionConfidence;
|
|
51
|
+
marker: string;
|
|
52
|
+
/** The rule table that owns this pass's rule — copied verbatim onto every finding it mints (ANG-11), whoever
|
|
53
|
+
* runs the pass. */
|
|
54
|
+
source: RedactionSource;
|
|
55
|
+
re: RegExp;
|
|
56
|
+
/** A literal replacement (only `$1` substitution is used by these passes) or a replacer fn. */
|
|
57
|
+
replace: string | ((match: string, ...groups: string[]) => string);
|
|
58
|
+
/** The value family a hit belongs to: one id, or — for a pass joining several rows — the id of the row that
|
|
59
|
+
* matches at the hit's offset in the pass input. Absent = the pass is no vocabulary row. */
|
|
60
|
+
family?: string | ((input: string, at: number) => string);
|
|
61
|
+
}
|
|
62
|
+
/** Which readings a run reads its text through (the policy's `readings`, as a set). */
|
|
63
|
+
interface ReadingSet {
|
|
64
|
+
readonly raw: boolean;
|
|
65
|
+
readonly formatFree: boolean;
|
|
66
|
+
}
|
|
67
|
+
/** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings,
|
|
68
|
+
* and the optional report. */
|
|
69
|
+
export interface EngineRun {
|
|
70
|
+
readonly overStrip: string;
|
|
71
|
+
readonly readings: ReadingSet;
|
|
72
|
+
readonly report?: RedactionReport | undefined;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Run the declared passes in order, collecting findings (original-input spans) when a report is given.
|
|
76
|
+
*
|
|
77
|
+
* FORMAT CHARACTERS (the `format-free` reading): the passes scan a VIEW of the input with every `\p{Cf}` removed, so a
|
|
78
|
+
* credential split by a zero-width character is one token to the scanner, not two fragments of which only the first (or
|
|
79
|
+
* neither) matches. The OUTPUT keeps every format character that sits OUTSIDE a replaced span — a joiner in an emoji
|
|
80
|
+
* sequence, a non-joiner in Persian text, a soft hyphen in a word — and drops only those INSIDE the span the marker
|
|
81
|
+
* replaces (they were part of the secret). A format-free input, or a run without the `format-free` reading, takes the
|
|
82
|
+
* plain sequential path. The view's replacements and every boundary scan's are taken as disjoint spans of their input (a
|
|
83
|
+
* later pass that matched across an earlier pass's inserted text is rendered from the removals instead), so the spans
|
|
84
|
+
* always map back exactly and no scan is skipped.
|
|
85
|
+
*/
|
|
86
|
+
export declare function runRedactionPasses(input: string, passes: readonly RedactionPass[], run: EngineRun): string;
|
|
87
|
+
/** Which form of a row a reader reads: `strict` (refuse-write — narrow, a hit fail-closed refuses) or `wide` (egress —
|
|
88
|
+
* broad, a hit is withheld). */
|
|
89
|
+
export type SecretGrade = "strict" | "wide";
|
|
90
|
+
/** The readings a text can be read through, a closed set: `raw` — the text as written; `format-free` — the text with every
|
|
91
|
+
* format character (`\p{Cf}`) removed, its hits mapped back, plus a scan from every place a format character stood (so a
|
|
92
|
+
* format character neither splits a credential nor hides its start). */
|
|
93
|
+
export declare const SECRET_READING_KINDS: readonly ["raw", "format-free"];
|
|
94
|
+
/** One reading, by its kind. A reading that takes a parameter carries it as a member beside `kind`. */
|
|
95
|
+
export type SecretReading = {
|
|
96
|
+
readonly kind: "raw";
|
|
97
|
+
} | {
|
|
98
|
+
readonly kind: "format-free";
|
|
99
|
+
};
|
|
100
|
+
/** What a caller reads the vocabulary for — every member required, none has a default here:
|
|
101
|
+
* - `grade` — which form of each row is read ({@link SecretGrade});
|
|
102
|
+
* - `readings` — the readings a text is read through, at least one ({@link SECRET_READING_KINDS}); the hits of every
|
|
103
|
+
* reading are joined;
|
|
104
|
+
* - `urlUserinfo` — a URL userinfo's rendering: `keep-username` keeps the user and withholds the password (a bare user is
|
|
105
|
+
* kept below 20 characters and withheld at or past it; a userinfo holding a second `@` is withheld whole), `whole`
|
|
106
|
+
* withholds the whole userinfo;
|
|
107
|
+
* - `onUnresolvedReading` — what a reading that cannot read a text to its end answers: `secret` (the whole text) or
|
|
108
|
+
* `ignore` (what it did read). `raw` and `format-free` always read to the end. */
|
|
109
|
+
export interface SecretScanPolicy {
|
|
110
|
+
readonly grade: SecretGrade;
|
|
111
|
+
readonly readings: readonly SecretReading[];
|
|
112
|
+
readonly urlUserinfo: "keep-username" | "whole";
|
|
113
|
+
readonly onUnresolvedReading: "secret" | "ignore";
|
|
114
|
+
}
|
|
115
|
+
/** The marks a hit is written as — the caller's own spellings, none of them a default here. A row's hit is written as
|
|
116
|
+
* `byFamily[row id]`, else `byKind[row kind]`, else `value`; a text no reading can afford is replaced whole by
|
|
117
|
+
* `overStrip`. Rows that run as one pass (they share a `pass`) are one alternation and write one mark: a `byFamily` that
|
|
118
|
+
* spells two of them differently is refused. A mark is a non-empty string. */
|
|
119
|
+
export interface SecretNotation {
|
|
120
|
+
readonly value: string;
|
|
121
|
+
readonly byKind?: Readonly<Record<string, string>>;
|
|
122
|
+
readonly byFamily?: Readonly<Record<string, string>>;
|
|
123
|
+
readonly overStrip: string;
|
|
124
|
+
}
|
|
125
|
+
/** Where the vocabulary sees a credential: the withheld part of one hit (a kept label or username outside it), in the
|
|
126
|
+
* text's own coordinates, with the row that found it. */
|
|
127
|
+
export interface SecretSpan {
|
|
128
|
+
readonly start: number;
|
|
129
|
+
readonly end: number;
|
|
130
|
+
/** The row's id. */
|
|
131
|
+
readonly family: string;
|
|
132
|
+
/** The row's kind. */
|
|
133
|
+
readonly kind: string;
|
|
134
|
+
}
|
|
135
|
+
/** What a scan found: spans (sorted by start; empty = nothing found), or the whole text — no reading could afford it
|
|
136
|
+
* (`scan-budget`), a reading could not read it to its end under `onUnresolvedReading: "secret"` (`reading-unresolved`),
|
|
137
|
+
* or a reading's hits could not be mapped back to the text (`mapping-failed`). Any reading that answers the whole text
|
|
138
|
+
* makes the whole answer the whole text. */
|
|
139
|
+
export type SecretScan = {
|
|
140
|
+
readonly outcome: "spans";
|
|
141
|
+
readonly spans: readonly SecretSpan[];
|
|
142
|
+
} | {
|
|
143
|
+
readonly outcome: "whole";
|
|
144
|
+
readonly reason: "reading-unresolved" | "scan-budget" | "mapping-failed";
|
|
145
|
+
};
|
|
146
|
+
/** What {@link renderSecrets} takes beside the policy and the notation: the caller's own passes, run after the vocabulary's
|
|
147
|
+
* in the same pipeline and through the same readings (each writes its own `marker`), and a report collector. */
|
|
148
|
+
export interface RenderOptions {
|
|
149
|
+
readonly passes?: readonly RedactionPass[] | undefined;
|
|
150
|
+
readonly report?: RedactionReport | undefined;
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* The wide forms rendered as passes, in the vocabulary's order: one pass per group (a group of several rows is ONE
|
|
154
|
+
* leftmost alternation, its hits attributed to the row that matches at the hit's offset). A row whose wide form
|
|
155
|
+
* captures a label keeps it ahead of the marker; the URL userinfo is split per `urlUserinfo`.
|
|
156
|
+
*/
|
|
157
|
+
export declare function egressPasses(groups: ReadonlyArray<readonly ValueFamilyRow[]>, markOf: (row: ValueFamilyRow) => string, urlUserinfo: SecretScanPolicy["urlUserinfo"]): RedactionPass[];
|
|
158
|
+
/**
|
|
159
|
+
* `text` with every credential the vocabulary finds under `policy` withheld, written in `notation`, and the caller's own
|
|
160
|
+
* passes ({@link RenderOptions.passes}) run after the vocabulary's in the same pipeline. One finding per hit goes to
|
|
161
|
+
* `options.report` when given (`report.preexistingMarkers` is the caller's to set: which strings are marks is its
|
|
162
|
+
* notation). The policy and the notation are read and checked on every call; what they read as picks a compilation kept
|
|
163
|
+
* across calls. A mark is written as spelled. Throws a `TypeError` on a policy or a notation that is not whole.
|
|
164
|
+
*/
|
|
165
|
+
export declare function renderSecrets(text: string, policy: SecretScanPolicy, notation: SecretNotation, options?: RenderOptions): string;
|
|
166
|
+
/**
|
|
167
|
+
* Where the vocabulary sees a credential in `text` under `policy`: every hit of the grade's form of every row, on every
|
|
168
|
+
* reading of the policy, as the hit's withheld part in `text`'s own coordinates (deduplicated, sorted by start) — or the
|
|
169
|
+
* whole text, when a reading cannot be afforded (more than 64 places where format characters stood: `scan-budget`). The
|
|
170
|
+
* matcher answers where a credential is; the renderer additionally leaves a hit that already reads as its own rendering
|
|
171
|
+
* (a value that is already a mark), which only a notation can tell. Throws a `TypeError` on a policy that is not whole.
|
|
172
|
+
*/
|
|
173
|
+
export declare function scanSecrets(text: string, policy: SecretScanPolicy): SecretScan;
|
|
174
|
+
/** Does the vocabulary see a credential in `text` under `policy`? {@link scanSecrets} answers the whole text or a span. */
|
|
175
|
+
export declare function isSecretShaped(text: string, policy: SecretScanPolicy): boolean;
|
|
176
|
+
export {};
|
|
@@ -0,0 +1,545 @@
|
|
|
1
|
+
import { ENGINE_VOCABULARY, egressPassGroups } from "./secret-vocabulary.js";
|
|
2
|
+
function passZeroMapper(batches) {
|
|
3
|
+
const cursors = batches.map(() => ({ i: 0, delta: 0 }));
|
|
4
|
+
return (pos) => {
|
|
5
|
+
let p = pos;
|
|
6
|
+
for (let b = batches.length - 1; b >= 0; b--) {
|
|
7
|
+
const edits = batches[b];
|
|
8
|
+
const c = cursors[b];
|
|
9
|
+
let mapped;
|
|
10
|
+
while (mapped === undefined && c.i < edits.length) {
|
|
11
|
+
const e = edits[c.i];
|
|
12
|
+
const outStart = e.at + c.delta;
|
|
13
|
+
if (p < outStart)
|
|
14
|
+
mapped = p - c.delta;
|
|
15
|
+
else if (p < outStart + e.insertedLen)
|
|
16
|
+
mapped = e.at;
|
|
17
|
+
else {
|
|
18
|
+
c.delta += e.insertedLen - e.removedLen;
|
|
19
|
+
c.i++;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
p = mapped ?? p - c.delta;
|
|
23
|
+
}
|
|
24
|
+
return p;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
const FORMAT_CHAR_RE = /\p{Cf}/u;
|
|
28
|
+
const MAX_FORMAT_SLOT_SCANS = 64;
|
|
29
|
+
function finding(pass, span, family) {
|
|
30
|
+
const f = { kind: pass.kind, confidence: pass.confidence, span, marker: pass.marker, source: pass.source };
|
|
31
|
+
if (family !== undefined)
|
|
32
|
+
f.family = family;
|
|
33
|
+
return f;
|
|
34
|
+
}
|
|
35
|
+
function insertedFor(pass, match, groups) {
|
|
36
|
+
return typeof pass.replace === "string"
|
|
37
|
+
? pass.replace.replace(/\$(\d)/g, (_m, d) => groups[Number(d) - 1] ?? "")
|
|
38
|
+
: pass.replace(match, ...groups);
|
|
39
|
+
}
|
|
40
|
+
function removedRegion(pass, match, groups, inserted) {
|
|
41
|
+
if (typeof pass.replace === "string") {
|
|
42
|
+
const parts = pass.replace.split(/(\$\d)/).filter((p) => p !== "");
|
|
43
|
+
const isRef = (p) => /^\$\d$/.test(p);
|
|
44
|
+
const refText = (p) => groups[Number(p[1]) - 1] ?? "";
|
|
45
|
+
let head = "";
|
|
46
|
+
for (const p of parts) {
|
|
47
|
+
if (!isRef(p))
|
|
48
|
+
break;
|
|
49
|
+
head += refText(p);
|
|
50
|
+
}
|
|
51
|
+
let tail = "";
|
|
52
|
+
for (let i = parts.length - 1; i >= 0 && isRef(parts[i]); i--)
|
|
53
|
+
tail = refText(parts[i]) + tail;
|
|
54
|
+
if (parts.every(isRef))
|
|
55
|
+
return [0, match.length];
|
|
56
|
+
if (!match.startsWith(head))
|
|
57
|
+
head = "";
|
|
58
|
+
if (!match.endsWith(tail) || head.length + tail.length > match.length)
|
|
59
|
+
tail = "";
|
|
60
|
+
return [head.length, match.length - tail.length];
|
|
61
|
+
}
|
|
62
|
+
const markerAt = pass.marker === "" ? -1 : inserted.lastIndexOf(pass.marker);
|
|
63
|
+
const headStop = markerAt < 0 ? inserted.length : markerAt;
|
|
64
|
+
const tailStop = markerAt < 0 ? inserted.length : inserted.length - markerAt - pass.marker.length;
|
|
65
|
+
let head = 0;
|
|
66
|
+
while (head < match.length && head < headStop && match[head] === inserted[head])
|
|
67
|
+
head++;
|
|
68
|
+
let tail = 0;
|
|
69
|
+
while (tail < match.length - head && tail < inserted.length - head && tail < tailStop && match[match.length - 1 - tail] === inserted[inserted.length - 1 - tail])
|
|
70
|
+
tail++;
|
|
71
|
+
return [head, match.length - tail];
|
|
72
|
+
}
|
|
73
|
+
const SOLO_SCAN_WORK_PER_CHAR = 8;
|
|
74
|
+
const SOLO_SCAN_WORK_FLOOR = 1 << 17;
|
|
75
|
+
function soloHits(input, passes, overStrip) {
|
|
76
|
+
const hits = [];
|
|
77
|
+
passes.forEach((pass, i) => {
|
|
78
|
+
const re = new RegExp(pass.re.source, pass.re.flags.includes("g") ? pass.re.flags : `${pass.re.flags}g`);
|
|
79
|
+
const rescans = pass.family !== undefined;
|
|
80
|
+
let work = Math.max(SOLO_SCAN_WORK_PER_CHAR * input.length, SOLO_SCAN_WORK_FLOOR);
|
|
81
|
+
for (let m = re.exec(input); m !== null; m = re.exec(input)) {
|
|
82
|
+
const at = m.index;
|
|
83
|
+
const familyAt = () => (typeof pass.family === "function" ? pass.family(input, at) : pass.family);
|
|
84
|
+
if (rescans) {
|
|
85
|
+
work -= m[0].length;
|
|
86
|
+
if (work < 0) {
|
|
87
|
+
hits.push({ start: at, end: input.length, marker: pass.marker === "" ? overStrip : pass.marker, pass: i, family: familyAt(), matchStart: at, matchEnd: input.length });
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
re.lastIndex = at + 1;
|
|
91
|
+
}
|
|
92
|
+
else
|
|
93
|
+
re.lastIndex = at + Math.max(m[0].length, 1);
|
|
94
|
+
const inserted = insertedFor(pass, m[0], m.slice(1));
|
|
95
|
+
if (inserted === m[0])
|
|
96
|
+
continue;
|
|
97
|
+
const [a, b] = removedRegion(pass, m[0], m.slice(1), inserted);
|
|
98
|
+
if (b <= a)
|
|
99
|
+
continue;
|
|
100
|
+
const family = familyAt();
|
|
101
|
+
hits.push({ start: at + a, end: at + b, marker: inserted.slice(a, inserted.length - (m[0].length - b)), pass: i, family, matchStart: at, matchEnd: at + m[0].length });
|
|
102
|
+
}
|
|
103
|
+
});
|
|
104
|
+
return hits;
|
|
105
|
+
}
|
|
106
|
+
function unremovedHits(hits, removals) {
|
|
107
|
+
const gone = [];
|
|
108
|
+
for (const r of [...removals].filter((x) => x.end > x.start).sort((x, y) => x.start - y.start || y.end - x.end)) {
|
|
109
|
+
const last = gone[gone.length - 1];
|
|
110
|
+
if (last !== undefined && r.start <= last[1])
|
|
111
|
+
last[1] = Math.max(last[1], r.end);
|
|
112
|
+
else
|
|
113
|
+
gone.push([r.start, r.end]);
|
|
114
|
+
}
|
|
115
|
+
return hits.filter((h) => {
|
|
116
|
+
let lo = 0;
|
|
117
|
+
let hi = gone.length - 1;
|
|
118
|
+
let at = -1;
|
|
119
|
+
while (lo <= hi) {
|
|
120
|
+
const mid = (lo + hi) >> 1;
|
|
121
|
+
if (gone[mid][0] <= h.start) {
|
|
122
|
+
at = mid;
|
|
123
|
+
lo = mid + 1;
|
|
124
|
+
}
|
|
125
|
+
else
|
|
126
|
+
hi = mid - 1;
|
|
127
|
+
}
|
|
128
|
+
return at === -1 || gone[at][1] < h.end;
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
function sequentialRemovals(batches) {
|
|
132
|
+
const removals = [];
|
|
133
|
+
for (let j = 0; j < batches.length; j++) {
|
|
134
|
+
const toPassZero = passZeroMapper(batches.slice(0, j));
|
|
135
|
+
for (const e of batches[j]) {
|
|
136
|
+
const start = toPassZero(e.at + e.head);
|
|
137
|
+
const end = toPassZero(e.at + e.removedLen - e.tail);
|
|
138
|
+
removals.push({ start, end: Math.max(start, end), marker: e.inserted.slice(e.head, e.insertedLen - e.tail), pass: e.pass, family: e.family });
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return removals;
|
|
142
|
+
}
|
|
143
|
+
function renderRemovals(input, removals, overStrip) {
|
|
144
|
+
const sorted = removals.filter((r) => r.end > r.start || r.marker !== "").sort((x, y) => x.start - y.start || y.end - x.end);
|
|
145
|
+
const edits = [];
|
|
146
|
+
let out = "";
|
|
147
|
+
let cursor = 0;
|
|
148
|
+
for (let i = 0; i < sorted.length;) {
|
|
149
|
+
const start = sorted[i].start;
|
|
150
|
+
let end = sorted[i].end;
|
|
151
|
+
let widest;
|
|
152
|
+
for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].marker === "" && sorted[i].start === end)); i++) {
|
|
153
|
+
const r = sorted[i];
|
|
154
|
+
if (r.end > end)
|
|
155
|
+
end = r.end;
|
|
156
|
+
if (r.marker !== "" && (widest === undefined || r.end - r.start > widest.end - widest.start))
|
|
157
|
+
widest = r;
|
|
158
|
+
}
|
|
159
|
+
const marker = widest?.marker ?? overStrip;
|
|
160
|
+
out += input.slice(cursor, start) + marker;
|
|
161
|
+
cursor = end;
|
|
162
|
+
edits.push({ at: start, removedLen: end - start, insertedLen: marker.length, inserted: marker, family: widest?.family, head: 0, tail: 0, pass: widest?.pass ?? sorted[i - 1].pass });
|
|
163
|
+
}
|
|
164
|
+
return { out: out + input.slice(cursor), edits };
|
|
165
|
+
}
|
|
166
|
+
function runPasses(input, passes, findings, toOriginal) {
|
|
167
|
+
const batches = [];
|
|
168
|
+
let cur = input;
|
|
169
|
+
passes.forEach((pass, passIndex) => {
|
|
170
|
+
const edits = [];
|
|
171
|
+
const passInput = cur;
|
|
172
|
+
const toPassZero = passZeroMapper(batches.slice());
|
|
173
|
+
cur = cur.replace(pass.re, (...args) => {
|
|
174
|
+
const match = args[0];
|
|
175
|
+
const offset = args[args.length - 2];
|
|
176
|
+
const groups = args.slice(1, -2);
|
|
177
|
+
const inserted = insertedFor(pass, match, groups);
|
|
178
|
+
if (inserted === match)
|
|
179
|
+
return inserted;
|
|
180
|
+
const family = typeof pass.family === "function" ? pass.family(passInput, offset) : pass.family;
|
|
181
|
+
if (findings !== undefined)
|
|
182
|
+
findings.push({ pass: passIndex, f: finding(pass, [toOriginal(toPassZero(offset)), toOriginal(toPassZero(offset + match.length))], family) });
|
|
183
|
+
const [a, b] = removedRegion(pass, match, groups, inserted);
|
|
184
|
+
edits.push({ at: offset, removedLen: match.length, insertedLen: inserted.length, inserted, family, head: a, tail: match.length - b, pass: passIndex });
|
|
185
|
+
return inserted;
|
|
186
|
+
});
|
|
187
|
+
batches.push(edits);
|
|
188
|
+
});
|
|
189
|
+
return { out: cur, batches };
|
|
190
|
+
}
|
|
191
|
+
function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = false) {
|
|
192
|
+
const found = report === undefined ? undefined : [];
|
|
193
|
+
const plain = runPasses(input, passes, found, toOriginal);
|
|
194
|
+
const solo = soloHits(input, passes, overStrip);
|
|
195
|
+
const sequential = solo.length === 0 ? [] : sequentialRemovals(plain.batches);
|
|
196
|
+
const outer = unremovedHits(solo, sequential);
|
|
197
|
+
const mappable = !disjoint || replacedSpans(plain.batches) !== undefined;
|
|
198
|
+
const whole = outer.length === 0 && mappable ? undefined : renderRemovals(input, [...sequential, ...outer], overStrip);
|
|
199
|
+
if (whole === undefined || (whole.out === plain.out && mappable)) {
|
|
200
|
+
if (report !== undefined)
|
|
201
|
+
report.findings.push(...found.map((x) => x.f));
|
|
202
|
+
return plain;
|
|
203
|
+
}
|
|
204
|
+
if (report !== undefined) {
|
|
205
|
+
const seen = new Set(found.map((x) => `${x.pass}:${x.f.span[0]}:${x.f.span[1]}`));
|
|
206
|
+
for (const h of outer) {
|
|
207
|
+
const span = [toOriginal(h.matchStart), toOriginal(h.matchEnd)];
|
|
208
|
+
const key = `${h.pass}:${span[0]}:${span[1]}`;
|
|
209
|
+
if (seen.has(key))
|
|
210
|
+
continue;
|
|
211
|
+
seen.add(key);
|
|
212
|
+
found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family) });
|
|
213
|
+
}
|
|
214
|
+
found.sort((x, y) => x.pass - y.pass || x.f.span[0] - y.f.span[0]);
|
|
215
|
+
report.findings.push(...found.map((x) => x.f));
|
|
216
|
+
}
|
|
217
|
+
return { out: whole.out, batches: [whole.edits] };
|
|
218
|
+
}
|
|
219
|
+
function replacedSpans(batches) {
|
|
220
|
+
const spans = [];
|
|
221
|
+
for (let j = 0; j < batches.length; j++) {
|
|
222
|
+
const toPassZero = passZeroMapper(batches.slice(0, j));
|
|
223
|
+
for (const e of batches[j]) {
|
|
224
|
+
const s0 = toPassZero(e.at);
|
|
225
|
+
const e0 = toPassZero(e.at + e.removedLen);
|
|
226
|
+
if (e0 - s0 !== e.removedLen)
|
|
227
|
+
return undefined;
|
|
228
|
+
spans.push({ start: s0, end: e0, inserted: e.inserted, pass: e.pass, family: e.family });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
spans.sort((a, b) => a.start - b.start);
|
|
232
|
+
for (let i = 1; i < spans.length; i++)
|
|
233
|
+
if (spans[i].start < spans[i - 1].end)
|
|
234
|
+
return undefined;
|
|
235
|
+
return spans;
|
|
236
|
+
}
|
|
237
|
+
function readFormatFree(input) {
|
|
238
|
+
const viewChars = [];
|
|
239
|
+
const viewToOriginal = [];
|
|
240
|
+
const formatAtSlot = [];
|
|
241
|
+
const charStartOffsets = [];
|
|
242
|
+
{
|
|
243
|
+
let orig = 0;
|
|
244
|
+
let viewOff = 0;
|
|
245
|
+
let slotBuf = "";
|
|
246
|
+
for (const ch of input) {
|
|
247
|
+
if (FORMAT_CHAR_RE.test(ch)) {
|
|
248
|
+
slotBuf += ch;
|
|
249
|
+
}
|
|
250
|
+
else {
|
|
251
|
+
formatAtSlot.push(slotBuf);
|
|
252
|
+
slotBuf = "";
|
|
253
|
+
viewToOriginal.push(orig);
|
|
254
|
+
charStartOffsets.push(viewOff);
|
|
255
|
+
viewChars.push(ch);
|
|
256
|
+
viewOff += ch.length;
|
|
257
|
+
}
|
|
258
|
+
orig += ch.length;
|
|
259
|
+
}
|
|
260
|
+
formatAtSlot.push(slotBuf);
|
|
261
|
+
viewToOriginal.push(input.length);
|
|
262
|
+
charStartOffsets.push(viewOff);
|
|
263
|
+
}
|
|
264
|
+
const view = viewChars.join("");
|
|
265
|
+
const offsetToCharIndex = new Map();
|
|
266
|
+
charStartOffsets.forEach((off, idx) => offsetToCharIndex.set(off, idx));
|
|
267
|
+
const toOriginal = (pos) => {
|
|
268
|
+
const idx = offsetToCharIndex.get(pos);
|
|
269
|
+
return idx === undefined ? input.length : viewToOriginal[idx];
|
|
270
|
+
};
|
|
271
|
+
const toOriginalSpan = (viewStart, viewEnd) => {
|
|
272
|
+
const startIdx = offsetToCharIndex.get(viewStart) ?? viewChars.length;
|
|
273
|
+
const endIdx = offsetToCharIndex.get(viewEnd) ?? viewChars.length;
|
|
274
|
+
const lastIdx = endIdx - 1;
|
|
275
|
+
const endOrig = lastIdx >= startIdx && lastIdx < viewChars.length ? viewToOriginal[lastIdx] + viewChars[lastIdx].length : viewToOriginal[startIdx];
|
|
276
|
+
return { start: viewToOriginal[startIdx], end: endOrig };
|
|
277
|
+
};
|
|
278
|
+
const cuts = [];
|
|
279
|
+
for (let k = 1; k < viewChars.length; k++)
|
|
280
|
+
if (formatAtSlot[k] !== "")
|
|
281
|
+
cuts.push(charStartOffsets[k]);
|
|
282
|
+
return { view, cuts, toOriginal, toOriginalSpan };
|
|
283
|
+
}
|
|
284
|
+
export function runRedactionPasses(input, passes, run) {
|
|
285
|
+
const { overStrip, readings, report } = run;
|
|
286
|
+
if (!readings.formatFree || !FORMAT_CHAR_RE.test(input))
|
|
287
|
+
return applyPasses(input, passes, overStrip, report, (pos) => pos).out;
|
|
288
|
+
const reading = readFormatFree(input);
|
|
289
|
+
const { view } = reading;
|
|
290
|
+
const { batches } = applyPasses(view, passes, overStrip, report, reading.toOriginal, true);
|
|
291
|
+
const viewSpans = replacedSpans(batches);
|
|
292
|
+
if (viewSpans === undefined)
|
|
293
|
+
return overStrip;
|
|
294
|
+
const spans = viewSpans.map((sp) => ({ ...reading.toOriginalSpan(sp.start, sp.end), inserted: sp.inserted }));
|
|
295
|
+
if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
|
|
296
|
+
return overStrip;
|
|
297
|
+
const extra = [];
|
|
298
|
+
if (readings.raw) {
|
|
299
|
+
const rawSpans = replacedSpans(applyPasses(input, passes, overStrip, undefined, (pos) => pos, true).batches);
|
|
300
|
+
if (rawSpans !== undefined)
|
|
301
|
+
extra.push(...rawSpans);
|
|
302
|
+
}
|
|
303
|
+
for (const cut of reading.cuts) {
|
|
304
|
+
const cutSpans = replacedSpans(applyPasses(view.slice(cut), passes, overStrip, undefined, (pos) => pos, true).batches);
|
|
305
|
+
if (cutSpans === undefined)
|
|
306
|
+
continue;
|
|
307
|
+
for (const c of cutSpans) {
|
|
308
|
+
const o = reading.toOriginalSpan(c.start + cut, c.end + cut);
|
|
309
|
+
extra.push({ ...c, start: o.start, end: o.end });
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
extra.sort((a, b) => a.start - b.start);
|
|
313
|
+
const firstEndingAfter = (pos) => {
|
|
314
|
+
let lo = 0;
|
|
315
|
+
let hi = spans.length;
|
|
316
|
+
while (lo < hi) {
|
|
317
|
+
const mid = (lo + hi) >> 1;
|
|
318
|
+
if (spans[mid].end > pos)
|
|
319
|
+
hi = mid;
|
|
320
|
+
else
|
|
321
|
+
lo = mid + 1;
|
|
322
|
+
}
|
|
323
|
+
return lo;
|
|
324
|
+
};
|
|
325
|
+
for (const r of extra) {
|
|
326
|
+
const at = firstEndingAfter(r.start);
|
|
327
|
+
const hit = spans[at];
|
|
328
|
+
if (hit !== undefined && hit.start < r.end) {
|
|
329
|
+
if (r.start < hit.start)
|
|
330
|
+
hit.start = r.start;
|
|
331
|
+
if (r.end > hit.end)
|
|
332
|
+
hit.end = r.end;
|
|
333
|
+
let next = at + 1;
|
|
334
|
+
while (next < spans.length && spans[next].start < hit.end)
|
|
335
|
+
hit.end = Math.max(hit.end, spans[next++].end);
|
|
336
|
+
if (next > at + 1)
|
|
337
|
+
spans.splice(at + 1, next - at - 1);
|
|
338
|
+
continue;
|
|
339
|
+
}
|
|
340
|
+
spans.splice(at, 0, { start: r.start, end: r.end, inserted: r.inserted });
|
|
341
|
+
if (report !== undefined)
|
|
342
|
+
report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
|
|
343
|
+
}
|
|
344
|
+
let result = "";
|
|
345
|
+
let cursor = 0;
|
|
346
|
+
for (const sp of spans) {
|
|
347
|
+
result += input.slice(cursor, sp.start) + sp.inserted;
|
|
348
|
+
cursor = sp.end;
|
|
349
|
+
}
|
|
350
|
+
result += input.slice(cursor);
|
|
351
|
+
return result;
|
|
352
|
+
}
|
|
353
|
+
const SECRET_GRADES = ["strict", "wide"];
|
|
354
|
+
export const SECRET_READING_KINDS = Object.freeze(["raw", "format-free"]);
|
|
355
|
+
function fail(what) {
|
|
356
|
+
throw new TypeError(`secret scan: ${what}`);
|
|
357
|
+
}
|
|
358
|
+
const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
359
|
+
function readPolicy(policy) {
|
|
360
|
+
if (!isRecord(policy))
|
|
361
|
+
fail("the policy is not an object");
|
|
362
|
+
const { grade, readings, urlUserinfo, onUnresolvedReading } = policy;
|
|
363
|
+
if (!SECRET_GRADES.includes(grade))
|
|
364
|
+
fail(`policy.grade must be one of ${SECRET_GRADES.join(" / ")}`);
|
|
365
|
+
if (!Array.isArray(readings) || readings.length === 0)
|
|
366
|
+
fail("policy.readings must be a non-empty array");
|
|
367
|
+
const kinds = new Set();
|
|
368
|
+
for (const r of readings) {
|
|
369
|
+
const named = isRecord(r) ? r.kind : undefined;
|
|
370
|
+
const kind = SECRET_READING_KINDS.find((k) => k === named);
|
|
371
|
+
if (kind === undefined)
|
|
372
|
+
fail(`every policy.readings entry is { kind } with kind one of ${SECRET_READING_KINDS.join(" / ")}`);
|
|
373
|
+
if (Object.keys(r).some((k) => k !== "kind"))
|
|
374
|
+
fail(`the reading "${kind}" takes no parameter`);
|
|
375
|
+
kinds.add(kind);
|
|
376
|
+
}
|
|
377
|
+
if (urlUserinfo !== "keep-username" && urlUserinfo !== "whole")
|
|
378
|
+
fail("policy.urlUserinfo must be keep-username / whole");
|
|
379
|
+
if (onUnresolvedReading !== "secret" && onUnresolvedReading !== "ignore")
|
|
380
|
+
fail("policy.onUnresolvedReading must be secret / ignore");
|
|
381
|
+
const set = { raw: kinds.has("raw"), formatFree: kinds.has("format-free") };
|
|
382
|
+
return { grade, readings: set, urlUserinfo, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
|
|
383
|
+
}
|
|
384
|
+
const ROW_IDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.id));
|
|
385
|
+
const ROW_KINDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.kind));
|
|
386
|
+
const GROUP_REFERENCE = /\$\d/;
|
|
387
|
+
function readNotation(notation) {
|
|
388
|
+
if (!isRecord(notation))
|
|
389
|
+
fail("the notation is not an object");
|
|
390
|
+
const { value, byKind, byFamily, overStrip } = notation;
|
|
391
|
+
const mark = (v, at) => typeof v !== "string" || v === "" ? fail(`${at} must be a non-empty string`) : GROUP_REFERENCE.test(v) ? fail(`${at} holds "$" before a digit, which the engine reads as a group of the match`) : v;
|
|
392
|
+
const base = mark(value, "notation.value");
|
|
393
|
+
const whole = mark(overStrip, "notation.overStrip");
|
|
394
|
+
const table = (t, at, names) => {
|
|
395
|
+
if (t === undefined)
|
|
396
|
+
return {};
|
|
397
|
+
if (!isRecord(t))
|
|
398
|
+
return fail(`${at} must be an object`);
|
|
399
|
+
const out = Object.create(null);
|
|
400
|
+
for (const [k, v] of Object.entries(t)) {
|
|
401
|
+
if (!names.has(k))
|
|
402
|
+
fail(`${at} names "${k}", which no row of the vocabulary is`);
|
|
403
|
+
out[k] = mark(v, `${at}["${k}"]`);
|
|
404
|
+
}
|
|
405
|
+
return out;
|
|
406
|
+
};
|
|
407
|
+
const kinds = table(byKind, "notation.byKind", ROW_KINDS);
|
|
408
|
+
const families = table(byFamily, "notation.byFamily", ROW_IDS);
|
|
409
|
+
return {
|
|
410
|
+
markOf: (row) => families[row.id] ?? kinds[row.kind] ?? base,
|
|
411
|
+
overStrip: whole,
|
|
412
|
+
key: JSON.stringify([base, whole, Object.entries(kinds), Object.entries(families)]),
|
|
413
|
+
};
|
|
414
|
+
}
|
|
415
|
+
const USERINFO_FAMILY = "url-userinfo";
|
|
416
|
+
if (!ROW_IDS.has(USERINFO_FAMILY))
|
|
417
|
+
throw new Error(`secret scan: the vocabulary has no "${USERINFO_FAMILY}" row`);
|
|
418
|
+
const BARE_USERNAME_MAX_LEN = 20;
|
|
419
|
+
function userinfoWithheld(userinfo, urlUserinfo) {
|
|
420
|
+
if (urlUserinfo === "whole" || userinfo.includes("@"))
|
|
421
|
+
return [0, userinfo.length];
|
|
422
|
+
const colonIdx = userinfo.indexOf(":");
|
|
423
|
+
if (colonIdx === -1)
|
|
424
|
+
return userinfo.length >= BARE_USERNAME_MAX_LEN ? [0, userinfo.length] : undefined;
|
|
425
|
+
return [colonIdx + 1, userinfo.length];
|
|
426
|
+
}
|
|
427
|
+
const capturesLabel = (re) => new RegExp(`${re.source}|`).exec("").length > 1;
|
|
428
|
+
export function egressPasses(groups, markOf, urlUserinfo) {
|
|
429
|
+
return groups.map((rows) => {
|
|
430
|
+
const head = rows[0];
|
|
431
|
+
const marker = markOf(head);
|
|
432
|
+
const split = rows.find((r) => markOf(r) !== marker);
|
|
433
|
+
if (split !== undefined)
|
|
434
|
+
fail(`notation.byFamily spells "${head.id}" and "${split.id}" differently, but the two rows run as one pass`);
|
|
435
|
+
const re = rows.length === 1 ? head.match : new RegExp(rows.map((r) => `(?:${r.match.source})`).join("|"), head.match.flags);
|
|
436
|
+
const sticky = rows.map((r) => ({ id: r.id, re: new RegExp(r.match.source, r.match.flags.replace("g", "") + "y") }));
|
|
437
|
+
const family = rows.length === 1 ? head.id : (input, at) => sticky.find((s) => ((s.re.lastIndex = at), s.re.test(input)))?.id ?? head.id;
|
|
438
|
+
const replace = head.id === USERINFO_FAMILY
|
|
439
|
+
? (_m, scheme, userinfoWithAt) => {
|
|
440
|
+
const userinfo = userinfoWithAt.slice(0, -1);
|
|
441
|
+
const cut = userinfoWithheld(userinfo, urlUserinfo);
|
|
442
|
+
return cut === undefined ? `${scheme}${userinfo}@` : `${scheme}${userinfo.slice(0, cut[0])}${marker}${userinfo.slice(cut[1])}@`;
|
|
443
|
+
}
|
|
444
|
+
: capturesLabel(head.match)
|
|
445
|
+
? `$1${marker}`
|
|
446
|
+
: marker;
|
|
447
|
+
return { kind: head.kind, confidence: head.confidence, marker, source: head.source, re, replace, family };
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
function strictPasses(markOf) {
|
|
451
|
+
return ENGINE_VOCABULARY.values.flatMap((row) => {
|
|
452
|
+
if (row.strict === undefined)
|
|
453
|
+
return [];
|
|
454
|
+
const marker = markOf(row);
|
|
455
|
+
return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: new RegExp(row.strict.source, `${row.strict.flags}g`), replace: marker, family: row.id }];
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
const COMPILED_PASSES = new Map();
|
|
459
|
+
const COMPILED_PASSES_MAX = 64;
|
|
460
|
+
function tablePasses(read, marks) {
|
|
461
|
+
const key = `${read.key}\u0000${marks.key}`;
|
|
462
|
+
let passes = COMPILED_PASSES.get(key);
|
|
463
|
+
if (passes === undefined) {
|
|
464
|
+
passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), marks.markOf, read.urlUserinfo) : strictPasses(marks.markOf));
|
|
465
|
+
if (COMPILED_PASSES.size >= COMPILED_PASSES_MAX)
|
|
466
|
+
COMPILED_PASSES.clear();
|
|
467
|
+
COMPILED_PASSES.set(key, passes);
|
|
468
|
+
}
|
|
469
|
+
return passes;
|
|
470
|
+
}
|
|
471
|
+
export function renderSecrets(text, policy, notation, options = {}) {
|
|
472
|
+
const read = readPolicy(policy);
|
|
473
|
+
const marks = readNotation(notation);
|
|
474
|
+
const { passes: extra, report } = options;
|
|
475
|
+
const own = tablePasses(read, marks);
|
|
476
|
+
return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, report });
|
|
477
|
+
}
|
|
478
|
+
const FORMS = {
|
|
479
|
+
strict: ENGINE_VOCABULARY.values.flatMap((row) => (row.strict === undefined ? [] : [{ row, re: new RegExp(row.strict.source, `${row.strict.flags}g`), cut: "whole" }])),
|
|
480
|
+
wide: ENGINE_VOCABULARY.values.map((row) => ({ row, re: new RegExp(row.match.source, row.match.flags), cut: row.id === USERINFO_FAMILY ? "userinfo" : capturesLabel(row.match) ? "label" : "whole" })),
|
|
481
|
+
};
|
|
482
|
+
function scannerFor(policy) {
|
|
483
|
+
const read = readPolicy(policy);
|
|
484
|
+
return { forms: FORMS[read.grade], readings: read.readings, urlUserinfo: read.urlUserinfo };
|
|
485
|
+
}
|
|
486
|
+
function formHits(input, scan) {
|
|
487
|
+
const hits = [];
|
|
488
|
+
for (const { row, re, cut } of scan.forms) {
|
|
489
|
+
let work = Math.max(SOLO_SCAN_WORK_PER_CHAR * input.length, SOLO_SCAN_WORK_FLOOR);
|
|
490
|
+
re.lastIndex = 0;
|
|
491
|
+
for (let m = re.exec(input); m !== null; m = re.exec(input)) {
|
|
492
|
+
const at = m.index;
|
|
493
|
+
work -= m[0].length;
|
|
494
|
+
if (work < 0) {
|
|
495
|
+
hits.push({ start: at, end: input.length, family: row.id, kind: row.kind });
|
|
496
|
+
break;
|
|
497
|
+
}
|
|
498
|
+
re.lastIndex = at + 1;
|
|
499
|
+
let start = at;
|
|
500
|
+
let end = at + m[0].length;
|
|
501
|
+
if (cut === "label")
|
|
502
|
+
start += (m[1] ?? "").length;
|
|
503
|
+
else if (cut === "userinfo") {
|
|
504
|
+
const scheme = m[1] ?? "";
|
|
505
|
+
const userinfo = (m[2] ?? "").slice(0, -1);
|
|
506
|
+
const withheld = userinfoWithheld(userinfo, scan.urlUserinfo);
|
|
507
|
+
if (withheld === undefined)
|
|
508
|
+
continue;
|
|
509
|
+
start = at + scheme.length + withheld[0];
|
|
510
|
+
end = at + scheme.length + withheld[1];
|
|
511
|
+
}
|
|
512
|
+
if (end > start)
|
|
513
|
+
hits.push({ start, end, family: row.id, kind: row.kind });
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
return hits;
|
|
517
|
+
}
|
|
518
|
+
export function scanSecrets(text, policy) {
|
|
519
|
+
const scan = scannerFor(policy);
|
|
520
|
+
let spans;
|
|
521
|
+
if (!scan.readings.formatFree || !FORMAT_CHAR_RE.test(text))
|
|
522
|
+
spans = formHits(text, scan);
|
|
523
|
+
else {
|
|
524
|
+
const reading = readFormatFree(text);
|
|
525
|
+
if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
|
|
526
|
+
return { outcome: "whole", reason: "scan-budget" };
|
|
527
|
+
const back = (h, shift) => ({ ...h, ...reading.toOriginalSpan(h.start + shift, h.end + shift) });
|
|
528
|
+
spans = formHits(reading.view, scan).map((h) => back(h, 0));
|
|
529
|
+
if (scan.readings.raw)
|
|
530
|
+
spans.push(...formHits(text, scan));
|
|
531
|
+
for (const cut of reading.cuts)
|
|
532
|
+
spans.push(...formHits(reading.view.slice(cut), scan).map((h) => back(h, cut)));
|
|
533
|
+
}
|
|
534
|
+
const seen = new Set();
|
|
535
|
+
const unique = spans.filter((s) => {
|
|
536
|
+
const key = `${s.start}:${s.end}:${s.family}`;
|
|
537
|
+
return seen.has(key) ? false : (seen.add(key), true);
|
|
538
|
+
});
|
|
539
|
+
unique.sort((a, b) => a.start - b.start || a.end - b.end || (a.family < b.family ? -1 : a.family > b.family ? 1 : 0));
|
|
540
|
+
return { outcome: "spans", spans: unique };
|
|
541
|
+
}
|
|
542
|
+
export function isSecretShaped(text, policy) {
|
|
543
|
+
const scan = scanSecrets(text, policy);
|
|
544
|
+
return scan.outcome === "whole" || scan.spans.length > 0;
|
|
545
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Where a text holds a credential — answered by ONE table, {@link CREDENTIAL_VOCABULARY}, and the readings taken off it:
|
|
3
|
+
* - the strict grade (`scanSecrets` with `grade: "strict"`, secret-scan.ts): the rows that carry a `strict` form. A
|
|
4
|
+
* hit fail-closed rejects a durable write, so a strict form is narrow (vendor-distinctive prefix, long body, word
|
|
5
|
+
* boundary): a wrongly refused ordinary note is that face's head risk.
|
|
6
|
+
* - egress ({@link egressPassGroups}): every row's wide `match`, in table order, grouped into passes. A hit replaces
|
|
7
|
+
* the value in text that is persisted and forwarded, so a wide form is broad: a missed credential is that face's
|
|
8
|
+
* head risk and over-redaction is cheap.
|
|
9
|
+
* - env names ({@link envNameView}): the exact credential names, the credential words by position, the
|
|
10
|
+
* connection-string name shape and the token-prefix value predicate a spawned child's environment is judged by.
|
|
11
|
+
* - record keys ({@link recordKeyView}): the same credential words, read as the name of a member of a structured record
|
|
12
|
+
* (a header name, a config member) rather than of an environment variable.
|
|
13
|
+
*
|
|
14
|
+
* A family's strict and wide forms live on the SAME row, and every text the strict form matches the wide form matches
|
|
15
|
+
* too (machine-judged over the fixture file by `scripts/verify-credential-vocabulary.mjs`): the two faces may differ in
|
|
16
|
+
* floors, never in vocabulary. Detection is shared; each layer renders its own marker.
|
|
17
|
+
*
|
|
18
|
+
* The table holds no layer's policy: which form a reader reads, how a text is read first, which marker replaces a hit
|
|
19
|
+
* and whether a URL's username is kept are the caller's, passed in whole (`SecretScanPolicy`, `SecretNotation` in
|
|
20
|
+
* secret-scan.ts). Zero imports by design: the gate reads this module alone.
|
|
21
|
+
*/
|
|
22
|
+
/** How sure a detection is. */
|
|
23
|
+
export type RedactionConfidence = "high" | "medium" | "low";
|
|
24
|
+
/** Which rule table a finding names as its minter. A closed set: a report's entries can only have been written by
|
|
25
|
+
* the engine's own tables, so "this finding says X" is an engine statement. */
|
|
26
|
+
export type RedactionSource = "arg-summary" | "untrusted-egress" | "secret-env";
|
|
27
|
+
/** One value family: "this run of characters is itself a credential". */
|
|
28
|
+
export interface ValueFamilyRow {
|
|
29
|
+
/** Stable family name, `^[a-z0-9-]+$`; a finding's `family` and a span's `family`. */
|
|
30
|
+
readonly id: string;
|
|
31
|
+
/** The finding `kind` this family reports (the coarse class a consumer groups by). */
|
|
32
|
+
readonly kind: string;
|
|
33
|
+
readonly source: RedactionSource;
|
|
34
|
+
readonly confidence: RedactionConfidence;
|
|
35
|
+
/** The wide form (egress), global. A capture group 1, when the form has one, is a label kept verbatim ahead of the
|
|
36
|
+
* marker (`AccountKey=`, `api_key=`). */
|
|
37
|
+
readonly match: RegExp;
|
|
38
|
+
/** The strict form, never global. Present = the strict grade reads this family (a reader that refuses on a hit, such
|
|
39
|
+
* as a durable write, refuses it); absent = that grade does not (being read there is the tightening, so it is
|
|
40
|
+
* written, never implied). */
|
|
41
|
+
readonly strict?: RegExp;
|
|
42
|
+
/** The egress pass this wide form runs in. Rows naming the same pass sit next to each other and join as ONE
|
|
43
|
+
* alternation, so a token is taken at its leftmost start whichever of them it is (a nested shape is never cut by
|
|
44
|
+
* an earlier family's pass). Absent = a pass of its own. */
|
|
45
|
+
readonly pass?: string;
|
|
46
|
+
}
|
|
47
|
+
/** Where a credential word counts in a name (env names are upper-cased words joined by `_`):
|
|
48
|
+
* - `leading` — at the start of the name or right after `_`;
|
|
49
|
+
* - `plural` — the word plus `S`, in the leading position;
|
|
50
|
+
* - `counted-plural` — a plural, unless a count word (`MAX_TOKENS`) stands right in front of it;
|
|
51
|
+
* - `separated` — only right after `_` (the shell's own `PWD` is none; a record has no such variable, so the record-key
|
|
52
|
+
* view reads it as `leading`);
|
|
53
|
+
* - `embedded` — anywhere (`AUTHTOKEN`, `MYSECRET`).
|
|
54
|
+
* In every position the word must end the name or be followed by `_` or a digit (`KEYBOARD` is no word). */
|
|
55
|
+
export type CredentialWordPosition = "leading" | "plural" | "counted-plural" | "separated" | "embedded";
|
|
56
|
+
/** One credential word: "a name holding this word names a credential". */
|
|
57
|
+
export interface CredentialWordRow {
|
|
58
|
+
readonly word: string;
|
|
59
|
+
readonly position: CredentialWordPosition;
|
|
60
|
+
/** Further words that name the same credential as a record key only, read in this row's position by the record-key
|
|
61
|
+
* view alone (`AUTH` → the `Authorization` header). The env-name view never reads them: a variable named
|
|
62
|
+
* `AUTHORIZATION` is not withheld by its name. */
|
|
63
|
+
readonly recordKeys?: readonly string[];
|
|
64
|
+
}
|
|
65
|
+
export interface CredentialVocabulary {
|
|
66
|
+
/** Value families, in egress pass order. */
|
|
67
|
+
readonly values: readonly ValueFamilyRow[];
|
|
68
|
+
readonly words: readonly CredentialWordRow[];
|
|
69
|
+
/** The words that turn a following counted plural into a number. */
|
|
70
|
+
readonly countWords: readonly string[];
|
|
71
|
+
/** A connection string named as such (`DB_CONN_STR`, `CONNECTION_STRINGS`), case-insensitive. */
|
|
72
|
+
readonly connectionStringName: RegExp;
|
|
73
|
+
/** Credential env names, each also judged under the `INPUT_` prefix (a GitHub Actions input). */
|
|
74
|
+
readonly exactEnvNames: readonly string[];
|
|
75
|
+
/** Credential-shaped names that hold no credential: declared names, never a pattern over names (the one pattern is
|
|
76
|
+
* {@link CredentialVocabulary.nonSecretNameShape}). Three readings of the same list: the env-name view and the
|
|
77
|
+
* record-member reading ({@link isSensitiveRecordKey}) take each as the WHOLE name (`CACHE_KEY` holds a setting,
|
|
78
|
+
* `SECRET_CACHE_KEY` a secret, in an environment and in a config or detail record alike); the HTTP header-name
|
|
79
|
+
* reading ({@link isSensitiveHeaderName}) takes it as the whole name or the TAIL of a longer one whose front part names
|
|
80
|
+
* no credential word (`cf-aig-cache-key`, `X-Idempotency-Key`: a gateway's header for the same non-secret thing;
|
|
81
|
+
* `secret-cache-key` stays a secret). */
|
|
82
|
+
readonly nonSecretExactNames: readonly string[];
|
|
83
|
+
/** The one name shape that holds no credential whatever words it carries: git's `GIT_CONFIG_KEY_<n…>` names a
|
|
84
|
+
* configuration key (its value sits in `GIT_CONFIG_VALUE_<n>`). Read by every name view, beside the exact names. */
|
|
85
|
+
readonly nonSecretNameShape: RegExp;
|
|
86
|
+
/** A leading vendor prefix that spells a credential word without being one (`AUTH0_`): dropped from a name before the
|
|
87
|
+
* credential words are looked for, in every name view alike. */
|
|
88
|
+
readonly nonCredentialPrefix: RegExp;
|
|
89
|
+
/** A value (or a password-less URL user) opening with a known token prefix. */
|
|
90
|
+
readonly tokenPrefix: RegExp;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* A URL of any scheme ({@link URI_SCHEME_SOURCE}, then `://`), to the first white space, quote or backtick, any case. URL
|
|
94
|
+
* grammar, not a credential form: a layer that withholds whole URLs (host internals in text shown to a person) reads this
|
|
95
|
+
* value, so the scheme is read one way wherever a URL is looked for. Global: read it with `replace` / `matchAll`, never
|
|
96
|
+
* with a `lastIndex` of the caller's own; a layer that runs it among its own passes runs its own copy (same source and
|
|
97
|
+
* flags), which no other reader of this object can recompile.
|
|
98
|
+
*/
|
|
99
|
+
export declare const URL_OF_ANY_SCHEME: RegExp;
|
|
100
|
+
/** The fewest characters a string can have and still be read as a credential: the floor of a vendor-prefixed token's body
|
|
101
|
+
* and of a bearer / authorization credential run in this table, and of the value rules of the env scrub. A shorter
|
|
102
|
+
* string names no secret, whoever holds it. */
|
|
103
|
+
export declare const CREDENTIAL_MIN_CHARS = 8;
|
|
104
|
+
export declare const CREDENTIAL_VOCABULARY: CredentialVocabulary;
|
|
105
|
+
/** The table the package's own readers run (the matcher, the renderer, the record-key and header-name readings, the
|
|
106
|
+
* env-name view's default): {@link CREDENTIAL_VOCABULARY} with every regex its own object, copied as this module is
|
|
107
|
+
* evaluated — before any importer's code runs. A regex is an object a caller can recompile (`RegExp.prototype.compile`
|
|
108
|
+
* rewrites its pattern, frozen or not) or whose `lastIndex` it can move; one read off the exported table changes only
|
|
109
|
+
* that caller's reading. Not exported from the package entry. */
|
|
110
|
+
export declare const ENGINE_VOCABULARY: CredentialVocabulary;
|
|
111
|
+
/** The egress view: the rows in table order, each run of adjacent rows naming the same `pass` as one group. */
|
|
112
|
+
export declare function egressPassGroups(vocabulary: CredentialVocabulary): ReadonlyArray<readonly ValueFamilyRow[]>;
|
|
113
|
+
/** The env-name view: what {@link CREDENTIAL_VOCABULARY} says about variable names and token-prefixed values. */
|
|
114
|
+
export interface EnvNameView {
|
|
115
|
+
/** Upper-cased, `-` read as `_`; each name also under `INPUT_`. */
|
|
116
|
+
readonly exactNames: ReadonlySet<string>;
|
|
117
|
+
readonly nonSecretExactNames: ReadonlySet<string>;
|
|
118
|
+
readonly nonSecretNameShape: RegExp;
|
|
119
|
+
readonly nonCredentialPrefix: RegExp;
|
|
120
|
+
/** A credential word in its position, case-insensitive. */
|
|
121
|
+
readonly credentialWord: RegExp;
|
|
122
|
+
readonly connectionStringName: RegExp;
|
|
123
|
+
/** A name opening with `namespace` (as written) that holds a credential word as ANY whole `_` segment after it —
|
|
124
|
+
* every position but `embedded`; a counted plural no count word touches — any case, `.` crossing line breaks. */
|
|
125
|
+
readonly anySegmentUnder: (namespace: string) => RegExp;
|
|
126
|
+
readonly tokenPrefix: RegExp;
|
|
127
|
+
}
|
|
128
|
+
/** The env-name view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
|
|
129
|
+
* Every regex of a view is the view's own object, so a reader that recompiles one changes only its own view. */
|
|
130
|
+
export declare function envNameView(vocabulary?: CredentialVocabulary): EnvNameView;
|
|
131
|
+
/** `camelCase` / `PascalCase` read as `_`-separated words, `OAuth` and `NextAuth` kept whole. */
|
|
132
|
+
export declare function splitCamelCase(name: string): string;
|
|
133
|
+
/** The record-key view: what {@link CREDENTIAL_VOCABULARY} says about the names a structured record carries — the member
|
|
134
|
+
* of a config or detail record and the HTTP header — the credential words of the env-name view, each row's `recordKeys`
|
|
135
|
+
* spellings besides, and a `separated` word standing alone (`pwd`, `pass`: a record has no shell variable of that name).
|
|
136
|
+
* The names the vocabulary declares non-secret stay non-secret here too, as the whole name; the header-name reading
|
|
137
|
+
* ({@link isSensitiveHeaderName}) alone also takes one as the tail of a longer name — a gateway prefixes the header it
|
|
138
|
+
* names the same non-secret thing with (`cf-aig-cache-key`), where a secret named `secret_cache_key` lives in a
|
|
139
|
+
* body or a config record and is read whole. */
|
|
140
|
+
export interface RecordKeyView {
|
|
141
|
+
/** A credential word in its position, case-insensitive. Applied to a key with `-` read as `_`, as written and
|
|
142
|
+
* camel-split ({@link isSensitiveRecordKey}, {@link isSensitiveHeaderName}). */
|
|
143
|
+
readonly credentialWord: RegExp;
|
|
144
|
+
readonly nonSecretExactNames: ReadonlySet<string>;
|
|
145
|
+
readonly nonSecretNameShape: RegExp;
|
|
146
|
+
readonly nonCredentialPrefix: RegExp;
|
|
147
|
+
}
|
|
148
|
+
export declare function recordKeyView(vocabulary: CredentialVocabulary): RecordKeyView;
|
|
149
|
+
/** Does `key`, the name of a record's MEMBER (`Proxy-Authorization`, `pwd`, `X_API_KEY`, `clientSecret`), name a credential?
|
|
150
|
+
* Case does not matter, `-` and `_` are one. A name the vocabulary declares non-secret is none as the WHOLE name only
|
|
151
|
+
* (`partition_key`; `x_partition_key` and `secret_cache_key` are still read by their words): a
|
|
152
|
+
* secret is named by the system that holds it, and what this reading decides is whether a value is shown or kept out of
|
|
153
|
+
* a record. The HTTP header-name reading is {@link isSensitiveHeaderName}. `view` is the vocabulary's record-key view
|
|
154
|
+
* unless a caller holds another. */
|
|
155
|
+
export declare function isSensitiveRecordKey(key: string, view?: RecordKeyView): boolean;
|
|
156
|
+
/** Does `name`, an HTTP header's name (`Authorization`, `X-Api-Key`, `cf-aig-cache-key`), name a credential? The
|
|
157
|
+
* record-key reading ({@link isSensitiveRecordKey}) plus one exemption: a declared non-secret name is none also as the
|
|
158
|
+
* TAIL of a longer name, at a `_` / `-` boundary, whose front part holds no credential word (`cf-aig-cache-key`,
|
|
159
|
+
* `X-Idempotency-Key`, `X-Partition-Key`; `secret-cache-key`, `clientAuth-cache-key` and `xcache-key`
|
|
160
|
+
* are still read by their words). A gateway prefixes the header it names the same non-secret thing with, and the reading
|
|
161
|
+
* that mistakes such a header for a credential drops the key that should have gone out. `view` is the vocabulary's
|
|
162
|
+
* record-key view unless a caller holds another. */
|
|
163
|
+
export declare function isSensitiveHeaderName(name: string, view?: RecordKeyView): boolean;
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
const URI_SCHEME_SOURCE = String.raw `(?<![A-Za-z0-9+.\-_])[0-9+.\-_]*[A-Za-z][A-Za-z0-9+.\-_]*`;
|
|
2
|
+
const URL_USERINFO_SOURCE = String.raw `(?:[^/\s"@]*@)+`;
|
|
3
|
+
export const URL_OF_ANY_SCHEME = new RegExp(String.raw `${URI_SCHEME_SOURCE}:\/\/[^\s"'\`]+`, "gi");
|
|
4
|
+
export const CREDENTIAL_MIN_CHARS = 8;
|
|
5
|
+
const VENDOR_BODY = `[A-Za-z0-9_-]{${CREDENTIAL_MIN_CHARS},}`;
|
|
6
|
+
const VALUE_END = String.raw `\s"';|&`;
|
|
7
|
+
const VALUE_CHAR = `[^${VALUE_END}]`;
|
|
8
|
+
const ESCAPE_DEPTH = 63;
|
|
9
|
+
const QUOTED_VALUE_MAX = 1024;
|
|
10
|
+
const PLAIN_QUOTES = [`"`, `'`, `"`, `'`, `"`, `'`, `%22`, `%27`];
|
|
11
|
+
const ESCAPED_QUOTES = [`"`, `'`, `x22`, `x27`, `u0022`, `u0027`, `042`, `047`];
|
|
12
|
+
const BACKSLASHES = String.raw `\\{1,${ESCAPE_DEPTH}}`;
|
|
13
|
+
const QUOTE = `(?:${PLAIN_QUOTES.join("|")}|${BACKSLASHES}(?:${ESCAPED_QUOTES.join("|")}))`;
|
|
14
|
+
const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE}))`;
|
|
15
|
+
const AFTER_CLOSING_QUOTE = String.raw `(?:[\s,}\]:]|["']|$)`;
|
|
16
|
+
const VALUE_RUN_END = `(?:(?<!\\\\)|(?!\\\\{0,${ESCAPE_DEPTH}}["']))`;
|
|
17
|
+
const GLUED_RUN = `${VALUE_CHAR}*${VALUE_RUN_END}`;
|
|
18
|
+
const pairedValue = (open, close) => `${open}(?!${AFTER_CLOSING_QUOTE})[^\\n]{0,${QUOTED_VALUE_MAX}}?${close}`;
|
|
19
|
+
const QUOTED_VALUE = [
|
|
20
|
+
...PLAIN_QUOTES.map((q) => pairedValue(q, q)),
|
|
21
|
+
...ESCAPED_QUOTES.map((q, i) => pairedValue(`(${BACKSLASHES})${q}`, `(?<!\\\\)(?:\\${i + 2})${q}`)),
|
|
22
|
+
`${QUOTE}(?!${AFTER_CLOSING_QUOTE})${GLUED_RUN}`,
|
|
23
|
+
].join("|");
|
|
24
|
+
const BEARER_TAIL = `(?:(?!${QUOTE})\\S${GLUED_RUN}|(?<==)(?:${QUOTED_VALUE}))`;
|
|
25
|
+
const HEADER_PARAM = String.raw `=[ \t]*(\\{0,${ESCAPE_DEPTH}})(["'])(?:(?!(?<!\\)\2\3)[^\r\n])*(?:(?<!\\)\2\3(?=[\s,;]|\\{0,${ESCAPE_DEPTH}}["']|$)|(?=[\r\n]|$))`;
|
|
26
|
+
const HEADER_VALUE = String.raw `(?:[A-Za-z][A-Za-z0-9!#$%&*+.^_|~-]*[ \t]+)?(?:${HEADER_PARAM}|[^\s"',]|[ \t]*,[ \t]*(?=[^\s"',])|[ \t]+(?==)|(?<==)[ \t]+(?=[^\s"',]))+${VALUE_RUN_END}`;
|
|
27
|
+
const headed = (body, flags = "g") => new RegExp(`${WORD_START}${body}`, flags);
|
|
28
|
+
const HUMP_BEFORE = "[a-z0-9]";
|
|
29
|
+
const KEYWORD_VALUE_WORDS = ["token", "secret", "key", "password", "passwd", "credential"];
|
|
30
|
+
const anyCase = (word) => [...word].map((c) => `[${c.toUpperCase()}${c.toLowerCase()}]`).join("");
|
|
31
|
+
const VALUE_ROW_FIELDS = [["Account", "Key"]];
|
|
32
|
+
const leftToItsRow = (keyword) => VALUE_ROW_FIELDS.filter(([, k]) => k.toLowerCase() === keyword).map(([front]) => `(?<!(?<![A-Za-z0-9])${front})`).join("");
|
|
33
|
+
const KEYWORD_HEAD = `(?:${WORD_START}(?:${KEYWORD_VALUE_WORDS.map(anyCase).join("|")})|(?<=${HUMP_BEFORE})(?:${KEYWORD_VALUE_WORDS.map((w) => leftToItsRow(w) + w[0].toUpperCase() + anyCase(w.slice(1))).join("|")})|(?<=[A-Z])(?:${KEYWORD_VALUE_WORDS.map((w) => w[0].toUpperCase() + w[1] + anyCase(w.slice(2))).join("|")}))`;
|
|
34
|
+
const KEY_QUOTE = `(?:\\\\{0,${ESCAPE_DEPTH}}["'])?`;
|
|
35
|
+
export const CREDENTIAL_VOCABULARY = {
|
|
36
|
+
values: [
|
|
37
|
+
{ id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/ },
|
|
38
|
+
{ id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/ },
|
|
39
|
+
{ id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/ },
|
|
40
|
+
{ id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test)_[A-Za-z0-9]{24,}\b/ },
|
|
41
|
+
{ id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`) },
|
|
42
|
+
{ id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("glpat-[A-Za-z0-9_-]{20,}") },
|
|
43
|
+
{ id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}") },
|
|
44
|
+
{ id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`) },
|
|
45
|
+
{ id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("AIza[A-Za-z0-9_-]{20,}"), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
|
|
46
|
+
{ id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprs]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprs]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/ },
|
|
47
|
+
{ id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/ },
|
|
48
|
+
{ id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("hf_[A-Za-z0-9]{20,}"), strict: /\bhf_(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{20,}\b/ },
|
|
49
|
+
{ id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/ },
|
|
50
|
+
{ id: "private-key", kind: "private-key-block", source: "arg-summary", confidence: "high", match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)/g, strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/ },
|
|
51
|
+
{ id: "bearer-token", kind: "bearer-token", source: "arg-summary", confidence: "high", match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`), strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/ },
|
|
52
|
+
{ id: "authorization-header", kind: "authorization-header", source: "arg-summary", confidence: "high", match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi") },
|
|
53
|
+
{ id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: /(AccountKey=)[A-Za-z0-9+/]{40,}={0,2}/g, strict: /AccountKey=[A-Za-z0-9+/]{40,}={0,2}/ },
|
|
54
|
+
{ id: "keyword-value", kind: "keyword-value", source: "arg-summary", confidence: "medium", match: new RegExp(String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "g") },
|
|
55
|
+
{ id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${URI_SCHEME_SOURCE}:\/\/)(${URL_USERINFO_SOURCE})`, "gi") },
|
|
56
|
+
{ id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: /([?&#](?:sig|signature|sas|code)=)[^&\s"'#]+/gi },
|
|
57
|
+
],
|
|
58
|
+
words: [
|
|
59
|
+
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE", "KEY"].map((word) => ({ word, position: "leading" })),
|
|
60
|
+
{ word: "AUTH", position: "leading", recordKeys: ["AUTHORIZATION", "AUTHENTICATION", "OAUTH", "AUTHN", "AUTHZ"] },
|
|
61
|
+
...["COOKIE", "PAT", "DSN", "WEBHOOK", "CREDENTIAL", "CREDENTIALS", "CREDS", "APIKEY", "ACCESSKEY", "SECRETKEY", "ACCOUNTKEY", "PRIVATEKEY", "AUTHKEY", "SSHKEY", "SIGNINGKEY", "MASTERKEY", "DEPLOYKEY", "ENCRYPTIONKEY", "PGPASSWORD", "SSHPASS"].map((word) => ({ word, position: "leading" })),
|
|
62
|
+
...["KEY", "SECRET", "PASSWORD", "CREDENTIAL"].map((word) => ({ word, position: "plural" })),
|
|
63
|
+
{ word: "TOKEN", position: "counted-plural" },
|
|
64
|
+
...["PWD", "PASS", "JWT"].map((word) => ({ word, position: "separated" })),
|
|
65
|
+
...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE"].map((word) => ({ word, position: "embedded" })),
|
|
66
|
+
],
|
|
67
|
+
countWords: ["MAX", "OUTPUT", "INPUT", "THINKING", "NUM", "TOTAL"],
|
|
68
|
+
connectionStringName: /CONN(ECT(ION)?)?_?STR(ING)?S?(?=$|[_0-9])/i,
|
|
69
|
+
exactEnvNames: [
|
|
70
|
+
"ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_ARTIFACTS_API_TOKEN", "CLAUDE_CODE_MEMORY_API_TOKEN",
|
|
71
|
+
"CLAUDE_CODE_SLACK_TAG_TOKEN", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_FOUNDRY_API_KEY", "ANTHROPIC_FOUNDRY_AUTH_TOKEN",
|
|
72
|
+
"ANTHROPIC_AWS_API_KEY", "ANTHROPIC_CUSTOM_HEADERS", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN",
|
|
73
|
+
"AWS_BEARER_TOKEN_BEDROCK", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_GHA_CREDS_PATH", "AZURE_CLIENT_SECRET",
|
|
74
|
+
"IDENTITY_HEADER", "MSI_SECRET", "AZURE_CLIENT_CERTIFICATE_PATH", "AZURE_CLIENT_CERTIFICATE_PASSWORD",
|
|
75
|
+
"AZURE_PASSWORD", "AZURE_FEDERATED_TOKEN_FILE", "AWS_WEB_IDENTITY_TOKEN_FILE",
|
|
76
|
+
"AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", "AWS_CONTAINER_CREDENTIALS_FULL_URI",
|
|
77
|
+
"AWS_CONTAINER_AUTHORIZATION_TOKEN", "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE", "CLOUDSDK_AUTH_ACCESS_TOKEN",
|
|
78
|
+
"GOOGLE_OAUTH_ACCESS_TOKEN", "CLAUDE_CODE_OAUTH_REFRESH_TOKEN", "HF_TOKEN", "HUGGING_FACE_HUB_TOKEN",
|
|
79
|
+
"HUGGINGFACEHUB_API_TOKEN", "NODE_AUTH_TOKEN", "NUGET_AUTH_TOKEN", "CARGO_REGISTRY_TOKEN", "TWINE_PASSWORD",
|
|
80
|
+
"TWINE_USERNAME", "PYPI_TOKEN", "PYPI_API_TOKEN", "UV_PUBLISH_TOKEN", "UV_PUBLISH_PASSWORD", "UV_PUBLISH_USERNAME",
|
|
81
|
+
"FLIT_PASSWORD", "FLIT_USERNAME", "HATCH_INDEX_AUTH", "HATCH_INDEX_USER", "GEM_HOST_API_KEY", "MATURIN_PYPI_TOKEN",
|
|
82
|
+
"MATURIN_PASSWORD", "MATURIN_USERNAME", "CONAN_LOGIN_USERNAME", "CONAN_PASSWORD", "ANACONDA_API_TOKEN",
|
|
83
|
+
"BINSTAR_API_TOKEN", "VAULT_TOKEN", "VAULT_AUTH_TOKEN", "VAULT_ROLE_ID", "VAULT_SECRET_ID", "CONSUL_HTTP_TOKEN",
|
|
84
|
+
"CONSUL_HTTP_AUTH", "NOMAD_TOKEN", "NOMAD_HTTP_AUTH", "CI_REGISTRY_USER", "CI_DEPLOY_USER", "JF_USER",
|
|
85
|
+
"FASTLANE_SESSION", "MATCH_GIT_BASIC_AUTHORIZATION", "SONAR_TOKEN", "SONARQUBE_SCANNER_PARAMS",
|
|
86
|
+
"SONAR_SCANNER_JSON_PARAMS", "SLACK_WEBHOOK_URL", "SLACK_WEBHOOK", "DISCORD_WEBHOOK", "DISCORD_WEBHOOK_URL",
|
|
87
|
+
"TEAMS_WEBHOOK_URL", "MS_TEAMS_WEBHOOK_URI", "ANTHROPIC_IDENTITY_TOKEN", "ANTHROPIC_IDENTITY_TOKEN_FILE",
|
|
88
|
+
"CLOUDSDK_AUTH_ACCESS_TOKEN_FILE", "CLOUDSDK_AUTH_AUTHORIZATION_TOKEN_FILE", "AZURE_AUTH_LOCATION",
|
|
89
|
+
"ACTIONS_ID_TOKEN_REQUEST_TOKEN", "ACTIONS_ID_TOKEN_REQUEST_URL", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RUNTIME_URL",
|
|
90
|
+
"ALL_INPUTS", "VSS_NUGET_EXTERNAL_FEED_ENDPOINTS", "ARTIFACTS_CREDENTIALPROVIDER_EXTERNAL_FEED_ENDPOINTS",
|
|
91
|
+
"VSS_NUGET_ACCESSTOKEN", "ARTIFACTS_CREDENTIALPROVIDER_ACCESSTOKEN", "COMPOSER_AUTH", "OVERRIDE_GITHUB_TOKEN",
|
|
92
|
+
"DEFAULT_WORKFLOW_TOKEN", "SSH_SIGNING_KEY",
|
|
93
|
+
],
|
|
94
|
+
nonSecretExactNames: [
|
|
95
|
+
"PARTITION_KEY", "SORT_KEY", "RANGE_KEY", "IDEMPOTENCY_KEY", "CACHE_KEY",
|
|
96
|
+
"DB_FOREIGN_KEYS", "JSON_SORT_KEYS",
|
|
97
|
+
"MAX_NEW_TOKENS", "MAX_COMPLETION_TOKENS", "AIDER_MAP_TOKENS", "AIDER_MAX_CHAT_HISTORY_TOKENS",
|
|
98
|
+
"MAX_BATCH_PREFILL_TOKENS", "MAX_WAITING_TOKENS", "MAX_TOP_N_TOKENS", "RATE_LIMIT_TOKENS",
|
|
99
|
+
"PROMPT_TOKENS", "CONTEXT_TOKENS",
|
|
100
|
+
],
|
|
101
|
+
nonSecretNameShape: /^GIT_CONFIG_KEY_[0-9][A-Za-z0-9_]*$/,
|
|
102
|
+
nonCredentialPrefix: /^AUTH0_/i,
|
|
103
|
+
tokenPrefix: /^(?:gh[opusr]_|github_pat_|glpat-|xox[abpr]-|sk-|pk-|AKIA|eyJ|ya29\.|npm_)/,
|
|
104
|
+
};
|
|
105
|
+
function freezeTable(node) {
|
|
106
|
+
if (typeof node !== "object" || node === null || node instanceof RegExp || Object.isFrozen(node))
|
|
107
|
+
return;
|
|
108
|
+
for (const value of Object.values(node))
|
|
109
|
+
freezeTable(value);
|
|
110
|
+
Object.freeze(node);
|
|
111
|
+
}
|
|
112
|
+
freezeTable(CREDENTIAL_VOCABULARY);
|
|
113
|
+
function withOwnRegexes(node) {
|
|
114
|
+
if (node instanceof RegExp)
|
|
115
|
+
return new RegExp(node.source, node.flags);
|
|
116
|
+
if (Array.isArray(node))
|
|
117
|
+
return Object.freeze(node.map(withOwnRegexes));
|
|
118
|
+
if (typeof node === "object" && node !== null)
|
|
119
|
+
return Object.freeze(Object.fromEntries(Object.entries(node).map(([k, v]) => [k, withOwnRegexes(v)])));
|
|
120
|
+
return node;
|
|
121
|
+
}
|
|
122
|
+
export const ENGINE_VOCABULARY = withOwnRegexes(CREDENTIAL_VOCABULARY);
|
|
123
|
+
export function egressPassGroups(vocabulary) {
|
|
124
|
+
const groups = [];
|
|
125
|
+
for (const row of vocabulary.values) {
|
|
126
|
+
const last = groups[groups.length - 1];
|
|
127
|
+
if (row.pass !== undefined && last !== undefined && last[0].pass === row.pass)
|
|
128
|
+
last.push(row);
|
|
129
|
+
else
|
|
130
|
+
groups.push([row]);
|
|
131
|
+
}
|
|
132
|
+
return groups;
|
|
133
|
+
}
|
|
134
|
+
const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
135
|
+
function credentialWordPattern(countWords, rows) {
|
|
136
|
+
const at = (position) => rows.filter((r) => r.position === position).map((r) => r.word);
|
|
137
|
+
const countedPlural = `(?<!(?:^|_)(?:${countWords.join("|")})_)(?:${at("counted-plural").join("|")})S`;
|
|
138
|
+
const [separated, embedded] = [at("separated"), at("embedded")];
|
|
139
|
+
const positions = [
|
|
140
|
+
`(^|_)(${at("leading").join("|")}|(${at("plural").join("|")})S|${countedPlural})`,
|
|
141
|
+
...(separated.length > 0 ? [`_(${separated.join("|")})`] : []),
|
|
142
|
+
...(embedded.length > 0 ? [`(${embedded.join("|")})`] : []),
|
|
143
|
+
];
|
|
144
|
+
const segmentWord = `(?:${[...at("leading"), ...at("plural").map((w) => `${w}S`), ...separated, countedPlural].join("|")})`;
|
|
145
|
+
return { credentialWord: new RegExp(`(${positions.join("|")})(?=$|[_0-9])`, "i"), segmentWord };
|
|
146
|
+
}
|
|
147
|
+
export function envNameView(vocabulary = ENGINE_VOCABULARY) {
|
|
148
|
+
const { credentialWord, segmentWord } = credentialWordPattern(vocabulary.countWords, vocabulary.words);
|
|
149
|
+
return {
|
|
150
|
+
exactNames: new Set(vocabulary.exactEnvNames.flatMap((name) => [name, `INPUT_${name}`])),
|
|
151
|
+
nonSecretExactNames: new Set(vocabulary.nonSecretExactNames),
|
|
152
|
+
nonSecretNameShape: withOwnRegexes(vocabulary.nonSecretNameShape),
|
|
153
|
+
nonCredentialPrefix: withOwnRegexes(vocabulary.nonCredentialPrefix),
|
|
154
|
+
credentialWord,
|
|
155
|
+
connectionStringName: withOwnRegexes(vocabulary.connectionStringName),
|
|
156
|
+
anySegmentUnder: (namespace) => new RegExp(`^${literally(namespace)}(?:.*_)?${segmentWord}(?:_|$)`, "is"),
|
|
157
|
+
tokenPrefix: withOwnRegexes(vocabulary.tokenPrefix),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
export function splitCamelCase(name) {
|
|
161
|
+
return ["OAuth", "NextAuth"]
|
|
162
|
+
.reduce((text, word) => text.replaceAll(word, word[0] + word.slice(1).toLowerCase()), name)
|
|
163
|
+
.replace(new RegExp(`(${HUMP_BEFORE})([A-Z])`, "g"), "$1_$2")
|
|
164
|
+
.replace(/([A-Z])([A-Z][a-z])/g, "$1_$2");
|
|
165
|
+
}
|
|
166
|
+
export function recordKeyView(vocabulary) {
|
|
167
|
+
const rows = vocabulary.words.flatMap((w) => [w.word, ...(w.recordKeys ?? [])].map((word) => ({ word, position: w.position === "separated" ? "leading" : w.position })));
|
|
168
|
+
return {
|
|
169
|
+
credentialWord: credentialWordPattern(vocabulary.countWords, rows).credentialWord,
|
|
170
|
+
nonSecretExactNames: new Set(vocabulary.nonSecretExactNames),
|
|
171
|
+
nonSecretNameShape: withOwnRegexes(vocabulary.nonSecretNameShape),
|
|
172
|
+
nonCredentialPrefix: withOwnRegexes(vocabulary.nonCredentialPrefix),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
const RECORD_KEYS = recordKeyView(ENGINE_VOCABULARY);
|
|
176
|
+
function holdsCredentialWord(text, view) {
|
|
177
|
+
const words = text.replace(view.nonCredentialPrefix, "").replace(/-/g, "_");
|
|
178
|
+
return view.credentialWord.test(words) || view.credentialWord.test(splitCamelCase(words));
|
|
179
|
+
}
|
|
180
|
+
function isNonSecretRecordName(key, source, view) {
|
|
181
|
+
const spelled = key.replace(/-/g, "_");
|
|
182
|
+
const folded = spelled.toUpperCase();
|
|
183
|
+
if (view.nonSecretExactNames.has(folded))
|
|
184
|
+
return true;
|
|
185
|
+
if (source !== "header")
|
|
186
|
+
return false;
|
|
187
|
+
if (folded.length !== spelled.length)
|
|
188
|
+
return false;
|
|
189
|
+
for (const name of view.nonSecretExactNames) {
|
|
190
|
+
if (folded.endsWith(`_${name}`) && !holdsCredentialWord(spelled.slice(0, spelled.length - name.length - 1), view))
|
|
191
|
+
return true;
|
|
192
|
+
}
|
|
193
|
+
return false;
|
|
194
|
+
}
|
|
195
|
+
function readsAsCredential(key, source, view) {
|
|
196
|
+
if (view.nonSecretNameShape.test(key))
|
|
197
|
+
return false;
|
|
198
|
+
return holdsCredentialWord(key, view) && !isNonSecretRecordName(key, source, view);
|
|
199
|
+
}
|
|
200
|
+
export function isSensitiveRecordKey(key, view = RECORD_KEYS) {
|
|
201
|
+
return readsAsCredential(key, "member", view);
|
|
202
|
+
}
|
|
203
|
+
export function isSensitiveHeaderName(name, view = RECORD_KEYS) {
|
|
204
|
+
return readsAsCredential(name, "header", view);
|
|
205
|
+
}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sema-agent/primitives",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.63.0",
|
|
4
4
|
"private": false,
|
|
5
|
-
"description": "Shared low-level definitions of the sema engine, importable by any layer: the credential reference (closed set of kinds, one reader). Zero dependencies.",
|
|
5
|
+
"description": "Shared low-level definitions of the sema engine, importable by any layer: the credential reference (closed set of kinds, one reader) and the secret-shape detection definition (one table, its matcher and its renderer, no layer's policy). Zero dependencies.",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"license": "BUSL-1.1",
|
|
8
8
|
"repository": {
|
|
@@ -31,5 +31,5 @@
|
|
|
31
31
|
"scripts": {
|
|
32
32
|
"build": "tsc -p tsconfig.json && tsc -p tsconfig.json --emitDeclarationOnly --removeComments false"
|
|
33
33
|
},
|
|
34
|
-
"gitHead": "
|
|
34
|
+
"gitHead": "bbc88f8338cca4452ef29328a57de79c18c47502"
|
|
35
35
|
}
|