@sema-agent/primitives 7.64.0 → 7.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -91,15 +91,19 @@ look a kind up in it only after `readCredentialRef` has accepted the value.
91
91
  ## Secret-shape detection
92
92
 
93
93
  `CREDENTIAL_VOCABULARY` is the one table of what a credential looks like: value families (each row with a wide form, and
94
- for some a strict form), the credential words a name is read by, and the names that look like credentials but are none.
94
+ for some a strict form, a fragment form — what is left of the credential when a text was cut inside its body — and the
95
+ form and token head an environment value is read by), the credential words a name is read by, the names that look like
96
+ credentials but are none, and `names`: the names that mark a credential in an assignment (`places: ["assignment"]`) or as a
97
+ URL query or fragment parameter (`"url-param"`).
95
98
  Every layer that looks for a secret reads this table, so a shape added to it is found everywhere at once. The table holds
96
99
  no layer's policy: which form is read, how a text is read first, how a URL's username is treated and which mark replaces
97
100
  a hit are the caller's, passed in whole on every call. Nothing here has a default.
98
101
 
99
102
  ```ts
100
103
  interface SecretScanPolicy {
101
- grade: "strict" | "wide"; // the strict forms (narrow: a hit may refuse a write) or the wide ones (broad: a hit is withheld)
102
- readings: SecretReading[]; // at least one of { kind: "raw" } (as written) / { kind: "format-free" } (format characters removed)
104
+ grade: "strict" | "wide" | "fragment"; // the strict forms (narrow: a hit may refuse a write), the wide ones (broad: a hit is withheld) or the fragment ones (a cut residue)
105
+ readings: SecretReading[]; // at least one of { kind: "raw" } (as written) / { kind: "format-free" } (format characters removed),
106
+ // and any of { kind: "percent-layers", depth: 1..8 } / { kind: "percent-boundary" } / { kind: "form-plus" }
103
107
  urlUserinfo: "keep-username" | "whole"; // a URL userinfo: keep the user and withhold the password, or withhold all of it
104
108
  onUnresolvedReading: "secret" | "ignore"; // what a reading that cannot read a text to its end answers
105
109
  }
@@ -113,11 +117,13 @@ interface SecretNotation {
113
117
 
114
118
  | Function | Answers |
115
119
  | --- | --- |
116
- | `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved`, `mapping-failed` |
120
+ | `scanSecrets(text, policy)` | `{ outcome: "spans", spans }` — every hit as `{ start, end, family, kind }` in `text`'s own coordinates, sorted (empty = none found) — or `{ outcome: "whole", reason }` with `reason` one of `scan-budget` (more than 64 places where format characters stood), `reading-unresolved` (the percent layers ran out, under `onUnresolvedReading: "secret"`), `mapping-failed`; a hit in a derived text is reported over the written units it was read from |
117
121
  | `isSecretShaped(text, policy)` | `scanSecrets` found a span or answered the whole text |
118
122
  | `renderSecrets(text, policy, notation, { passes?, report? })` | `text` with every hit replaced by its mark; `passes` are the caller's own rules, run after the table's in the same pipeline; `report.findings` receives one finding per hit (`report.preexistingMarkers` is the caller's to set) |
119
123
  | `isSensitiveRecordKey(key)` / `isSensitiveHeaderName(name)` | a record member / an HTTP header name names a credential |
120
- | `envNameView()` | the readings of environment variable names (exact names, credential words, the token-prefix value predicate, a namespace's names `anySegmentUnder(namespace)`) |
124
+ | `envNameView()` | the readings of environment variable names (exact names, credential words, a namespace's names `anySegmentUnder(namespace)`) |
125
+ | `envValueView()` | the readings of environment values, off the rows: `forms` (each row's env value form, in table order), `tokenHead` (each row's token head, one anchored alternation), `flagWords` |
126
+ | `secretReadingViews(text, readings)` | the texts the readings derive — the written text first, each once — each with `toOriginalSpan(start, end)` (the whole written units a span was read from), and `unresolved` (the percent layers ran out while one more would still decode) |
121
127
 
122
128
  A policy or a notation that is not whole (a missing or unknown member, an empty mark, a mark holding `$` before a digit,
123
129
  a `byKind` / `byFamily` key no row has, two rows of one pass spelled differently) throws a `TypeError`. Both are read on
@@ -151,5 +157,5 @@ removed or re-kinded export is named as breaking in the `@sema-agent/core` relea
151
157
  | --- | --- |
152
158
  | Credential reference | `readCredentialRef` `CREDENTIAL_REF_KINDS` `CREDENTIAL_REF_TRUST` `CREDENTIAL_REF_TEXT_MAX` `CREDENTIAL_REF_MAP_MAX` `CREDENTIAL_REF_PROBLEM_REASONS` |
153
159
  | Types | `CredentialRef` `CredentialRefKind` `CredentialRefRequestAdmission` `CredentialRefProblem` `CredentialRefProblemReason` `CredentialRefReading` |
154
- | Secret-shape detection | `CREDENTIAL_VOCABULARY` `CREDENTIAL_MIN_CHARS` `URL_OF_ANY_SCHEME` `SECRET_READING_KINDS` `scanSecrets` `isSecretShaped` `renderSecrets` `isSensitiveRecordKey` `isSensitiveHeaderName` `envNameView` `splitCamelCase` |
155
- | Types | `CredentialVocabulary` `ValueFamilyRow` `CredentialWordRow` `CredentialWordPosition` `EnvNameView` `RecordKeyView` `RedactionConfidence` `RedactionSource` `SecretGrade` `SecretReading` `SecretScanPolicy` `SecretNotation` `SecretScan` `SecretSpan` `RenderOptions` `RedactionPass` `RedactionFinding` `RedactionReport` `SummarizableFinding` |
160
+ | Secret-shape detection | `CREDENTIAL_VOCABULARY` `CREDENTIAL_MIN_CHARS` `URL_OF_ANY_SCHEME` `SECRET_READING_KINDS` `scanSecrets` `isSecretShaped` `renderSecrets` `isSensitiveRecordKey` `isSensitiveHeaderName` `envNameView` `envValueView` `secretReadingViews` `splitCamelCase` |
161
+ | Types | `CredentialVocabulary` `ValueFamilyRow` `CredentialWordRow` `CredentialWordPosition` `CredentialNameRow` `CredentialNamePlace` `EnvNameView` `EnvValueView` `RecordKeyView` `RedactionConfidence` `RedactionSource` `SecretGrade` `SecretReading` `SecretReadingKind` `SecretReadingView` `SecretReadingViews` `SecretScanPolicy` `SecretNotation` `SecretScan` `SecretSpan` `RenderOptions` `RedactionPass` `RedactionFinding` `RedactionReport` `SummarizableFinding` |
package/dist/index.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  /** The package entry of `@sema-agent/primitives`; core imports the package by name, never these modules. */
2
2
  export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, type CredentialRef, type CredentialRefKind, type CredentialRefRequestAdmission, type CredentialRefProblem, type CredentialRefProblemReason, type CredentialRefReading, } from "./credential-ref.js";
3
- export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type EnvNameView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
4
- export { SECRET_READING_KINDS, scanSecrets, isSecretShaped, renderSecrets, type SecretGrade, type SecretReading, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
3
+ export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, type CredentialVocabulary, type ValueFamilyRow, type CredentialWordRow, type CredentialWordPosition, type CredentialNameRow, type CredentialNamePlace, type EnvNameView, type EnvValueView, type RecordKeyView, type RedactionConfidence, type RedactionSource, } from "./secret-vocabulary.js";
4
+ export { SECRET_READING_KINDS, secretReadingViews, type SecretReading, type SecretReadingKind, type SecretReadingView, type SecretReadingViews, } from "./secret-readings.js";
5
+ export { scanSecrets, isSecretShaped, renderSecrets, type SecretGrade, type SecretScanPolicy, type SecretNotation, type SecretScan, type SecretSpan, type RenderOptions, type RedactionPass, type RedactionFinding, type RedactionReport, type SummarizableFinding, } from "./secret-scan.js";
package/dist/index.js CHANGED
@@ -1,3 +1,4 @@
1
1
  export { CREDENTIAL_REF_KINDS, CREDENTIAL_REF_TRUST, CREDENTIAL_REF_TEXT_MAX, CREDENTIAL_REF_MAP_MAX, CREDENTIAL_REF_PROBLEM_REASONS, readCredentialRef, } from "./credential-ref.js";
2
- export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
3
- export { SECRET_READING_KINDS, scanSecrets, isSecretShaped, renderSecrets, } from "./secret-scan.js";
2
+ export { CREDENTIAL_VOCABULARY, CREDENTIAL_MIN_CHARS, URL_OF_ANY_SCHEME, envNameView, envValueView, splitCamelCase, isSensitiveRecordKey, isSensitiveHeaderName, } from "./secret-vocabulary.js";
3
+ export { SECRET_READING_KINDS, secretReadingViews, } from "./secret-readings.js";
4
+ export { scanSecrets, isSecretShaped, renderSecrets, } from "./secret-scan.js";
@@ -0,0 +1,80 @@
1
+ /**
2
+ * The readings a text is read through before the vocabulary's forms run over it — a closed set, each reading named and
3
+ * each a pure function of the text:
4
+ * - CHARACTER readings decide how one text is scanned: `raw` (as written) and `format-free` (every format character
5
+ * `\p{Cf}` removed, its hits mapped back — the engine's own reading, secret-scan.ts);
6
+ * - TEXT readings derive more texts from the one written, each scanned the same way and each hit mapped back to the
7
+ * written text's coordinates: `percent-layers` (up to `depth` tolerant percent-decoded layers), `percent-boundary` (each
8
+ * run of `%HH` escapes read as UTF-8 in one go, so an encoded no-break or zero-width space is the delimiter it stands
9
+ * for) and `form-plus` (`+` read as a space, the form-urlencoded way).
10
+ * A view's characters each map back to the span of the written text they were read from: a decoded escape to its three
11
+ * characters, a code point decoded from a run to all of its escapes. A hit in a view maps back to the whole units it
12
+ * touches, so a mapped-back span never cuts an escape (fail-closed: more is withheld, never less).
13
+ */
14
+ /** The readings, a closed set ({@link SecretReading}). */
15
+ export declare const SECRET_READING_KINDS: readonly ["raw", "format-free", "percent-layers", "percent-boundary", "form-plus"];
16
+ /** One reading's name. */
17
+ export type SecretReadingKind = (typeof SECRET_READING_KINDS)[number];
18
+ /** One reading, by its kind. `percent-layers` takes `depth`, the most decoded layers it reads (an integer 1 … 8); every
19
+ * other reading takes no parameter. */
20
+ export type SecretReading = {
21
+ readonly kind: "raw";
22
+ } | {
23
+ readonly kind: "format-free";
24
+ } | {
25
+ readonly kind: "percent-layers";
26
+ readonly depth: number;
27
+ } | {
28
+ readonly kind: "percent-boundary";
29
+ } | {
30
+ readonly kind: "form-plus";
31
+ };
32
+ /** The most percent-decoded layers a `percent-layers` reading may read. */
33
+ export declare const PERCENT_LAYERS_MAX_DEPTH = 8;
34
+ /** A readings list read once and checked whole: which character readings scan a text, and which text readings derive
35
+ * the texts scanned. `key` names what it reads as. */
36
+ export interface ReadingSet {
37
+ readonly raw: boolean;
38
+ readonly formatFree: boolean;
39
+ /** The `percent-layers` depth, when the reading is read. */
40
+ readonly percentLayers?: number | undefined;
41
+ readonly percentBoundary?: boolean | undefined;
42
+ readonly formPlus?: boolean | undefined;
43
+ }
44
+ /** `readings` checked whole and read as a set: a non-empty array of {@link SecretReading}, holding `raw` or `format-free`
45
+ * (a text reading derives texts; a character reading scans them), with one `percent-layers` at most (one depth). A
46
+ * parameterless reading listed twice is the same reading. Throws a `TypeError` naming the first fault. */
47
+ export declare function readReadings(readings: unknown, at?: string): ReadingSet;
48
+ /** Does the set read a text reading (one that derives texts beyond the written one)? */
49
+ export declare const readsTextReadings: (set: ReadingSet) => boolean;
50
+ /** A text derived by the text readings, and the way back. */
51
+ export interface SecretReadingView {
52
+ /** The derived text. */
53
+ readonly text: string;
54
+ /** A span of `text` [start, end) as the span of the written text it was read from: the whole units (a decoded escape, a
55
+ * code point decoded from a run) the span touches. An empty span maps to the written position it stands at. */
56
+ readonly toOriginalSpan: (start: number, end: number) => {
57
+ readonly start: number;
58
+ readonly end: number;
59
+ };
60
+ }
61
+ /** What {@link secretReadingViews} returns: the views, the written text first, each text once (first seen first), and
62
+ * whether the percent layers ran out while one more layer would still decode something. */
63
+ export interface SecretReadingViews {
64
+ readonly views: readonly SecretReadingView[];
65
+ readonly unresolved: boolean;
66
+ }
67
+ /** `text` through one {@link percentDecodeLayer} (the same tolerant layer the `percent-layers` reading reads), without
68
+ * the way back. */
69
+ export declare function percentDecodedOnce(text: string): string;
70
+ /** The views a checked set of readings derives from `text` (the written text first, each text once) — the engine's entry;
71
+ * {@link secretReadingViews} is the checked one. */
72
+ export declare function readingViews(text: string, set: ReadingSet): SecretReadingViews;
73
+ /**
74
+ * The texts `readings` derive from `text` and the way back from each: the written text first, then its percent-decoded
75
+ * layers (`percent-layers`, up to `depth`, stopping at the first layer that decodes nothing), each of those read the
76
+ * boundary way (`percent-boundary`), and each of those with `+` read as a space (`form-plus`) — each text once, first seen
77
+ * first. `unresolved` = the layers ran out while one more would still decode something. The character readings (`raw`,
78
+ * `format-free`) derive no text: they say how each text is scanned. Throws a `TypeError` on readings that are not whole.
79
+ */
80
+ export declare function secretReadingViews(text: string, readings: readonly SecretReading[]): SecretReadingViews;
@@ -0,0 +1,184 @@
1
+ export const SECRET_READING_KINDS = Object.freeze(["raw", "format-free", "percent-layers", "percent-boundary", "form-plus"]);
2
+ export const PERCENT_LAYERS_MAX_DEPTH = 8;
3
+ function fail(what) {
4
+ throw new TypeError(`secret scan: ${what}`);
5
+ }
6
+ const isRecord = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
7
+ export function readReadings(readings, at = "policy.readings") {
8
+ if (!Array.isArray(readings) || readings.length === 0)
9
+ fail(`${at} must be a non-empty array`);
10
+ const kinds = new Set();
11
+ let depth;
12
+ const list = readings;
13
+ for (const entry of list) {
14
+ const kind = isRecord(entry) ? SECRET_READING_KINDS.find((k) => k === entry.kind) : undefined;
15
+ if (kind === undefined || !isRecord(entry))
16
+ fail(`every ${at} entry is { kind } with kind one of ${SECRET_READING_KINDS.join(" / ")}`);
17
+ const members = kind === "percent-layers" ? ["kind", "depth"] : ["kind"];
18
+ if (Object.keys(entry).some((k) => !members.includes(k)))
19
+ fail(kind === "percent-layers" ? `the reading "percent-layers" takes \`depth\` only` : `the reading "${kind}" takes no parameter`);
20
+ if (kind === "percent-layers") {
21
+ if (depth !== undefined)
22
+ fail(`${at} lists "percent-layers" twice (one depth is read)`);
23
+ const d = entry.depth;
24
+ if (typeof d !== "number" || !Number.isInteger(d) || d < 1 || d > PERCENT_LAYERS_MAX_DEPTH)
25
+ fail(`the reading "percent-layers" takes \`depth\`, an integer 1 … ${PERCENT_LAYERS_MAX_DEPTH}`);
26
+ depth = d;
27
+ }
28
+ kinds.add(kind);
29
+ }
30
+ if (!kinds.has("raw") && !kinds.has("format-free"))
31
+ fail(`${at} must hold "raw" or "format-free" (a text reading derives texts; one of those two scans them)`);
32
+ return { raw: kinds.has("raw"), formatFree: kinds.has("format-free"), percentLayers: depth, percentBoundary: kinds.has("percent-boundary"), formPlus: kinds.has("form-plus") };
33
+ }
34
+ export const readsTextReadings = (set) => set.percentLayers !== undefined || set.percentBoundary === true || set.formPlus === true;
35
+ const fromOf = (t, i) => (t.from === undefined ? i : t.from[i]);
36
+ const toOf = (t, i) => (t.to === undefined ? i + 1 : t.to[i]);
37
+ function derive(src, build) {
38
+ let text = "";
39
+ const from = [];
40
+ const to = [];
41
+ build((chars, a, b) => {
42
+ const start = fromOf(src, a);
43
+ const end = toOf(src, b - 1);
44
+ for (let k = 0; k < chars.length; k++) {
45
+ from.push(start);
46
+ to.push(end);
47
+ }
48
+ text += chars;
49
+ });
50
+ return { text, from: Int32Array.from(from), to: Int32Array.from(to) };
51
+ }
52
+ function percentDecodeLayer(src) {
53
+ if (src.text.indexOf("%") === -1)
54
+ return src;
55
+ const re = /%([0-7][0-9A-Fa-f])/g;
56
+ if (!re.test(src.text))
57
+ return src;
58
+ re.lastIndex = 0;
59
+ return derive(src, (push) => {
60
+ let cursor = 0;
61
+ for (let m = re.exec(src.text); m !== null; m = re.exec(src.text)) {
62
+ for (let i = cursor; i < m.index; i++)
63
+ push(src.text[i], i, i + 1);
64
+ push(String.fromCharCode(parseInt(m[1], 16)), m.index, m.index + 3);
65
+ cursor = m.index + 3;
66
+ }
67
+ for (let i = cursor; i < src.text.length; i++)
68
+ push(src.text[i], i, i + 1);
69
+ });
70
+ }
71
+ export function percentDecodedOnce(text) {
72
+ return percentDecodeLayer({ text }).text;
73
+ }
74
+ function utf8CodePointLengths(bytes) {
75
+ const lengths = [];
76
+ for (let i = 0; i < bytes.length;) {
77
+ const lead = bytes[i];
78
+ if (lead < 0x80) {
79
+ lengths.push(1);
80
+ i += 1;
81
+ continue;
82
+ }
83
+ const shape = lead >= 0xc2 && lead <= 0xdf ? [1, 0x80, 0xbf]
84
+ : lead === 0xe0 ? [2, 0xa0, 0xbf]
85
+ : (lead >= 0xe1 && lead <= 0xec) || lead === 0xee || lead === 0xef ? [2, 0x80, 0xbf]
86
+ : lead === 0xed ? [2, 0x80, 0x9f]
87
+ : lead === 0xf0 ? [3, 0x90, 0xbf]
88
+ : lead >= 0xf1 && lead <= 0xf3 ? [3, 0x80, 0xbf]
89
+ : lead === 0xf4 ? [3, 0x80, 0x8f]
90
+ : undefined;
91
+ if (shape === undefined)
92
+ return undefined;
93
+ const [need, lo, hi] = shape;
94
+ for (let k = 1; k <= need; k++) {
95
+ const c = bytes[i + k];
96
+ if (c === undefined || c < (k === 1 ? lo : 0x80) || c > (k === 1 ? hi : 0xbf))
97
+ return undefined;
98
+ }
99
+ lengths.push(need + 1);
100
+ i += need + 1;
101
+ }
102
+ return lengths;
103
+ }
104
+ function boundaryReading(src) {
105
+ if (src.text.indexOf("%") === -1)
106
+ return src;
107
+ const re = /(?:%[0-9A-Fa-f]{2})+/g;
108
+ if (!re.test(src.text))
109
+ return src;
110
+ re.lastIndex = 0;
111
+ return derive(src, (push) => {
112
+ let cursor = 0;
113
+ for (let m = re.exec(src.text); m !== null; m = re.exec(src.text)) {
114
+ for (let i = cursor; i < m.index; i++)
115
+ push(src.text[i], i, i + 1);
116
+ const run = m[0];
117
+ const bytes = [];
118
+ for (let i = 0; i < run.length; i += 3)
119
+ bytes.push(parseInt(run.slice(i + 1, i + 3), 16));
120
+ const lengths = utf8CodePointLengths(bytes);
121
+ if (lengths === undefined)
122
+ bytes.forEach((b, k) => push(b < 0x80 ? String.fromCharCode(b) : " ", m.index + 3 * k, m.index + 3 * k + 3));
123
+ else {
124
+ let k = 0;
125
+ for (const n of lengths) {
126
+ const a = m.index + 3 * k;
127
+ push(decodeURIComponent(run.slice(3 * k, 3 * (k + n))), a, a + 3 * n);
128
+ k += n;
129
+ }
130
+ }
131
+ cursor = m.index + run.length;
132
+ }
133
+ for (let i = cursor; i < src.text.length; i++)
134
+ push(src.text[i], i, i + 1);
135
+ });
136
+ }
137
+ const formPlus = (src) => (src.text.indexOf("+") === -1 ? src : { text: src.text.replace(/\+/g, " "), from: src.from, to: src.to });
138
+ function viewOf(t, writtenLength) {
139
+ return {
140
+ text: t.text,
141
+ toOriginalSpan: (start, end) => {
142
+ if (end <= start) {
143
+ const at = start >= t.text.length ? writtenLength : fromOf(t, start);
144
+ return { start: at, end: at };
145
+ }
146
+ let lo = Infinity;
147
+ let hi = -Infinity;
148
+ for (let i = start; i < end; i++) {
149
+ lo = Math.min(lo, fromOf(t, i));
150
+ hi = Math.max(hi, toOf(t, i));
151
+ }
152
+ return { start: lo, end: hi };
153
+ },
154
+ };
155
+ }
156
+ export function readingViews(text, set) {
157
+ let texts = [{ text }];
158
+ let unresolved = false;
159
+ if (set.percentLayers !== undefined) {
160
+ let cur = texts[0];
161
+ for (let i = 0; i < set.percentLayers; i++) {
162
+ const next = percentDecodeLayer(cur);
163
+ if (next === cur)
164
+ break;
165
+ texts.push(next);
166
+ cur = next;
167
+ }
168
+ unresolved = texts.length === set.percentLayers + 1 && percentDecodeLayer(cur) !== cur;
169
+ }
170
+ const once = (list) => {
171
+ const seen = new Set();
172
+ return list.filter((t) => (seen.has(t.text) ? false : (seen.add(t.text), true)));
173
+ };
174
+ if (set.percentBoundary === true)
175
+ texts = once([...texts, ...texts.map(boundaryReading)]);
176
+ if (set.formPlus === true)
177
+ texts = once([...texts, ...texts.map(formPlus)]);
178
+ return { views: once(texts).map((t) => viewOf(t, text.length)), unresolved };
179
+ }
180
+ export function secretReadingViews(text, readings) {
181
+ if (typeof text !== "string")
182
+ fail("the text is not a string");
183
+ return readingViews(text, readReadings(readings, "readings"));
184
+ }
@@ -17,6 +17,7 @@
17
17
  * edits (best-effort where redactions overlap — a span landing inside an earlier replacement anchors to that edit's start).
18
18
  */
19
19
  import { type RedactionConfidence, type RedactionSource, type ValueFamilyRow } from "./secret-vocabulary.js";
20
+ import { type ReadingSet, type SecretReading } from "./secret-readings.js";
20
21
  /** One redaction hit: what was detected, where (ORIGINAL-input coordinates), how confidently, the marker
21
22
  * literal written in its place, which rule table minted it, and — present when a vocabulary row minted it — the
22
23
  * value family (the row's stable id; a host-leak hit such as a URL or a path names none). */
@@ -59,53 +60,40 @@ export interface RedactionPass {
59
60
  * matches at the hit's offset in the pass input. Absent = the pass is no vocabulary row. */
60
61
  family?: string | ((input: string, at: number) => string);
61
62
  }
62
- /** Which readings a run reads its text through (the policy's `readings`, as a set). */
63
- interface ReadingSet {
64
- readonly raw: boolean;
65
- readonly formatFree: boolean;
66
- }
67
- /** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings,
68
- * and the optional report. */
63
+ /** One run of the engine: the marker a text no reading can afford is replaced with whole (`overStrip`), the readings
64
+ * (the policy's, as a set), what a text the percent layers cannot read to its end answers (the policy's
65
+ * `onUnresolvedReading`; read only when the readings hold `percent-layers`), and the optional report. */
69
66
  export interface EngineRun {
70
67
  readonly overStrip: string;
71
68
  readonly readings: ReadingSet;
69
+ readonly onUnresolvedReading?: SecretScanPolicy["onUnresolvedReading"] | undefined;
72
70
  readonly report?: RedactionReport | undefined;
73
71
  }
74
72
  /**
75
73
  * Run the declared passes in order, collecting findings (original-input spans) when a report is given.
76
74
  *
77
- * FORMAT CHARACTERS (the `format-free` reading): the passes scan a VIEW of the input with every `\p{Cf}` removed, so a
78
- * credential split by a zero-width character is one token to the scanner, not two fragments of which only the first (or
79
- * neither) matches. The OUTPUT keeps every format character that sits OUTSIDE a replaced span — a joiner in an emoji
80
- * sequence, a non-joiner in Persian text, a soft hyphen in a word — and drops only those INSIDE the span the marker
81
- * replaces (they were part of the secret). A format-free input, or a run without the `format-free` reading, takes the
82
- * plain sequential path. The view's replacements and every boundary scan's are taken as disjoint spans of their input (a
83
- * later pass that matched across an earlier pass's inserted text is rendered from the removals instead), so the spans
84
- * always map back exactly and no scan is skipped.
75
+ * TEXT READINGS (`percent-layers` / `percent-boundary` / `form-plus`): the written text is rendered as below, and every
76
+ * text the readings derive from it ({@link readingViews}) is read by the same passes through the same character readings;
77
+ * each place a derived text withholds maps back to the written text (the whole escapes it touches) and joins the written
78
+ * text's own removals. A derived text that withholds nothing the written one did not leaves the output — findings and
79
+ * all — the written text's. A text the percent layers cannot read to its end is replaced whole when the run says
80
+ * `onUnresolvedReading: "secret"`.
85
81
  */
86
82
  export declare function runRedactionPasses(input: string, passes: readonly RedactionPass[], run: EngineRun): string;
87
- /** Which form of a row a reader reads: `strict` (refuse-write — narrow, a hit fail-closed refuses) or `wide` (egress —
88
- * broad, a hit is withheld). */
89
- export type SecretGrade = "strict" | "wide";
90
- /** The readings a text can be read through, a closed set: `raw` — the text as written; `format-free` — the text with every
91
- * format character (`\p{Cf}`) removed, its hits mapped back, plus a scan from every place a format character stood (so a
92
- * format character neither splits a credential nor hides its start). */
93
- export declare const SECRET_READING_KINDS: readonly ["raw", "format-free"];
94
- /** One reading, by its kind. A reading that takes a parameter carries it as a member beside `kind`. */
95
- export type SecretReading = {
96
- readonly kind: "raw";
97
- } | {
98
- readonly kind: "format-free";
99
- };
83
+ /** Which form of a row a reader reads: `strict` (refuse-write — narrow, a hit fail-closed refuses), `wide` (egress —
84
+ * broad, a hit is withheld) or `fragment` (what is left of a credential a cut shortened — a vendor prefix and a residue
85
+ * that still holds two digits or capitals; the rows that carry one). */
86
+ export type SecretGrade = "strict" | "wide" | "fragment";
100
87
  /** What a caller reads the vocabulary for — every member required, none has a default here:
101
88
  * - `grade` — which form of each row is read ({@link SecretGrade});
102
- * - `readings` — the readings a text is read through, at least one ({@link SECRET_READING_KINDS}); the hits of every
103
- * reading are joined;
89
+ * - `readings` — the readings a text is read through (secret-readings.ts): `raw` or `format-free` at least, which scan a
90
+ * text, and any of the text readings, which derive more texts to scan; the hits of every reading are joined;
104
91
  * - `urlUserinfo` — a URL userinfo's rendering: `keep-username` keeps the user and withholds the password (a bare user is
105
92
  * kept below 20 characters and withheld at or past it; a userinfo holding a second `@` is withheld whole), `whole`
106
93
  * withholds the whole userinfo;
107
94
  * - `onUnresolvedReading` — what a reading that cannot read a text to its end answers: `secret` (the whole text) or
108
- * `ignore` (what it did read). `raw` and `format-free` always read to the end. */
95
+ * `ignore` (what it did read). Only `percent-layers` can run out (its depth reached while one more layer would still
96
+ * decode something); every other reading reads to the end. */
109
97
  export interface SecretScanPolicy {
110
98
  readonly grade: SecretGrade;
111
99
  readonly readings: readonly SecretReading[];
@@ -165,12 +153,13 @@ export declare function egressPasses(groups: ReadonlyArray<readonly ValueFamilyR
165
153
  export declare function renderSecrets(text: string, policy: SecretScanPolicy, notation: SecretNotation, options?: RenderOptions): string;
166
154
  /**
167
155
  * Where the vocabulary sees a credential in `text` under `policy`: every hit of the grade's form of every row, on every
168
- * reading of the policy, as the hit's withheld part in `text`'s own coordinates (deduplicated, sorted by start) — or the
169
- * whole text, when a reading cannot be afforded (more than 64 places where format characters stood: `scan-budget`). The
156
+ * reading of the policy — each text the text readings derive, scanned by the character readings — as the hit's withheld
157
+ * part in `text`'s own coordinates (the whole written units a derived hit was read from; deduplicated, sorted by start) —
158
+ * or the whole text, when a reading cannot be afforded (more than 64 places where format characters stood: `scan-budget`)
159
+ * or the percent layers ran out before the text did under `onUnresolvedReading: "secret"` (`reading-unresolved`). The
170
160
  * matcher answers where a credential is; the renderer additionally leaves a hit that already reads as its own rendering
171
161
  * (a value that is already a mark), which only a notation can tell. Throws a `TypeError` on a policy that is not whole.
172
162
  */
173
163
  export declare function scanSecrets(text: string, policy: SecretScanPolicy): SecretScan;
174
164
  /** Does the vocabulary see a credential in `text` under `policy`? {@link scanSecrets} answers the whole text or a span. */
175
165
  export declare function isSecretShaped(text: string, policy: SecretScanPolicy): boolean;
176
- export {};
@@ -1,4 +1,5 @@
1
1
  import { ENGINE_VOCABULARY, egressPassGroups } from "./secret-vocabulary.js";
2
+ import { percentDecodedOnce, readReadings, readingViews, readsTextReadings } from "./secret-readings.js";
2
3
  function passZeroMapper(batches) {
3
4
  const cursors = batches.map(() => ({ i: 0, delta: 0 }));
4
5
  return (pos) => {
@@ -70,6 +71,10 @@ function removedRegion(pass, match, groups, inserted) {
70
71
  tail++;
71
72
  return [head, match.length - tail];
72
73
  }
74
+ function appendAll(into, items) {
75
+ for (const item of items)
76
+ into.push(item);
77
+ }
73
78
  const SOLO_SCAN_WORK_PER_CHAR = 8;
74
79
  const SOLO_SCAN_WORK_FLOOR = 1 << 17;
75
80
  function soloHits(input, passes, overStrip) {
@@ -135,7 +140,8 @@ function sequentialRemovals(batches) {
135
140
  for (const e of batches[j]) {
136
141
  const start = toPassZero(e.at + e.head);
137
142
  const end = toPassZero(e.at + e.removedLen - e.tail);
138
- removals.push({ start, end: Math.max(start, end), marker: e.inserted.slice(e.head, e.insertedLen - e.tail), pass: e.pass, family: e.family });
143
+ const marker = e.inserted.slice(e.head, e.insertedLen - e.tail);
144
+ removals.push({ start, end: Math.max(start, end), marker, pass: e.pass, family: e.family, matchStart: toPassZero(e.at), matchEnd: toPassZero(e.at + e.removedLen) });
139
145
  }
140
146
  }
141
147
  return removals;
@@ -149,7 +155,7 @@ function renderRemovals(input, removals, overStrip) {
149
155
  const start = sorted[i].start;
150
156
  let end = sorted[i].end;
151
157
  let widest;
152
- for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].marker === "" && sorted[i].start === end)); i++) {
158
+ for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].start === end && (sorted[i].marker === "" || sorted[i].marker === widest?.marker))); i++) {
153
159
  const r = sorted[i];
154
160
  if (r.end > end)
155
161
  end = r.end;
@@ -198,7 +204,7 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
198
204
  const whole = outer.length === 0 && mappable ? undefined : renderRemovals(input, [...sequential, ...outer], overStrip);
199
205
  if (whole === undefined || (whole.out === plain.out && mappable)) {
200
206
  if (report !== undefined)
201
- report.findings.push(...found.map((x) => x.f));
207
+ appendAll(report.findings, found.map((x) => x.f));
202
208
  return plain;
203
209
  }
204
210
  if (report !== undefined) {
@@ -212,7 +218,7 @@ function applyPasses(input, passes, overStrip, report, toOriginal, disjoint = fa
212
218
  found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family) });
213
219
  }
214
220
  found.sort((x, y) => x.pass - y.pass || x.f.span[0] - y.f.span[0]);
215
- report.findings.push(...found.map((x) => x.f));
221
+ appendAll(report.findings, found.map((x) => x.f));
216
222
  }
217
223
  return { out: whole.out, batches: [whole.edits] };
218
224
  }
@@ -283,75 +289,131 @@ function readFormatFree(input) {
283
289
  }
284
290
  export function runRedactionPasses(input, passes, run) {
285
291
  const { overStrip, readings, report } = run;
286
- if (!readings.formatFree || !FORMAT_CHAR_RE.test(input))
287
- return applyPasses(input, passes, overStrip, report, (pos) => pos).out;
288
- const reading = readFormatFree(input);
289
- const { view } = reading;
290
- const { batches } = applyPasses(view, passes, overStrip, report, reading.toOriginal, true);
291
- const viewSpans = replacedSpans(batches);
292
- if (viewSpans === undefined)
292
+ if (!readsTextReadings(readings))
293
+ return renderOneText(input, passes, overStrip, readings, report).out;
294
+ const { views, unresolved } = readingViews(input, readings);
295
+ if (unresolved && run.onUnresolvedReading === "secret")
293
296
  return overStrip;
294
- const spans = viewSpans.map((sp) => ({ ...reading.toOriginalSpan(sp.start, sp.end), inserted: sp.inserted }));
295
- if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
297
+ const base = renderOneText(input, passes, overStrip, readings, report);
298
+ if (views.length === 1)
299
+ return base.out;
300
+ const removals = base.removals();
301
+ if (removals === undefined)
296
302
  return overStrip;
297
303
  const extra = [];
298
- if (readings.raw) {
299
- const rawSpans = replacedSpans(applyPasses(input, passes, overStrip, undefined, (pos) => pos, true).batches);
300
- if (rawSpans !== undefined)
301
- extra.push(...rawSpans);
304
+ for (const view of views.slice(1)) {
305
+ const found = textRegions(view, passes, overStrip, readings);
306
+ if (found === undefined)
307
+ return overStrip;
308
+ appendAll(extra, found);
302
309
  }
303
- for (const cut of reading.cuts) {
304
- const cutSpans = replacedSpans(applyPasses(view.slice(cut), passes, overStrip, undefined, (pos) => pos, true).batches);
305
- if (cutSpans === undefined)
306
- continue;
307
- for (const c of cutSpans) {
308
- const o = reading.toOriginalSpan(c.start + cut, c.end + cut);
309
- extra.push({ ...c, start: o.start, end: o.end });
310
+ const seen = new Set();
311
+ const added = coveredNot(extra, removals).filter((r) => {
312
+ const key = `${r.pass}:${r.start}:${r.end}`;
313
+ return seen.has(key) ? false : (seen.add(key), true);
314
+ });
315
+ if (added.length === 0)
316
+ return base.out;
317
+ if (report !== undefined)
318
+ for (const r of added)
319
+ report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
320
+ return renderRemovals(input, [...removals, ...added], overStrip).out;
321
+ }
322
+ function textRegions(view, passes, overStrip, readings) {
323
+ const regions = [];
324
+ const read = (text, back) => {
325
+ const run = applyPasses(text, passes, overStrip, undefined, (pos) => pos);
326
+ for (const r of [...sequentialRemovals(run.batches), ...soloHits(text, passes, overStrip)]) {
327
+ if (r.end <= r.start)
328
+ continue;
329
+ const inView = back(r.start, r.end);
330
+ const written = view.toOriginalSpan(inView.start, inView.end);
331
+ regions.push({ start: written.start, end: written.end, marker: r.marker, pass: r.pass, family: r.family });
310
332
  }
333
+ };
334
+ const asWritten = (start, end) => ({ start, end });
335
+ const text = view.text;
336
+ if (readings.raw || !FORMAT_CHAR_RE.test(text))
337
+ read(text, asWritten);
338
+ if (readings.formatFree && FORMAT_CHAR_RE.test(text)) {
339
+ const reading = readFormatFree(text);
340
+ if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
341
+ return undefined;
342
+ read(reading.view, reading.toOriginalSpan);
343
+ for (const cut of reading.cuts)
344
+ read(reading.view.slice(cut), (start, end) => reading.toOriginalSpan(start + cut, end + cut));
311
345
  }
312
- extra.sort((a, b) => a.start - b.start);
313
- const firstEndingAfter = (pos) => {
314
- let lo = 0;
315
- let hi = spans.length;
316
- while (lo < hi) {
317
- const mid = (lo + hi) >> 1;
318
- if (spans[mid].end > pos)
319
- hi = mid;
320
- else
321
- lo = mid + 1;
322
- }
323
- return lo;
346
+ return regions;
347
+ }
348
+ function coveredNot(regions, removals) {
349
+ return unremovedHits(regions, removals);
350
+ }
351
+ function renderOneText(input, passes, overStrip, readings, report) {
352
+ const whole = { out: overStrip, removals: () => undefined };
353
+ if (!readings.formatFree || !FORMAT_CHAR_RE.test(input)) {
354
+ const plain = applyPasses(input, passes, overStrip, report, (pos) => pos);
355
+ return { out: plain.out, removals: () => sequentialRemovals(plain.batches) };
356
+ }
357
+ const reading = readFormatFree(input);
358
+ const { view } = reading;
359
+ const { batches } = applyPasses(view, passes, overStrip, report, reading.toOriginal, true);
360
+ if (replacedSpans(batches) === undefined)
361
+ return whole;
362
+ const back = (r, shift) => {
363
+ const kept = reading.toOriginalSpan(r.start + shift, r.end + shift);
364
+ const match = reading.toOriginalSpan(r.matchStart + shift, r.matchEnd + shift);
365
+ return { ...r, start: kept.start, end: kept.end, matchStart: match.start, matchEnd: match.end };
324
366
  };
325
- for (const r of extra) {
326
- const at = firstEndingAfter(r.start);
327
- const hit = spans[at];
328
- if (hit !== undefined && hit.start < r.end) {
329
- if (r.start < hit.start)
330
- hit.start = r.start;
331
- if (r.end > hit.end)
332
- hit.end = r.end;
367
+ const removals = sequentialRemovals(batches).map((r) => back(r, 0));
368
+ if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
369
+ return whole;
370
+ const extra = [];
371
+ if (readings.raw)
372
+ appendAll(extra, sequentialRemovals(applyPasses(input, passes, overStrip, undefined, (pos) => pos, true).batches));
373
+ for (const cut of reading.cuts)
374
+ appendAll(extra, sequentialRemovals(applyPasses(view.slice(cut), passes, overStrip, undefined, (pos) => pos, true).batches).map((r) => back(r, cut)));
375
+ if (report !== undefined) {
376
+ const gone = [];
377
+ const firstEndingAfter = (pos) => {
378
+ let lo = 0;
379
+ let hi = gone.length;
380
+ while (lo < hi) {
381
+ const mid = (lo + hi) >> 1;
382
+ if (gone[mid][1] > pos)
383
+ hi = mid;
384
+ else
385
+ lo = mid + 1;
386
+ }
387
+ return lo;
388
+ };
389
+ const join = (r) => {
390
+ if (r.end <= r.start)
391
+ return false;
392
+ const at = firstEndingAfter(r.start);
393
+ const hit = gone[at];
394
+ if (hit === undefined || hit[0] >= r.end) {
395
+ gone.splice(at, 0, [r.start, r.end]);
396
+ return true;
397
+ }
398
+ hit[0] = Math.min(hit[0], r.start);
399
+ hit[1] = Math.max(hit[1], r.end);
333
400
  let next = at + 1;
334
- while (next < spans.length && spans[next].start < hit.end)
335
- hit.end = Math.max(hit.end, spans[next++].end);
401
+ while (next < gone.length && gone[next][0] < hit[1])
402
+ hit[1] = Math.max(hit[1], gone[next++][1]);
336
403
  if (next > at + 1)
337
- spans.splice(at + 1, next - at - 1);
338
- continue;
339
- }
340
- spans.splice(at, 0, { start: r.start, end: r.end, inserted: r.inserted });
341
- if (report !== undefined)
342
- report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
343
- }
344
- let result = "";
345
- let cursor = 0;
346
- for (const sp of spans) {
347
- result += input.slice(cursor, sp.start) + sp.inserted;
348
- cursor = sp.end;
404
+ gone.splice(at + 1, next - at - 1);
405
+ return false;
406
+ };
407
+ for (const r of removals)
408
+ join(r);
409
+ for (const r of [...extra].sort((a, b) => a.start - b.start))
410
+ if (join(r))
411
+ report.findings.push(finding(passes[r.pass], [r.matchStart, r.matchEnd], r.family));
349
412
  }
350
- result += input.slice(cursor);
351
- return result;
413
+ const all = [...removals, ...extra];
414
+ return { out: renderRemovals(input, all, overStrip).out, removals: () => all };
352
415
  }
353
- const SECRET_GRADES = ["strict", "wide"];
354
- export const SECRET_READING_KINDS = Object.freeze(["raw", "format-free"]);
416
+ const SECRET_GRADES = ["strict", "wide", "fragment"];
355
417
  function fail(what) {
356
418
  throw new TypeError(`secret scan: ${what}`);
357
419
  }
@@ -362,24 +424,12 @@ function readPolicy(policy) {
362
424
  const { grade, readings, urlUserinfo, onUnresolvedReading } = policy;
363
425
  if (!SECRET_GRADES.includes(grade))
364
426
  fail(`policy.grade must be one of ${SECRET_GRADES.join(" / ")}`);
365
- if (!Array.isArray(readings) || readings.length === 0)
366
- fail("policy.readings must be a non-empty array");
367
- const kinds = new Set();
368
- for (const r of readings) {
369
- const named = isRecord(r) ? r.kind : undefined;
370
- const kind = SECRET_READING_KINDS.find((k) => k === named);
371
- if (kind === undefined)
372
- fail(`every policy.readings entry is { kind } with kind one of ${SECRET_READING_KINDS.join(" / ")}`);
373
- if (Object.keys(r).some((k) => k !== "kind"))
374
- fail(`the reading "${kind}" takes no parameter`);
375
- kinds.add(kind);
376
- }
427
+ const set = readReadings(readings);
377
428
  if (urlUserinfo !== "keep-username" && urlUserinfo !== "whole")
378
429
  fail("policy.urlUserinfo must be keep-username / whole");
379
430
  if (onUnresolvedReading !== "secret" && onUnresolvedReading !== "ignore")
380
431
  fail("policy.onUnresolvedReading must be secret / ignore");
381
- const set = { raw: kinds.has("raw"), formatFree: kinds.has("format-free") };
382
- return { grade, readings: set, urlUserinfo, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
432
+ return { grade, readings: set, urlUserinfo, onUnresolvedReading, key: `${grade} ${set.raw} ${set.formatFree} ${urlUserinfo}` };
383
433
  }
384
434
  const ROW_IDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.id));
385
435
  const ROW_KINDS = new Set(ENGINE_VOCABULARY.values.map((r) => r.kind));
@@ -416,10 +466,19 @@ const USERINFO_FAMILY = "url-userinfo";
416
466
  if (!ROW_IDS.has(USERINFO_FAMILY))
417
467
  throw new Error(`secret scan: the vocabulary has no "${USERINFO_FAMILY}" row`);
418
468
  const BARE_USERNAME_MAX_LEN = 20;
469
+ const DECODED_USERNAME_FORMS = ENGINE_VOCABULARY.values.map((row) => new RegExp(row.match.source, row.match.flags.replace("g", "")));
470
+ const readsDecodedAsCredential = (username) => {
471
+ if (!username.includes("%"))
472
+ return false;
473
+ const decoded = percentDecodedOnce(username);
474
+ return decoded !== username && DECODED_USERNAME_FORMS.some((re) => re.test(decoded));
475
+ };
419
476
  function userinfoWithheld(userinfo, urlUserinfo) {
420
477
  if (urlUserinfo === "whole" || userinfo.includes("@"))
421
478
  return [0, userinfo.length];
422
479
  const colonIdx = userinfo.indexOf(":");
480
+ if (readsDecodedAsCredential(colonIdx === -1 ? userinfo : userinfo.slice(0, colonIdx)))
481
+ return [0, userinfo.length];
423
482
  if (colonIdx === -1)
424
483
  return userinfo.length >= BARE_USERNAME_MAX_LEN ? [0, userinfo.length] : undefined;
425
484
  return [colonIdx + 1, userinfo.length];
@@ -455,13 +514,21 @@ function strictPasses(markOf) {
455
514
  return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: new RegExp(row.strict.source, `${row.strict.flags}g`), replace: marker, family: row.id }];
456
515
  });
457
516
  }
517
+ function fragmentPasses(markOf) {
518
+ return ENGINE_VOCABULARY.values.flatMap((row) => {
519
+ if (row.fragment === undefined)
520
+ return [];
521
+ const marker = markOf(row);
522
+ return [{ kind: row.kind, confidence: row.confidence, marker, source: row.source, re: row.fragment, replace: capturesLabel(row.fragment) ? `$1${marker}` : marker, family: row.id }];
523
+ });
524
+ }
458
525
  const COMPILED_PASSES = new Map();
459
526
  const COMPILED_PASSES_MAX = 64;
460
527
  function tablePasses(read, marks) {
461
528
  const key = `${read.key}\u0000${marks.key}`;
462
529
  let passes = COMPILED_PASSES.get(key);
463
530
  if (passes === undefined) {
464
- passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), marks.markOf, read.urlUserinfo) : strictPasses(marks.markOf));
531
+ passes = Object.freeze(read.grade === "wide" ? egressPasses(egressPassGroups(ENGINE_VOCABULARY), marks.markOf, read.urlUserinfo) : read.grade === "strict" ? strictPasses(marks.markOf) : fragmentPasses(marks.markOf));
465
532
  if (COMPILED_PASSES.size >= COMPILED_PASSES_MAX)
466
533
  COMPILED_PASSES.clear();
467
534
  COMPILED_PASSES.set(key, passes);
@@ -473,15 +540,16 @@ export function renderSecrets(text, policy, notation, options = {}) {
473
540
  const marks = readNotation(notation);
474
541
  const { passes: extra, report } = options;
475
542
  const own = tablePasses(read, marks);
476
- return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, report });
543
+ return runRedactionPasses(text, extra === undefined || extra.length === 0 ? own : [...own, ...extra], { overStrip: marks.overStrip, readings: read.readings, onUnresolvedReading: read.onUnresolvedReading, report });
477
544
  }
478
545
  const FORMS = {
479
546
  strict: ENGINE_VOCABULARY.values.flatMap((row) => (row.strict === undefined ? [] : [{ row, re: new RegExp(row.strict.source, `${row.strict.flags}g`), cut: "whole" }])),
480
547
  wide: ENGINE_VOCABULARY.values.map((row) => ({ row, re: new RegExp(row.match.source, row.match.flags), cut: row.id === USERINFO_FAMILY ? "userinfo" : capturesLabel(row.match) ? "label" : "whole" })),
548
+ fragment: ENGINE_VOCABULARY.values.flatMap((row) => (row.fragment === undefined ? [] : [{ row, re: new RegExp(row.fragment.source, row.fragment.flags), cut: capturesLabel(row.fragment) ? "label" : "whole" }])),
481
549
  };
482
550
  function scannerFor(policy) {
483
551
  const read = readPolicy(policy);
484
- return { forms: FORMS[read.grade], readings: read.readings, urlUserinfo: read.urlUserinfo };
552
+ return { forms: FORMS[read.grade], readings: read.readings, urlUserinfo: read.urlUserinfo, onUnresolvedReading: read.onUnresolvedReading };
485
553
  }
486
554
  function formHits(input, scan) {
487
555
  const hits = [];
@@ -515,21 +583,31 @@ function formHits(input, scan) {
515
583
  }
516
584
  return hits;
517
585
  }
586
+ function characterHits(text, scan) {
587
+ if (!scan.readings.formatFree || !FORMAT_CHAR_RE.test(text))
588
+ return formHits(text, scan);
589
+ const reading = readFormatFree(text);
590
+ if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
591
+ return undefined;
592
+ const back = (h, shift) => ({ ...h, ...reading.toOriginalSpan(h.start + shift, h.end + shift) });
593
+ const spans = formHits(reading.view, scan).map((h) => back(h, 0));
594
+ if (scan.readings.raw)
595
+ appendAll(spans, formHits(text, scan));
596
+ for (const cut of reading.cuts)
597
+ appendAll(spans, formHits(reading.view.slice(cut), scan).map((h) => back(h, cut)));
598
+ return spans;
599
+ }
518
600
  export function scanSecrets(text, policy) {
519
601
  const scan = scannerFor(policy);
520
- let spans;
521
- if (!scan.readings.formatFree || !FORMAT_CHAR_RE.test(text))
522
- spans = formHits(text, scan);
523
- else {
524
- const reading = readFormatFree(text);
525
- if (reading.cuts.length > MAX_FORMAT_SLOT_SCANS)
602
+ const derived = readsTextReadings(scan.readings) ? readingViews(text, scan.readings) : undefined;
603
+ if (derived?.unresolved === true && scan.onUnresolvedReading === "secret")
604
+ return { outcome: "whole", reason: "reading-unresolved" };
605
+ const spans = [];
606
+ for (const view of derived?.views ?? [undefined]) {
607
+ const read = characterHits(view?.text ?? text, scan);
608
+ if (read === undefined)
526
609
  return { outcome: "whole", reason: "scan-budget" };
527
- const back = (h, shift) => ({ ...h, ...reading.toOriginalSpan(h.start + shift, h.end + shift) });
528
- spans = formHits(reading.view, scan).map((h) => back(h, 0));
529
- if (scan.readings.raw)
530
- spans.push(...formHits(text, scan));
531
- for (const cut of reading.cuts)
532
- spans.push(...formHits(reading.view.slice(cut), scan).map((h) => back(h, cut)));
610
+ appendAll(spans, view === undefined || view.text === text ? read : read.map((h) => ({ ...h, ...view.toOriginalSpan(h.start, h.end) })));
533
611
  }
534
612
  const seen = new Set();
535
613
  const unique = spans.filter((s) => {
@@ -6,14 +6,20 @@
6
6
  * - egress ({@link egressPassGroups}): every row's wide `match`, in table order, grouped into passes. A hit replaces
7
7
  * the value in text that is persisted and forwarded, so a wide form is broad: a missed credential is that face's
8
8
  * head risk and over-redaction is cheap.
9
- * - env names ({@link envNameView}): the exact credential names, the credential words by position, the
10
- * connection-string name shape and the token-prefix value predicate a spawned child's environment is judged by.
9
+ * - the fragment grade (`scanSecrets` with `grade: "fragment"`): the rows that carry a `fragment` form — what the residue
10
+ * of a credential cut inside its body still reads as (a lower floor than the wide form, a word start, and two
11
+ * upper-case letters or digits in the body where the prefix is short).
12
+ * - env names ({@link envNameView}): the exact credential names, the credential words by position and the
13
+ * connection-string name shape a spawned child's environment is judged by.
14
+ * - env values ({@link envValueView}): every row's `envValue` form and the vendor token heads (`envValueHead`) a
15
+ * child's environment VALUE is judged by.
11
16
  * - record keys ({@link recordKeyView}): the same credential words, read as the name of a member of a structured record
12
17
  * (a header name, a config member) rather than of an environment variable.
13
18
  *
14
- * A family's strict and wide forms live on the SAME row, and every text the strict form matches the wide form matches
15
- * too (machine-judged over the fixture file by `scripts/verify-credential-vocabulary.mjs`): the two faces may differ in
16
- * floors, never in vocabulary. Detection is shared; each layer renders its own marker.
19
+ * A family's strict, wide, fragment and env forms live on the SAME row, and every text the strict form matches the wide
20
+ * form matches too (machine-judged over the fixture file by `scripts/verify-credential-vocabulary.mjs`): the two faces may
21
+ * differ in floors, never in vocabulary. Detection is shared; each layer renders its own marker. The credential NAMES a
22
+ * value stands beside (a keyword ahead of `=`, a URL parameter) are one list too ({@link CredentialVocabulary.names}).
17
23
  *
18
24
  * The table holds no layer's policy: which form a reader reads, how a text is read first, which marker replaces a hit
19
25
  * and whether a URL's username is kept are the caller's, passed in whole (`SecretScanPolicy`, `SecretNotation` in
@@ -43,6 +49,26 @@ export interface ValueFamilyRow {
43
49
  * alternation, so a token is taken at its leftmost start whichever of them it is (a nested shape is never cut by
44
50
  * an earlier family's pass). Absent = a pass of its own. */
45
51
  readonly pass?: string;
52
+ /** The fragment form, global: the residue of this family's credential when a text was cut inside its body. Present =
53
+ * the fragment grade reads this family; absent = it does not (a family with no distinctive head has no residue that
54
+ * still reads as one). */
55
+ readonly fragment?: RegExp;
56
+ /** The env value form, neither global nor sticky: a value it matches carries this family's credential, read by the env
57
+ * value view ({@link envValueView}) on a spawned child's environment. */
58
+ readonly envValue?: RegExp;
59
+ /** The head a token of this family opens with, unanchored, no flags: present = a value or a password-less URL user
60
+ * opening with it is token-shaped to the env value view ({@link EnvValueView.tokenHead}); absent = the family's tokens
61
+ * are not judged by their head there. */
62
+ readonly envValueHead?: RegExp;
63
+ }
64
+ /** Where a credential NAME is read beside its value: `assignment` — a keyword ahead of `=` / `:` in free text (the
65
+ * keyword-value row, camel humps included); `url-param` — the name of a URL query or fragment parameter, any case (the
66
+ * signed-url-param row). */
67
+ export type CredentialNamePlace = "assignment" | "url-param";
68
+ /** One credential name: "the value this name stands beside is a credential", in each place it lists. */
69
+ export interface CredentialNameRow {
70
+ readonly name: string;
71
+ readonly places: readonly CredentialNamePlace[];
46
72
  }
47
73
  /** Where a credential word counts in a name (env names are upper-cased words joined by `_`):
48
74
  * - `leading` — at the start of the name or right after `_`;
@@ -86,8 +112,12 @@ export interface CredentialVocabulary {
86
112
  /** A leading vendor prefix that spells a credential word without being one (`AUTH0_`): dropped from a name before the
87
113
  * credential words are looked for, in every name view alike. */
88
114
  readonly nonCredentialPrefix: RegExp;
89
- /** A value (or a password-less URL user) opening with a known token prefix. */
90
- readonly tokenPrefix: RegExp;
115
+ /** The credential names a value stands beside, one list for every place a name is read: the keyword-value row reads the
116
+ * `assignment` names (in this order), the signed-url-param row the `url-param` names. */
117
+ readonly names: readonly CredentialNameRow[];
118
+ /** The words a setting's value is when it is a switch, not a secret (lower case, read in any case): the env value view's
119
+ * assignment form leaves a value spelled as one of them. */
120
+ readonly flagWords: readonly string[];
91
121
  }
92
122
  /**
93
123
  * A URL of any scheme ({@link URI_SCHEME_SOURCE}, then `://`), to the first white space, quote or backtick, any case. URL
@@ -110,7 +140,7 @@ export declare const CREDENTIAL_VOCABULARY: CredentialVocabulary;
110
140
  export declare const ENGINE_VOCABULARY: CredentialVocabulary;
111
141
  /** The egress view: the rows in table order, each run of adjacent rows naming the same `pass` as one group. */
112
142
  export declare function egressPassGroups(vocabulary: CredentialVocabulary): ReadonlyArray<readonly ValueFamilyRow[]>;
113
- /** The env-name view: what {@link CREDENTIAL_VOCABULARY} says about variable names and token-prefixed values. */
143
+ /** The env-name view: what {@link CREDENTIAL_VOCABULARY} says about variable names. */
114
144
  export interface EnvNameView {
115
145
  /** Upper-cased, `-` read as `_`; each name also under `INPUT_`. */
116
146
  readonly exactNames: ReadonlySet<string>;
@@ -123,11 +153,27 @@ export interface EnvNameView {
123
153
  /** A name opening with `namespace` (as written) that holds a credential word as ANY whole `_` segment after it —
124
154
  * every position but `embedded`; a counted plural no count word touches — any case, `.` crossing line breaks. */
125
155
  readonly anySegmentUnder: (namespace: string) => RegExp;
126
- readonly tokenPrefix: RegExp;
127
156
  }
128
157
  /** The env-name view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
129
158
  * Every regex of a view is the view's own object, so a reader that recompiles one changes only its own view. */
130
159
  export declare function envNameView(vocabulary?: CredentialVocabulary): EnvNameView;
160
+ /** The env value view: what {@link CREDENTIAL_VOCABULARY} says about a spawned child's environment VALUES, every member
161
+ * derived from the rows. */
162
+ export interface EnvValueView {
163
+ /** Every row's env value form, in table order: a value one of them matches carries a credential. */
164
+ readonly forms: ReadonlyArray<{
165
+ readonly family: string;
166
+ readonly re: RegExp;
167
+ }>;
168
+ /** A value (or a password-less URL user) opening with a vendor token head: every row's `envValueHead`, in table order,
169
+ * as one anchored alternation. */
170
+ readonly tokenHead: RegExp;
171
+ /** The words a setting's value is when it is a switch, lower case (compare a value lower-cased). */
172
+ readonly flagWords: ReadonlySet<string>;
173
+ }
174
+ /** The env value view of `vocabulary`; with no argument, of the package's own copy of the table ({@link ENGINE_VOCABULARY}).
175
+ * Every regex of a view is the view's own object. */
176
+ export declare function envValueView(vocabulary?: CredentialVocabulary): EnvValueView;
131
177
  /** `camelCase` / `PascalCase` read as `_`-separated words, `OAuth` and `NextAuth` kept whole. */
132
178
  export declare function splitCamelCase(name: string): string;
133
179
  /** The record-key view: what {@link CREDENTIAL_VOCABULARY} says about the names a structured record carries — the member
@@ -11,7 +11,8 @@ const PLAIN_QUOTES = [`"`, `'`, `&quot;`, `&apos;`, `&#34;`, `&#39;`, `%22`, `%2
11
11
  const ESCAPED_QUOTES = [`"`, `'`, `x22`, `x27`, `u0022`, `u0027`, `042`, `047`];
12
12
  const BACKSLASHES = String.raw `\\{1,${ESCAPE_DEPTH}}`;
13
13
  const QUOTE = `(?:${PLAIN_QUOTES.join("|")}|${BACKSLASHES}(?:${ESCAPED_QUOTES.join("|")}))`;
14
- const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE}))`;
14
+ const ESCAPE_TAIL = String.raw `(?:\\(?:[bfnrt]|u[0-9A-Fa-f]{4})|%[0-9A-Fa-f]{2})`;
15
+ const WORD_START = `(?:(?<![A-Za-z0-9])|(?<=${QUOTE})|(?<=${ESCAPE_TAIL}))`;
15
16
  const AFTER_CLOSING_QUOTE = String.raw `(?:[\s,}\]:]|["']|$)`;
16
17
  const VALUE_RUN_END = `(?:(?<!\\\\)|(?!\\\\{0,${ESCAPE_DEPTH}}["']))`;
17
18
  const GLUED_RUN = `${VALUE_CHAR}*${VALUE_RUN_END}`;
@@ -25,35 +26,105 @@ const BEARER_TAIL = `(?:(?!${QUOTE})\\S${GLUED_RUN}|(?<==)(?:${QUOTED_VALUE}))`;
25
26
  const HEADER_PARAM = String.raw `=[ \t]*(\\{0,${ESCAPE_DEPTH}})(["'])(?:(?!(?<!\\)\2\3)[^\r\n])*(?:(?<!\\)\2\3(?=[\s,;]|\\{0,${ESCAPE_DEPTH}}["']|$)|(?=[\r\n]|$))`;
26
27
  const HEADER_VALUE = String.raw `(?:[A-Za-z][A-Za-z0-9!#$%&*+.^_|~-]*[ \t]+)?(?:${HEADER_PARAM}|[^\s"',]|[ \t]*,[ \t]*(?=[^\s"',])|[ \t]+(?==)|(?<==)[ \t]+(?=[^\s"',]))+${VALUE_RUN_END}`;
27
28
  const headed = (body, flags = "g") => new RegExp(`${WORD_START}${body}`, flags);
29
+ const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
28
30
  const HUMP_BEFORE = "[a-z0-9]";
29
- const KEYWORD_VALUE_WORDS = ["token", "secret", "key", "password", "passwd", "credential"];
31
+ const CREDENTIAL_NAMES = [
32
+ ...["token", "secret", "key", "password", "passwd"].map((name) => ({ name, places: ["assignment", "url-param"] })),
33
+ { name: "credential", places: ["assignment"] },
34
+ ...[
35
+ "api_key", "apikey", "api-key", "access_token", "refresh_token", "id_token", "auth", "authorization", "proxy-authorization",
36
+ "pwd", "client_secret", "private_key", "signature", "sig", "x-amz-signature", "x-amz-security-token", "x-amz-credential", "sas", "code",
37
+ ].map((name) => ({ name, places: ["url-param"] })),
38
+ ];
39
+ const namesAt = (place) => CREDENTIAL_NAMES.filter((n) => n.places.includes(place)).map((n) => n.name);
40
+ const KEYWORD_VALUE_WORDS = namesAt("assignment");
41
+ const FLAG_WORDS = ["true", "false", "none", "null", "yes", "no", "on", "off", "enabled", "disabled", "required", "optional"];
42
+ const residue = (head, cls, min) => String.raw `${head}(?=${cls}{0,96}[0-9A-Z]${cls}{0,96}[0-9A-Z])${cls}{${min},}`;
43
+ const GITLAB_HEAD = "gl(?:pat|dt|rt|rtr|cbt|ptt|oas|ft|imt|agent|wt|soat|ffct)-";
30
44
  const anyCase = (word) => [...word].map((c) => `[${c.toUpperCase()}${c.toLowerCase()}]`).join("");
31
- const VALUE_ROW_FIELDS = [["Account", "Key"]];
32
- const leftToItsRow = (keyword) => VALUE_ROW_FIELDS.filter(([, k]) => k.toLowerCase() === keyword).map(([front]) => `(?<!(?<![A-Za-z0-9])${front})`).join("");
45
+ const AZURE_ACCOUNT_KEY = { front: "Account", keyword: "Key", separator: "=", body: "A-Za-z0-9+/" };
46
+ const AZURE_FIELD = `${AZURE_ACCOUNT_KEY.front}${AZURE_ACCOUNT_KEY.keyword}${AZURE_ACCOUNT_KEY.separator}`;
47
+ const AZURE_BODY = `[${AZURE_ACCOUNT_KEY.body}]{40,}={0,2}`;
48
+ const VALUE_ROW_FIELDS = [
49
+ [AZURE_ACCOUNT_KEY.front, AZURE_ACCOUNT_KEY.keyword, AZURE_ACCOUNT_KEY.separator, `[${AZURE_ACCOUNT_KEY.body}<]`],
50
+ ];
51
+ const leftToItsRow = (keyword) => VALUE_ROW_FIELDS.filter(([, k]) => k.toLowerCase() === keyword)
52
+ .map(([front, k, separator, valueStart]) => `(?!(?<=${WORD_START}${front})${k}${separator}${valueStart})`)
53
+ .join("");
33
54
  const KEYWORD_HEAD = `(?:${WORD_START}(?:${KEYWORD_VALUE_WORDS.map(anyCase).join("|")})|(?<=${HUMP_BEFORE})(?:${KEYWORD_VALUE_WORDS.map((w) => leftToItsRow(w) + w[0].toUpperCase() + anyCase(w.slice(1))).join("|")})|(?<=[A-Z])(?:${KEYWORD_VALUE_WORDS.map((w) => w[0].toUpperCase() + w[1] + anyCase(w.slice(2))).join("|")}))`;
34
55
  const KEY_QUOTE = `(?:\\\\{0,${ESCAPE_DEPTH}}["'])?`;
35
56
  export const CREDENTIAL_VOCABULARY = {
36
57
  values: [
37
- { id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/ },
38
- { id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/ },
39
- { id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/ },
40
- { id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test)_[A-Za-z0-9]{24,}\b/ },
41
- { id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`) },
42
- { id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("glpat-[A-Za-z0-9_-]{20,}") },
43
- { id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}") },
44
- { id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`) },
58
+ { id: "github-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`gh[opsur][-_]${VENDOR_BODY}`), strict: /\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b/, fragment: headed(residue("(?:ghp|gho|ghu|ghs|ghr)_", "[A-Za-z0-9]", 8)), envValueHead: /gh[opusr]_/ },
59
+ { id: "github-fine-grained-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`github_pat[-_]${VENDOR_BODY}`), strict: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/, fragment: /github_pat_[A-Za-z0-9_]{8,}/g, envValueHead: /github_pat_/ },
60
+ { id: "openai-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`sk-${VENDOR_BODY}`), strict: /\bsk-(?:ant-(?:api03-)?|proj-)?[A-Za-z0-9_]{20,}\b/, fragment: headed(residue("sk-", "[A-Za-z0-9_-]", 6)), envValueHead: /sk-/ },
61
+ { id: "stripe-secret-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`(?:sk_|rk[-_])${VENDOR_BODY}`), strict: /\b[sr]k_(?:live|test|org)_[A-Za-z0-9]{24,}\b/, fragment: headed(residue("(?:sk|rk)_(?:live|test|org)_", "[A-Za-z0-9_-]", 8)) },
62
+ { id: "stripe-publishable-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`pk[-_]${VENDOR_BODY}`), envValueHead: /pk-/ },
63
+ { id: "gitlab-pat", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(`${GITLAB_HEAD}[A-Za-z0-9_-]{20,}`), strict: new RegExp(String.raw `\b${GITLAB_HEAD}(?=[A-Za-z0-9_]*[0-9])[A-Za-z0-9_]{20,}\b`), fragment: headed(residue(GITLAB_HEAD, "[A-Za-z0-9_-]", 8)), envValueHead: /glpat-/ },
64
+ { id: "npm-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("npm_[A-Za-z0-9]{36,}"), envValueHead: /npm_/ },
65
+ { id: "google-oauth-access-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed(String.raw `ya29\.[A-Za-z0-9_.-]{20,}`), envValueHead: /ya29\./ },
66
+ { id: "pypi-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", pass: "vendor-prefixed", match: headed("pypi-[A-Za-z0-9_-]{85,}"), strict: /\bpypi-(?=[A-Za-z0-9_-]*[0-9])[A-Za-z0-9_-]{85,}/, fragment: headed(residue("pypi-", "[A-Za-z0-9_-]", 8)) },
45
67
  { id: "google-api-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("AIza[A-Za-z0-9_-]{20,}"), strict: /\bAIza[0-9A-Za-z_-]{35}\b/ },
46
- { id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprs]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprs]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/ },
47
- { id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/ },
68
+ { id: "slack-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("xox[baprse]-[A-Za-z0-9-]{10,}"), strict: /\bxox[baprse]-(?:[A-Za-z0-9]+-){1,3}[A-Za-z0-9]{16,}\b/, fragment: headed(residue("xox[baprse]-", "[A-Za-z0-9-]", 4)), envValueHead: /xox[abpr]-/ },
69
+ { id: "slack-app-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed(String.raw `xapp-\d-[A-Za-z0-9-]{20,}`), strict: /\bxapp-\d-(?=[A-Za-z0-9-]{20})(?:[A-Za-z0-9]+-){1,3}(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{16,}\b/, fragment: headed(residue(String.raw `xapp-\d-`, "[A-Za-z0-9-]", 4)) },
70
+ { id: "aws-access-key-id", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("(?:A3T[A-Z0-9]|AKIA|ASIA|ABIA|ACCA)[0-9A-Z]{12,}"), strict: /\bAKIA[0-9A-Z]{16}\b/, fragment: headed(`(?:${residue("AKIA", "[0-9A-Z]", 6)}|${residue("ASIA", "[0-9A-Z]", 8)})`), envValueHead: /AKIA/ },
48
71
  { id: "huggingface-token", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: headed("hf_[A-Za-z0-9]{20,}"), strict: /\bhf_(?=[A-Za-z0-9]*[0-9])[A-Za-z0-9]{20,}\b/ },
49
- { id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/ },
50
- { id: "private-key", kind: "private-key-block", source: "arg-summary", confidence: "high", match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)/g, strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/ },
51
- { id: "bearer-token", kind: "bearer-token", source: "arg-summary", confidence: "high", match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`), strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/ },
52
- { id: "authorization-header", kind: "authorization-header", source: "arg-summary", confidence: "high", match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi") },
53
- { id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: /(AccountKey=)[A-Za-z0-9+/]{40,}={0,2}/g, strict: /AccountKey=[A-Za-z0-9+/]{40,}={0,2}/ },
54
- { id: "keyword-value", kind: "keyword-value", source: "arg-summary", confidence: "medium", match: new RegExp(String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "g") },
72
+ { id: "jwt", kind: "jwt", source: "arg-summary", confidence: "high", match: headed("eyJ[A-Za-z0-9._-]{20,}"), strict: /\beyJ[A-Za-z0-9_-]+\.eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]{16,}\b/, envValueHead: /eyJ/ },
73
+ {
74
+ id: "private-key",
75
+ kind: "private-key-block",
76
+ source: "arg-summary",
77
+ confidence: "high",
78
+ match: /-----BEGIN[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?(?:-----END[ A-Z0-9]*PRIVATE KEY(?: BLOCK)?-----|$)|---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]*?(?:---- END SSH2 ENCRYPTED PRIVATE KEY ----|$)|PuTTY-User-Key-File-\d+:[\s\S]*?(?:Private-MAC:[ \t]*[0-9A-Fa-f]+|$)|Private-Lines:[ \t]*\d+(?:\r?\n[A-Za-z0-9+/=]+(?=\r?\n|$))+/g,
79
+ strict: /-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----/,
80
+ envValue: /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/,
81
+ },
82
+ {
83
+ id: "bearer-token",
84
+ kind: "bearer-token",
85
+ source: "arg-summary",
86
+ confidence: "high",
87
+ match: headed(String.raw `()[Bb][Ee][Aa][Rr][Ee][Rr]\s+[A-Za-z0-9._~+/-]{${CREDENTIAL_MIN_CHARS},}=*(?:${BEARER_TAIL})?`),
88
+ strict: /\bAuthorization:\s*Bearer\s+(?=[A-Za-z0-9._~+/-]*[0-9])[A-Za-z0-9._~+/-]{20,}=*/,
89
+ envValue: new RegExp(String.raw `\b(?:Bearer|Basic)\s+(?=[A-Za-z._~+\/=-]*[0-9]|(?:[A-Za-z0-9._~+\/=-]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9.-]*[_~+\/=])[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`),
90
+ },
91
+ {
92
+ id: "authorization-header",
93
+ kind: "authorization-header",
94
+ source: "arg-summary",
95
+ confidence: "high",
96
+ match: headed(String.raw `(authorization(?:\\{0,${ESCAPE_DEPTH}}["'])?[ \t]*[=:][ \t]*(?:\\{0,${ESCAPE_DEPTH}}["'])?)${HEADER_VALUE}`, "gi"),
97
+ envValue: new RegExp(String.raw `\bauthorization\s*[:=]\s*["']?[a-z][a-z0-9_-]*\s+[A-Za-z0-9._~+\/=-]{${CREDENTIAL_MIN_CHARS},}`, "i"),
98
+ },
99
+ { id: "basic-credential", kind: "scheme-credential", source: "arg-summary", confidence: "high", match: headed(String.raw `[Bb][Aa][Ss][Ii][Cc]\s+(?=[A-Za-z+/]*[0-9]|(?:[A-Za-z0-9+/]*?[a-z][A-Z](?![a-z])){2}|[A-Za-z0-9]*[+/=])[A-Za-z0-9+/]{${CREDENTIAL_MIN_CHARS},}={0,2}`) },
100
+ {
101
+ id: "token-scheme",
102
+ kind: "scheme-credential",
103
+ source: "arg-summary",
104
+ confidence: "high",
105
+ match: headed(String.raw `()(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=([A-Za-z0-9_~+=-]{20,}))\2(?![\/.])`),
106
+ envValue: /\b(?:[Tt]oken|TOKEN)\s+(?=[A-Za-z0-9_~+=-]*[0-9])(?=[A-Za-z0-9_~+=-]*[A-Z])(?=([A-Za-z0-9_~+=-]{20,}))\1(?![\/.])/,
107
+ },
108
+ { id: "azure-storage-account-key", kind: "prefixed-token", source: "arg-summary", confidence: "high", match: new RegExp(`(${AZURE_FIELD})${AZURE_BODY}`, "g"), strict: new RegExp(`${AZURE_FIELD}${AZURE_BODY}`) },
109
+ {
110
+ id: "keyword-value",
111
+ kind: "keyword-value",
112
+ source: "arg-summary",
113
+ confidence: "medium",
114
+ match: new RegExp(String.raw `(${KEYWORD_HEAD}${KEY_QUOTE}\s*[=:]\s*${KEY_QUOTE})${VALUE_CHAR}{4,}${VALUE_RUN_END}`, "g"),
115
+ envValue: new RegExp(String.raw `(?:(?:^|[;&?#,{]|\/:)\s*|\s)["']?(?!-*jobserver-auth\s*[=:])(?:[a-z0-9_.-]{0,64}(?:password|passwd|pwd|secret|token|(?:account|access|api|private|subscription)[-_]?key|signature|sig|credential)|[a-z0-9_.-]{0,63}[_.-]auth)["']?\s*[=:]\s*["']?(?!(?:${FLAG_WORDS.join("|")})(?:$|[;&,\s"']))[^;&,\s"']+`, "i"),
116
+ },
117
+ { id: "sonar-login", kind: "keyword-value", source: "arg-summary", confidence: "high", match: new RegExp(String.raw `(${KEY_QUOTE}sonar\.login${KEY_QUOTE}\s*[:=]\s*${KEY_QUOTE})[^\s"',}]{${CREDENTIAL_MIN_CHARS},}${VALUE_RUN_END}`, "g"), envValue: new RegExp(String.raw `["']?sonar\.login["']?\s*[:=]\s*["']?[^\s"',}]{${CREDENTIAL_MIN_CHARS},}`) },
55
118
  { id: "url-userinfo", kind: "url-credentials", source: "untrusted-egress", confidence: "medium", match: new RegExp(String.raw `(${URI_SCHEME_SOURCE}:\/\/)(${URL_USERINFO_SOURCE})`, "gi") },
56
- { id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: /([?&#](?:sig|signature|sas|code)=)[^&\s"'#]+/gi },
119
+ {
120
+ id: "webhook-url",
121
+ kind: "webhook-url",
122
+ source: "untrusted-egress",
123
+ confidence: "high",
124
+ match: /(https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/))[\w\/@.~-]{16,}/gi,
125
+ envValue: /https:\/\/(?:hooks\.slack\.com\/(?:services|workflows|triggers)\/|(?:ptb\.|canary\.)?discord(?:app)?\.com\/api\/webhooks\/\d+\/|[\w.-]+\.webhook\.office\.com\/webhookb2\/)[\w\/@.~-]{16,}/i,
126
+ },
127
+ { id: "signed-url-param", kind: "signed-url-param", source: "untrusted-egress", confidence: "high", match: new RegExp(String.raw `([?&#](?:${namesAt("url-param").map(literally).join("|")})=)[^&\s"'#]+${VALUE_RUN_END}`, "gi") },
57
128
  ],
58
129
  words: [
59
130
  ...["TOKEN", "SECRET", "PASSWORD", "PASSWD", "PASSPHRASE", "KEY"].map((word) => ({ word, position: "leading" })),
@@ -100,7 +171,8 @@ export const CREDENTIAL_VOCABULARY = {
100
171
  ],
101
172
  nonSecretNameShape: /^GIT_CONFIG_KEY_[0-9][A-Za-z0-9_]*$/,
102
173
  nonCredentialPrefix: /^AUTH0_/i,
103
- tokenPrefix: /^(?:gh[opusr]_|github_pat_|glpat-|xox[abpr]-|sk-|pk-|AKIA|eyJ|ya29\.|npm_)/,
174
+ names: CREDENTIAL_NAMES,
175
+ flagWords: FLAG_WORDS,
104
176
  };
105
177
  function freezeTable(node) {
106
178
  if (typeof node !== "object" || node === null || node instanceof RegExp || Object.isFrozen(node))
@@ -131,7 +203,6 @@ export function egressPassGroups(vocabulary) {
131
203
  }
132
204
  return groups;
133
205
  }
134
- const literally = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
135
206
  function credentialWordPattern(countWords, rows) {
136
207
  const at = (position) => rows.filter((r) => r.position === position).map((r) => r.word);
137
208
  const countedPlural = `(?<!(?:^|_)(?:${countWords.join("|")})_)(?:${at("counted-plural").join("|")})S`;
@@ -154,7 +225,14 @@ export function envNameView(vocabulary = ENGINE_VOCABULARY) {
154
225
  credentialWord,
155
226
  connectionStringName: withOwnRegexes(vocabulary.connectionStringName),
156
227
  anySegmentUnder: (namespace) => new RegExp(`^${literally(namespace)}(?:.*_)?${segmentWord}(?:_|$)`, "is"),
157
- tokenPrefix: withOwnRegexes(vocabulary.tokenPrefix),
228
+ };
229
+ }
230
+ export function envValueView(vocabulary = ENGINE_VOCABULARY) {
231
+ const heads = vocabulary.values.flatMap((row) => (row.envValueHead === undefined ? [] : [row.envValueHead.source]));
232
+ return {
233
+ forms: Object.freeze(vocabulary.values.flatMap((row) => (row.envValue === undefined ? [] : [Object.freeze({ family: row.id, re: withOwnRegexes(row.envValue) })]))),
234
+ tokenHead: new RegExp(heads.length === 0 ? "(?!)" : `^(?:${heads.join("|")})`),
235
+ flagWords: new Set(vocabulary.flagWords),
158
236
  };
159
237
  }
160
238
  export function splitCamelCase(name) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sema-agent/primitives",
3
- "version": "7.64.0",
3
+ "version": "7.65.0",
4
4
  "private": false,
5
5
  "description": "Shared low-level definitions of the sema engine, importable by any layer: the credential reference (closed set of kinds, one reader) and the secret-shape detection definition (one table, its matcher and its renderer, no layer's policy). Zero dependencies.",
6
6
  "type": "module",
@@ -31,5 +31,5 @@
31
31
  "scripts": {
32
32
  "build": "tsc -p tsconfig.json && tsc -p tsconfig.json --emitDeclarationOnly --removeComments false"
33
33
  },
34
- "gitHead": "29e59bdc3537c17524e49692f2487e2b0fba88cd"
34
+ "gitHead": "5ab3f8045ee0d9965d58344531544df295422b69"
35
35
  }