@redact-secret/core 0.1.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +202 -0
- package/dist/adapters/node-stream.d.ts +51 -0
- package/dist/adapters/node-stream.js +79 -0
- package/dist/adapters/shared.d.ts +33 -0
- package/dist/adapters/shared.js +81 -0
- package/dist/adapters/web-stream.d.ts +57 -0
- package/dist/adapters/web-stream.js +115 -0
- package/dist/errors.d.ts +41 -0
- package/dist/errors.js +82 -0
- package/dist/formatters.d.ts +23 -0
- package/dist/formatters.js +22 -0
- package/dist/index.d.ts +64 -0
- package/dist/index.js +62 -0
- package/dist/native.d.ts +80 -0
- package/dist/native.js +21 -0
- package/dist/runtime/browser.d.ts +83 -0
- package/dist/runtime/browser.js +143 -0
- package/dist/runtime/node.d.ts +58 -0
- package/dist/runtime/node.js +122 -0
- package/dist/runtime.d.ts +29 -0
- package/dist/runtime.js +256 -0
- package/dist/session.d.ts +13 -0
- package/dist/session.js +15 -0
- package/dist/types.d.ts +117 -0
- package/dist/types.js +14 -0
- package/dist/version.d.ts +8 -0
- package/dist/version.js +8 -0
- package/package.json +66 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Omiologic
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
# @redact-secret/core
|
|
2
|
+
|
|
3
|
+
Deterministic secret detection and redaction for browser and server
|
|
4
|
+
JavaScript/TypeScript applications.
|
|
5
|
+
|
|
6
|
+
One typed API, two artifacts: the package's `exports` map selects the Node
|
|
7
|
+
N-API addon on Node.js and the browser WebAssembly build everywhere else
|
|
8
|
+
(`decision-define-runtime-bindings`). Every built-in detector runs in the Rust
|
|
9
|
+
core, so both runtimes see the same findings for the same input.
|
|
10
|
+
|
|
11
|
+
## Install
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
npm install @redact-secret/core
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
These registry instructions apply after an approved release is published.
|
|
18
|
+
Node.js 20, 22, and 24 are supported, on glibc Linux, macOS, and Windows
|
|
19
|
+
(x64 and arm64). npm does not ship a musl/Alpine addon. Browser applications
|
|
20
|
+
need ES2022 and WebAssembly support. The package is ESM only.
|
|
21
|
+
|
|
22
|
+
See the [JavaScript guide](https://github.com/redact-secret/redact-secret/blob/main/docs/guides/javascript.md)
|
|
23
|
+
for browser asset loading and troubleshooting.
|
|
24
|
+
|
|
25
|
+
## Initialize once, then work synchronously
|
|
26
|
+
|
|
27
|
+
Every runtime requires one successful `await initialize()` before any
|
|
28
|
+
synchronous operation. On Node the underlying setup has nothing to await, but
|
|
29
|
+
the call stays part of the contract so the usage model does not vary by
|
|
30
|
+
runtime. It is idempotent, so any number of call sites may await it; a failed
|
|
31
|
+
attempt is not cached and may be retried.
|
|
32
|
+
|
|
33
|
+
```ts
|
|
34
|
+
import { initialize, scanAndRedact } from "@redact-secret/core";
|
|
35
|
+
|
|
36
|
+
await initialize();
|
|
37
|
+
|
|
38
|
+
const { text, findings } = scanAndRedact("API_KEY=SYNTHETIC_REVOKED_VALUE");
|
|
39
|
+
|
|
40
|
+
if (findings.some((finding) => finding.action === "block")) {
|
|
41
|
+
throw new Error("Blocked sensitive input");
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
console.log(text);
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Calling a synchronous operation first throws `SecretScanError` with code
|
|
48
|
+
`NOT_INITIALIZED`, without inspecting the input.
|
|
49
|
+
|
|
50
|
+
## Scan, redact, or both
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
import { initialize, redact, scan } from "@redact-secret/core";
|
|
54
|
+
|
|
55
|
+
await initialize();
|
|
56
|
+
|
|
57
|
+
const input = "API_KEY=SYNTHETIC_REVOKED_VALUE";
|
|
58
|
+
const findings = scan(input);
|
|
59
|
+
const redacted = redact(input, findings);
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
`scanAndRedact` does the same in one call, so the text and the findings cannot
|
|
63
|
+
disagree. `redact` expects the findings `scan` returned for that same input.
|
|
64
|
+
|
|
65
|
+
Every finding is frozen and carries only safe metadata — id, type, detector,
|
|
66
|
+
confidence, action, and a range. It never carries the matched value.
|
|
67
|
+
|
|
68
|
+
## Offsets are UTF-16 code units
|
|
69
|
+
|
|
70
|
+
`start` and `end` index the JavaScript string you passed in, so
|
|
71
|
+
`input.slice(start, end)` selects exactly the matched span. The exported
|
|
72
|
+
`RANGE_UNIT` states this, and the Rust core's UTF-8 byte offsets are converted
|
|
73
|
+
by each binding without changing the selected span.
|
|
74
|
+
|
|
75
|
+
```ts
|
|
76
|
+
import { initialize, RANGE_UNIT, scan } from "@redact-secret/core";
|
|
77
|
+
|
|
78
|
+
await initialize();
|
|
79
|
+
|
|
80
|
+
const input = "🔑 API_KEY=SYNTHETIC_REVOKED_VALUE";
|
|
81
|
+
const [finding] = scan(input);
|
|
82
|
+
|
|
83
|
+
if (finding !== undefined) {
|
|
84
|
+
console.log(RANGE_UNIT, finding.start, finding.end);
|
|
85
|
+
// Do not log input.slice(finding.start, finding.end): it is plaintext.
|
|
86
|
+
}
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
## Custom policy and placeholder formatter
|
|
90
|
+
|
|
91
|
+
The first stable extension surface is a policy and a placeholder formatter.
|
|
92
|
+
Both receive safe metadata only, never the input or a matched value. Custom
|
|
93
|
+
detector callbacks are not part of this API.
|
|
94
|
+
|
|
95
|
+
```ts
|
|
96
|
+
import {
|
|
97
|
+
initialize,
|
|
98
|
+
scanAndRedact,
|
|
99
|
+
typedPlaceholderFormatter,
|
|
100
|
+
} from "@redact-secret/core";
|
|
101
|
+
import type { SecretPolicy } from "@redact-secret/core";
|
|
102
|
+
|
|
103
|
+
await initialize();
|
|
104
|
+
|
|
105
|
+
const policy: SecretPolicy = {
|
|
106
|
+
evaluate: (finding) => (finding.confidence === "high" ? "block" : "warn"),
|
|
107
|
+
};
|
|
108
|
+
|
|
109
|
+
const result = scanAndRedact("api_key=SYNTHETIC_REVOKED_VALUE", {
|
|
110
|
+
policy,
|
|
111
|
+
placeholderFormatter: typedPlaceholderFormatter,
|
|
112
|
+
});
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Omit `policy` to use the built-in policy, which runs in Rust. Pass
|
|
116
|
+
`defaultPlaceholderFormatter` to name the built-in placeholder format
|
|
117
|
+
(`<SECRET_1>`, `<SECRET_2>`, ...) explicitly; `typedPlaceholderFormatter`
|
|
118
|
+
names the finding type instead (`<JWT_1>`).
|
|
119
|
+
|
|
120
|
+
## Incremental and stream availability
|
|
121
|
+
|
|
122
|
+
The current Node and WebAssembly artifacts do not build incremental sessions.
|
|
123
|
+
After initialization, `createIncrementalSanitizer`, `createNodeStreamSanitizer`,
|
|
124
|
+
and `createWebStreamSanitizer` fail with `INCREMENTAL_UNAVAILABLE` when using
|
|
125
|
+
these artifacts. Whole-input operations remain supported.
|
|
126
|
+
|
|
127
|
+
The session types and stream subpaths are exported contracts, not evidence of
|
|
128
|
+
runtime support. Adapter constructors accept a supplied session, but the
|
|
129
|
+
package cannot create one with either current artifact. Use bounded whole-input
|
|
130
|
+
scanning, or use Python, Rust, or CLI streaming. Do not scan chunks independently:
|
|
131
|
+
a credential may cross a chunk boundary.
|
|
132
|
+
|
|
133
|
+
See the [streaming guide](https://github.com/redact-secret/redact-secret/blob/main/docs/guides/streaming.md).
|
|
134
|
+
|
|
135
|
+
## Errors
|
|
136
|
+
|
|
137
|
+
Every failure is a `SecretScanError` carrying nothing but a fixed `code` and
|
|
138
|
+
its fixed message — never the input, a matched value, a placeholder, or a
|
|
139
|
+
failing callback's own message.
|
|
140
|
+
|
|
141
|
+
```ts
|
|
142
|
+
import { initialize, scan, SecretScanError } from "@redact-secret/core";
|
|
143
|
+
import type { SecretScanErrorCode } from "@redact-secret/core";
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
await initialize();
|
|
147
|
+
scan("API_KEY=SYNTHETIC_REVOKED_VALUE");
|
|
148
|
+
} catch (error) {
|
|
149
|
+
if (error instanceof SecretScanError) {
|
|
150
|
+
const code: SecretScanErrorCode = error.code;
|
|
151
|
+
console.error(code);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
`NOT_INITIALIZED` and `INITIALIZATION_FAILED` come from the binding layer,
|
|
157
|
+
`INVALID_CHUNK` and `INVALID_UTF8` from the stream adapters,
|
|
158
|
+
`UNPAIRED_SURROGATE` from this package's own input check — a JavaScript
|
|
159
|
+
string may contain a lone UTF-16 surrogate, which has no UTF-8
|
|
160
|
+
representation, so `scan`, `redact`, `scanAndRedact`, and an incremental
|
|
161
|
+
sanitizer's `append` all reject one with this fixed code before it reaches
|
|
162
|
+
either binding, identically on Node.js and in the browser — and
|
|
163
|
+
`INCREMENTAL_UNAVAILABLE` from both current runtime adapters. Core failures
|
|
164
|
+
are mapped to the same fixed error vocabulary.
|
|
165
|
+
|
|
166
|
+
## Public API
|
|
167
|
+
|
|
168
|
+
Runtime values: `initialize`, `scan`, `redact`, `scanAndRedact`,
|
|
169
|
+
`createIncrementalSanitizer`, `defaultPlaceholderFormatter`,
|
|
170
|
+
`typedPlaceholderFormatter`, `SecretScanError`, `RANGE_UNIT`, `VERSION`.
|
|
171
|
+
|
|
172
|
+
Types: `DetectedSecretFinding`, `SecretFinding`, `SecretAction`,
|
|
173
|
+
`SecretConfidence`, `SecretPolicy`, `PolicyContext`, `PlaceholderFormatter`,
|
|
174
|
+
`PlaceholderContext`, `ScanOptions`, `RedactOptions`, `ScanAndRedactOptions`,
|
|
175
|
+
`ScanResult`, `IncrementalSanitizer`, `IncrementalSanitizerOptions`,
|
|
176
|
+
`IncrementalSanitizerResult`, `IncrementalSanitizerState`,
|
|
177
|
+
`IncrementalLimits`, `IncrementalSecretPolicy`, `IncrementalPolicyContext`,
|
|
178
|
+
`RangeUnit`, `SecretScanErrorCode`.
|
|
179
|
+
|
|
180
|
+
Stream subpaths: `@redact-secret/core/node-stream` exports
|
|
181
|
+
`createNodeStreamSanitizer`, `NodeStreamSanitizer`, and `SecretScanError`;
|
|
182
|
+
`@redact-secret/core/web-stream` exports `createWebStreamSanitizer`,
|
|
183
|
+
`WebStreamSanitizer`, and `SecretScanError`.
|
|
184
|
+
|
|
185
|
+
The root export and those two stream subpaths are the executable public API.
|
|
186
|
+
`@redact-secret/core/package.json` also exposes package metadata. Internal
|
|
187
|
+
modules are unreachable through the `exports` map. `VERSION` is
|
|
188
|
+
the shared product version; the Rust crate, this package, the Python package,
|
|
189
|
+
and the CLI are released in lockstep
|
|
190
|
+
(`decision-release-bindings-in-lockstep`), and `initialize()` refuses an
|
|
191
|
+
artifact that reports a different one.
|
|
192
|
+
|
|
193
|
+
## Security
|
|
194
|
+
|
|
195
|
+
Client-side scanning is preventive UX; server-side scanning is the
|
|
196
|
+
authoritative enforcement boundary. See
|
|
197
|
+
[SECURITY.md](https://github.com/redact-secret/redact-secret/blob/main/SECURITY.md)
|
|
198
|
+
for the security model and private vulnerability reporting.
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
[MIT](https://github.com/redact-secret/redact-secret/blob/main/LICENSE)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Node.js stream adapter: a byte-to-byte `Transform` over one incremental
|
|
3
|
+
* session (`decision-define-runtime-bindings`).
|
|
4
|
+
*
|
|
5
|
+
* This module is reached only through the package's `./node-stream` subpath.
|
|
6
|
+
* It is the one published module that imports `node:stream`; the root export
|
|
7
|
+
* and `./web-stream` never resolve a Node-only module, so a browser bundle
|
|
8
|
+
* that uses them pulls none of this in.
|
|
9
|
+
*
|
|
10
|
+
* ```ts
|
|
11
|
+
* await initialize();
|
|
12
|
+
* await pipeline(source, createNodeStreamSanitizer({ limits }), destination);
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Backpressure, error propagation, and teardown are Node's own: the transform
|
|
16
|
+
* pushes into its readable side and lets the stream machinery stall the
|
|
17
|
+
* producer, and `_destroy` — which Node runs for `destroy()`, for a failed
|
|
18
|
+
* `pipeline`, and for a downstream error alike — aborts the session so the
|
|
19
|
+
* plaintext it was still deciding about is discarded rather than flushed.
|
|
20
|
+
*/
|
|
21
|
+
import { Transform } from "node:stream";
|
|
22
|
+
import type { TransformCallback } from "node:stream";
|
|
23
|
+
import type { IncrementalSanitizer, IncrementalSanitizerOptions, SecretFinding } from "../types.js";
|
|
24
|
+
/** A byte-to-byte Node transform backed by one incremental session. */
|
|
25
|
+
export declare class NodeStreamSanitizer extends Transform {
|
|
26
|
+
#private;
|
|
27
|
+
/**
|
|
28
|
+
* Wraps `session`, which this transform owns: it is finalized when the
|
|
29
|
+
* stream ends normally and aborted on every other exit.
|
|
30
|
+
*/
|
|
31
|
+
constructor(session: IncrementalSanitizer);
|
|
32
|
+
/**
|
|
33
|
+
* Every finding the session has finalized so far, frozen, with absolute
|
|
34
|
+
* UTF-16 offsets into the logical whole-stream input. It stays empty when
|
|
35
|
+
* the stream is destroyed before anything settles.
|
|
36
|
+
*/
|
|
37
|
+
get findings(): readonly SecretFinding[];
|
|
38
|
+
_transform(chunk: Uint8Array, _encoding: BufferEncoding, callback: TransformCallback): void;
|
|
39
|
+
_flush(callback: TransformCallback): void;
|
|
40
|
+
_destroy(error: Error | null, callback: (error?: Error | null) => void): void;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Opens one incremental session and wraps it in a Node transform.
|
|
44
|
+
*
|
|
45
|
+
* Requires a successful `await initialize()`, like every other synchronous
|
|
46
|
+
* operation in this package; it throws `NOT_INITIALIZED` otherwise.
|
|
47
|
+
*/
|
|
48
|
+
export declare function createNodeStreamSanitizer(options: IncrementalSanitizerOptions): NodeStreamSanitizer;
|
|
49
|
+
export { SecretScanError } from "../errors.js";
|
|
50
|
+
export type { SecretScanErrorCode } from "../errors.js";
|
|
51
|
+
export type { IncrementalLimits, IncrementalSanitizer, IncrementalSanitizerOptions, IncrementalSecretPolicy, SecretFinding, } from "../types.js";
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Node.js stream adapter: a byte-to-byte `Transform` over one incremental
|
|
3
|
+
* session (`decision-define-runtime-bindings`).
|
|
4
|
+
*
|
|
5
|
+
* This module is reached only through the package's `./node-stream` subpath.
|
|
6
|
+
* It is the one published module that imports `node:stream`; the root export
|
|
7
|
+
* and `./web-stream` never resolve a Node-only module, so a browser bundle
|
|
8
|
+
* that uses them pulls none of this in.
|
|
9
|
+
*
|
|
10
|
+
* ```ts
|
|
11
|
+
* await initialize();
|
|
12
|
+
* await pipeline(source, createNodeStreamSanitizer({ limits }), destination);
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Backpressure, error propagation, and teardown are Node's own: the transform
|
|
16
|
+
* pushes into its readable side and lets the stream machinery stall the
|
|
17
|
+
* producer, and `_destroy` — which Node runs for `destroy()`, for a failed
|
|
18
|
+
* `pipeline`, and for a downstream error alike — aborts the session so the
|
|
19
|
+
* plaintext it was still deciding about is discarded rather than flushed.
|
|
20
|
+
*/
|
|
21
|
+
import { Transform } from "node:stream";
|
|
22
|
+
import { runtime } from "../session.js";
|
|
23
|
+
import { createStreamSanitizerRuntime } from "./shared.js";
|
|
24
|
+
/** A byte-to-byte Node transform backed by one incremental session. */
|
|
25
|
+
export class NodeStreamSanitizer extends Transform {
|
|
26
|
+
#runtime;
|
|
27
|
+
/**
|
|
28
|
+
* Wraps `session`, which this transform owns: it is finalized when the
|
|
29
|
+
* stream ends normally and aborted on every other exit.
|
|
30
|
+
*/
|
|
31
|
+
constructor(session) {
|
|
32
|
+
super();
|
|
33
|
+
this.#runtime = createStreamSanitizerRuntime(session);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Every finding the session has finalized so far, frozen, with absolute
|
|
37
|
+
* UTF-16 offsets into the logical whole-stream input. It stays empty when
|
|
38
|
+
* the stream is destroyed before anything settles.
|
|
39
|
+
*/
|
|
40
|
+
get findings() {
|
|
41
|
+
return this.#runtime.findings;
|
|
42
|
+
}
|
|
43
|
+
_transform(chunk, _encoding, callback) {
|
|
44
|
+
try {
|
|
45
|
+
const { text } = this.#runtime.append(chunk);
|
|
46
|
+
if (text.length > 0)
|
|
47
|
+
this.push(text, "utf8");
|
|
48
|
+
callback();
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
callback(error);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
_flush(callback) {
|
|
55
|
+
try {
|
|
56
|
+
const { text } = this.#runtime.finalize();
|
|
57
|
+
if (text.length > 0)
|
|
58
|
+
this.push(text, "utf8");
|
|
59
|
+
callback();
|
|
60
|
+
}
|
|
61
|
+
catch (error) {
|
|
62
|
+
callback(error);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
_destroy(error, callback) {
|
|
66
|
+
this.#runtime.abort();
|
|
67
|
+
callback(error);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Opens one incremental session and wraps it in a Node transform.
|
|
72
|
+
*
|
|
73
|
+
* Requires a successful `await initialize()`, like every other synchronous
|
|
74
|
+
* operation in this package; it throws `NOT_INITIALIZED` otherwise.
|
|
75
|
+
*/
|
|
76
|
+
export function createNodeStreamSanitizer(options) {
|
|
77
|
+
return new NodeStreamSanitizer(runtime.createIncrementalSanitizer(options));
|
|
78
|
+
}
|
|
79
|
+
export { SecretScanError } from "../errors.js";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runtime-neutral half of both stream adapters: one incremental session,
|
|
3
|
+
* one stateful UTF-8 decoder, and the finding list they produce together.
|
|
4
|
+
*
|
|
5
|
+
* Nothing here imports a `node:` module or touches a host global beyond
|
|
6
|
+
* `TextDecoder`, which both Node.js and browsers provide, so
|
|
7
|
+
* `web-stream.ts` can build on it unchanged (`decision-define-runtime-bindings`).
|
|
8
|
+
*
|
|
9
|
+
* The decoder is fatal and stateful: a chunk may end in the middle of a
|
|
10
|
+
* multibyte character, and the bytes that character continues into arrive in
|
|
11
|
+
* the next chunk. Malformed input is a hard failure rather than a replacement
|
|
12
|
+
* character, because a silent `U+FFFD` would change the text a detector sees.
|
|
13
|
+
*
|
|
14
|
+
* Retained plaintext — the tail the session is still deciding about — never
|
|
15
|
+
* leaves this module. Every termination that is not a normal finalization
|
|
16
|
+
* aborts the session, which discards it.
|
|
17
|
+
*/
|
|
18
|
+
import type { IncrementalSanitizer, IncrementalSanitizerResult, SecretFinding } from "../types.js";
|
|
19
|
+
/** One incremental session wrapped as a byte sink. */
|
|
20
|
+
export interface StreamSanitizerRuntime {
|
|
21
|
+
/**
|
|
22
|
+
* Every finding whose detection window has closed so far, frozen.
|
|
23
|
+
*
|
|
24
|
+
* Offsets are absolute UTF-16 code units into the logical whole-stream
|
|
25
|
+
* input, exactly as the session reported them; this module never rebases
|
|
26
|
+
* them onto a chunk or onto the sanitized output.
|
|
27
|
+
*/
|
|
28
|
+
readonly findings: readonly SecretFinding[];
|
|
29
|
+
append(chunk: Uint8Array): IncrementalSanitizerResult;
|
|
30
|
+
finalize(): IncrementalSanitizerResult;
|
|
31
|
+
abort(): void;
|
|
32
|
+
}
|
|
33
|
+
export declare function createStreamSanitizerRuntime(session: IncrementalSanitizer): StreamSanitizerRuntime;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runtime-neutral half of both stream adapters: one incremental session,
|
|
3
|
+
* one stateful UTF-8 decoder, and the finding list they produce together.
|
|
4
|
+
*
|
|
5
|
+
* Nothing here imports a `node:` module or touches a host global beyond
|
|
6
|
+
* `TextDecoder`, which both Node.js and browsers provide, so
|
|
7
|
+
* `web-stream.ts` can build on it unchanged (`decision-define-runtime-bindings`).
|
|
8
|
+
*
|
|
9
|
+
* The decoder is fatal and stateful: a chunk may end in the middle of a
|
|
10
|
+
* multibyte character, and the bytes that character continues into arrive in
|
|
11
|
+
* the next chunk. Malformed input is a hard failure rather than a replacement
|
|
12
|
+
* character, because a silent `U+FFFD` would change the text a detector sees.
|
|
13
|
+
*
|
|
14
|
+
* Retained plaintext — the tail the session is still deciding about — never
|
|
15
|
+
* leaves this module. Every termination that is not a normal finalization
|
|
16
|
+
* aborts the session, which discards it.
|
|
17
|
+
*/
|
|
18
|
+
import { SecretScanError } from "../errors.js";
|
|
19
|
+
export function createStreamSanitizerRuntime(session) {
|
|
20
|
+
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
21
|
+
const findings = [];
|
|
22
|
+
/** Idempotent, and safe after the session has already left `accepting`. */
|
|
23
|
+
function abort() {
|
|
24
|
+
if (session.state === "accepting")
|
|
25
|
+
session.abort();
|
|
26
|
+
}
|
|
27
|
+
function decode(chunk, stream = false) {
|
|
28
|
+
try {
|
|
29
|
+
return decoder.decode(chunk, { stream });
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
abort();
|
|
33
|
+
throw new SecretScanError("INVALID_UTF8");
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function append(chunk) {
|
|
37
|
+
if (!(chunk instanceof Uint8Array)) {
|
|
38
|
+
abort();
|
|
39
|
+
throw new SecretScanError("INVALID_CHUNK");
|
|
40
|
+
}
|
|
41
|
+
const result = session.append(decode(chunk, true));
|
|
42
|
+
findings.push(...result.findings);
|
|
43
|
+
return result;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Flushes the decoder, then the session.
|
|
47
|
+
*
|
|
48
|
+
* A stream that ends mid-character fails here: the final `decode()` with
|
|
49
|
+
* `stream: false` rejects the truncated sequence rather than emitting a
|
|
50
|
+
* replacement character.
|
|
51
|
+
*/
|
|
52
|
+
function finalize() {
|
|
53
|
+
const decoded = decode();
|
|
54
|
+
let decodedResult;
|
|
55
|
+
let finalResult;
|
|
56
|
+
try {
|
|
57
|
+
decodedResult = session.append(decoded);
|
|
58
|
+
finalResult = session.finalize();
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
abort();
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
findings.push(...decodedResult.findings, ...finalResult.findings);
|
|
65
|
+
return Object.freeze({
|
|
66
|
+
text: decodedResult.text + finalResult.text,
|
|
67
|
+
findings: Object.freeze([
|
|
68
|
+
...decodedResult.findings,
|
|
69
|
+
...finalResult.findings,
|
|
70
|
+
]),
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
return Object.freeze({
|
|
74
|
+
get findings() {
|
|
75
|
+
return Object.freeze([...findings]);
|
|
76
|
+
},
|
|
77
|
+
append,
|
|
78
|
+
finalize,
|
|
79
|
+
abort,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Web Streams adapter: a byte-to-string `TransformStream` over one
|
|
3
|
+
* incremental session (`decision-define-runtime-bindings`).
|
|
4
|
+
*
|
|
5
|
+
* This module is reached through the package's `./web-stream` subpath and
|
|
6
|
+
* imports nothing from `node:`, so a browser bundle resolves only it, the
|
|
7
|
+
* root module, and the WebAssembly artifact.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* await initialize();
|
|
11
|
+
* await response.body.pipeThrough(createWebStreamSanitizer({ limits }))
|
|
12
|
+
* .pipeTo(destination);
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Backpressure is the platform's: writes stall while the readable side is
|
|
16
|
+
* full and resume when a reader pulls. The readable and writable sides are
|
|
17
|
+
* wrapped so that a reader's `cancel()` and a writer's `abort()` — the two
|
|
18
|
+
* ways a Web stream ends early — abort the session first, discarding the
|
|
19
|
+
* plaintext it was still deciding about, before the underlying stream is torn
|
|
20
|
+
* down.
|
|
21
|
+
*/
|
|
22
|
+
import type { IncrementalSanitizer, IncrementalSanitizerOptions, SecretFinding } from "../types.js";
|
|
23
|
+
/** A byte-to-string Web transform backed by one incremental session. */
|
|
24
|
+
export declare class WebStreamSanitizer extends TransformStream<Uint8Array, string> {
|
|
25
|
+
#private;
|
|
26
|
+
/**
|
|
27
|
+
* Wraps `session`, which this transform owns: it is finalized when the
|
|
28
|
+
* writable side closes normally and aborted on every other exit.
|
|
29
|
+
*/
|
|
30
|
+
constructor(session: IncrementalSanitizer);
|
|
31
|
+
get readable(): ReadableStream<string>;
|
|
32
|
+
get writable(): WritableStream<Uint8Array>;
|
|
33
|
+
/**
|
|
34
|
+
* Every finding the session has finalized so far, frozen, with absolute
|
|
35
|
+
* UTF-16 offsets into the logical whole-stream input. It stays empty when
|
|
36
|
+
* the stream is cancelled or aborted before anything settles.
|
|
37
|
+
*/
|
|
38
|
+
get findings(): readonly SecretFinding[];
|
|
39
|
+
/**
|
|
40
|
+
* Discards retained plaintext before explicit early termination.
|
|
41
|
+
*
|
|
42
|
+
* Idempotent, and safe after the session has already ended: a later
|
|
43
|
+
* `close()` on the writable side then fails with `INVALID_STATE` rather
|
|
44
|
+
* than flushing anything.
|
|
45
|
+
*/
|
|
46
|
+
abort(): void;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Opens one incremental session and wraps it in a Web transform.
|
|
50
|
+
*
|
|
51
|
+
* Requires a successful `await initialize()`, like every other synchronous
|
|
52
|
+
* operation in this package; it throws `NOT_INITIALIZED` otherwise.
|
|
53
|
+
*/
|
|
54
|
+
export declare function createWebStreamSanitizer(options: IncrementalSanitizerOptions): WebStreamSanitizer;
|
|
55
|
+
export { SecretScanError } from "../errors.js";
|
|
56
|
+
export type { SecretScanErrorCode } from "../errors.js";
|
|
57
|
+
export type { IncrementalLimits, IncrementalSanitizer, IncrementalSanitizerOptions, IncrementalSecretPolicy, SecretFinding, } from "../types.js";
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Web Streams adapter: a byte-to-string `TransformStream` over one
|
|
3
|
+
* incremental session (`decision-define-runtime-bindings`).
|
|
4
|
+
*
|
|
5
|
+
* This module is reached through the package's `./web-stream` subpath and
|
|
6
|
+
* imports nothing from `node:`, so a browser bundle resolves only it, the
|
|
7
|
+
* root module, and the WebAssembly artifact.
|
|
8
|
+
*
|
|
9
|
+
* ```ts
|
|
10
|
+
* await initialize();
|
|
11
|
+
* await response.body.pipeThrough(createWebStreamSanitizer({ limits }))
|
|
12
|
+
* .pipeTo(destination);
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* Backpressure is the platform's: writes stall while the readable side is
|
|
16
|
+
* full and resume when a reader pulls. The readable and writable sides are
|
|
17
|
+
* wrapped so that a reader's `cancel()` and a writer's `abort()` — the two
|
|
18
|
+
* ways a Web stream ends early — abort the session first, discarding the
|
|
19
|
+
* plaintext it was still deciding about, before the underlying stream is torn
|
|
20
|
+
* down.
|
|
21
|
+
*/
|
|
22
|
+
import { runtime } from "../session.js";
|
|
23
|
+
import { createStreamSanitizerRuntime } from "./shared.js";
|
|
24
|
+
/** A byte-to-string Web transform backed by one incremental session. */
|
|
25
|
+
export class WebStreamSanitizer extends TransformStream {
|
|
26
|
+
#runtime;
|
|
27
|
+
#readable;
|
|
28
|
+
#writable;
|
|
29
|
+
/**
|
|
30
|
+
* Wraps `session`, which this transform owns: it is finalized when the
|
|
31
|
+
* writable side closes normally and aborted on every other exit.
|
|
32
|
+
*/
|
|
33
|
+
constructor(session) {
|
|
34
|
+
const sanitizer = createStreamSanitizerRuntime(session);
|
|
35
|
+
super({
|
|
36
|
+
transform(chunk, controller) {
|
|
37
|
+
const { text } = sanitizer.append(chunk);
|
|
38
|
+
if (text.length > 0)
|
|
39
|
+
controller.enqueue(text);
|
|
40
|
+
},
|
|
41
|
+
flush(controller) {
|
|
42
|
+
const { text } = sanitizer.finalize();
|
|
43
|
+
if (text.length > 0)
|
|
44
|
+
controller.enqueue(text);
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
this.#runtime = sanitizer;
|
|
48
|
+
const source = super.readable.getReader();
|
|
49
|
+
this.#readable = new ReadableStream({
|
|
50
|
+
async pull(controller) {
|
|
51
|
+
try {
|
|
52
|
+
const result = await source.read();
|
|
53
|
+
if (result.done)
|
|
54
|
+
controller.close();
|
|
55
|
+
else
|
|
56
|
+
controller.enqueue(result.value);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
controller.error(error);
|
|
60
|
+
}
|
|
61
|
+
},
|
|
62
|
+
async cancel(reason) {
|
|
63
|
+
sanitizer.abort();
|
|
64
|
+
await source.cancel(reason);
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
const sink = super.writable.getWriter();
|
|
68
|
+
this.#writable = new WritableStream({
|
|
69
|
+
write(chunk) {
|
|
70
|
+
return sink.write(chunk);
|
|
71
|
+
},
|
|
72
|
+
close() {
|
|
73
|
+
return sink.close();
|
|
74
|
+
},
|
|
75
|
+
async abort(reason) {
|
|
76
|
+
sanitizer.abort();
|
|
77
|
+
await sink.abort(reason);
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
get readable() {
|
|
82
|
+
return this.#readable;
|
|
83
|
+
}
|
|
84
|
+
get writable() {
|
|
85
|
+
return this.#writable;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Every finding the session has finalized so far, frozen, with absolute
|
|
89
|
+
* UTF-16 offsets into the logical whole-stream input. It stays empty when
|
|
90
|
+
* the stream is cancelled or aborted before anything settles.
|
|
91
|
+
*/
|
|
92
|
+
get findings() {
|
|
93
|
+
return this.#runtime.findings;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Discards retained plaintext before explicit early termination.
|
|
97
|
+
*
|
|
98
|
+
* Idempotent, and safe after the session has already ended: a later
|
|
99
|
+
* `close()` on the writable side then fails with `INVALID_STATE` rather
|
|
100
|
+
* than flushing anything.
|
|
101
|
+
*/
|
|
102
|
+
abort() {
|
|
103
|
+
this.#runtime.abort();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Opens one incremental session and wraps it in a Web transform.
|
|
108
|
+
*
|
|
109
|
+
* Requires a successful `await initialize()`, like every other synchronous
|
|
110
|
+
* operation in this package; it throws `NOT_INITIALIZED` otherwise.
|
|
111
|
+
*/
|
|
112
|
+
export function createWebStreamSanitizer(options) {
|
|
113
|
+
return new WebStreamSanitizer(runtime.createIncrementalSanitizer(options));
|
|
114
|
+
}
|
|
115
|
+
export { SecretScanError } from "../errors.js";
|
package/dist/errors.d.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The single sanitized error type this package throws.
|
|
3
|
+
*
|
|
4
|
+
* Every code and message below is fixed and input-free: no error carries the
|
|
5
|
+
* scanned input, a matched value, a placeholder, or a failing callback's own
|
|
6
|
+
* message (`decision-define-runtime-bindings`). Sixteen codes come from the
|
|
7
|
+
* Rust core; `NOT_INITIALIZED` and `INITIALIZATION_FAILED` are produced by the
|
|
8
|
+
* binding layer, `INVALID_CHUNK` and `INVALID_UTF8` by the stream adapters,
|
|
9
|
+
* `UNPAIRED_SURROGATE` by this package's own runtime-neutral input check, and
|
|
10
|
+
* `INCREMENTAL_UNAVAILABLE` by any binding that does not implement
|
|
11
|
+
* incremental sanitization — the browser binding always (`bindings/wasm` has
|
|
12
|
+
* no such export by design) and the Node binding until `bindings/node` gains
|
|
13
|
+
* one; and this package normalizes all of them into the same class so
|
|
14
|
+
* `instanceof SecretScanError` holds on every runtime and every subpath.
|
|
15
|
+
*
|
|
16
|
+
* `UNPAIRED_SURROGATE` resolves `B/F-10`
|
|
17
|
+
* (`docs/audits/deferred-quality-backlog.md`): a JavaScript string may hold a
|
|
18
|
+
* lone UTF-16 surrogate, which has no UTF-8 representation, so it cannot
|
|
19
|
+
* cross into the Rust core's `&str` without either a silent `U+FFFD`
|
|
20
|
+
* substitution (which would make `redact`'s output a transcoded copy, not the
|
|
21
|
+
* caller's input with only findings replaced) or a stable rejection. This
|
|
22
|
+
* package rejects, once, in `runtime.ts`'s `requireString`, before the string
|
|
23
|
+
* reaches either binding — so the error and this check are identical on the
|
|
24
|
+
* Node and browser runtimes by construction rather than by parallel
|
|
25
|
+
* implementation.
|
|
26
|
+
*/
|
|
27
|
+
export type SecretScanErrorCode = "INVALID_INPUT" | "INVALID_OPTIONS" | "INVALID_DETECTOR" | "DETECTOR_FAILURE" | "INVALID_CANDIDATE" | "POLICY_FAILURE" | "INVALID_POLICY_ACTION" | "INVALID_FINDINGS" | "PLACEHOLDER_FAILURE" | "INVALID_PLACEHOLDER" | "INVALID_LIMITS" | "INPUT_LIMIT_EXCEEDED" | "BUFFER_LIMIT_EXCEEDED" | "TOKEN_LIMIT_EXCEEDED" | "MULTILINE_LIMIT_EXCEEDED" | "INVALID_STATE" | "NOT_INITIALIZED" | "INITIALIZATION_FAILED" | "INVALID_CHUNK" | "INVALID_UTF8" | "UNPAIRED_SURROGATE" | "INCREMENTAL_UNAVAILABLE";
|
|
28
|
+
/** A sanitized failure. It carries nothing but its fixed code and message. */
|
|
29
|
+
export declare class SecretScanError extends Error {
|
|
30
|
+
readonly code: SecretScanErrorCode;
|
|
31
|
+
constructor(code: SecretScanErrorCode);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Rewrites whatever a binding threw as a {@link SecretScanError}.
|
|
35
|
+
*
|
|
36
|
+
* The Node addon throws an N-API error whose `code` is the core's fixed code;
|
|
37
|
+
* the WebAssembly binding throws a `js_sys::Error` with the same property. A
|
|
38
|
+
* value that carries no recognized code is replaced by `fallback` rather than
|
|
39
|
+
* surfaced, so a host-specific message can never reach a caller.
|
|
40
|
+
*/
|
|
41
|
+
export declare function toSecretScanError(thrown: unknown, fallback: SecretScanErrorCode): SecretScanError;
|