@traffic-twin/sdk 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +171 -2
- package/dist/cjs/agent.d.ts +143 -0
- package/dist/cjs/agent.js +927 -0
- package/dist/cjs/capture/body.d.ts +11 -0
- package/dist/cjs/capture/body.js +73 -0
- package/dist/cjs/capture/common.d.ts +75 -0
- package/dist/cjs/capture/common.js +180 -0
- package/dist/cjs/capture/context.d.ts +28 -0
- package/dist/cjs/capture/context.js +92 -0
- package/dist/cjs/capture/inbound.d.ts +12 -0
- package/dist/cjs/capture/inbound.js +195 -0
- package/dist/cjs/capture/outbound-http.d.ts +4 -0
- package/dist/cjs/capture/outbound-http.js +200 -0
- package/dist/cjs/capture/outbound-undici.d.ts +2 -0
- package/dist/cjs/capture/outbound-undici.js +242 -0
- package/dist/cjs/capture/wire.d.ts +27 -0
- package/dist/cjs/capture/wire.js +68 -0
- package/dist/cjs/export/event.d.ts +46 -0
- package/dist/cjs/export/event.js +91 -0
- package/dist/cjs/export/exporter.d.ts +50 -0
- package/dist/cjs/export/exporter.js +192 -0
- package/dist/cjs/export/pipeline.d.ts +20 -0
- package/dist/cjs/export/pipeline.js +40 -0
- package/dist/cjs/fleet/asn1.d.ts +44 -0
- package/dist/cjs/fleet/asn1.js +150 -0
- package/dist/cjs/fleet/client.d.ts +84 -0
- package/dist/cjs/fleet/client.js +216 -0
- package/dist/cjs/fleet/http.d.ts +17 -0
- package/dist/cjs/fleet/http.js +80 -0
- package/dist/cjs/fleet/store.d.ts +18 -0
- package/dist/cjs/fleet/store.js +115 -0
- package/dist/cjs/index.d.ts +14 -0
- package/dist/cjs/index.js +69 -0
- package/dist/cjs/log.d.ts +4 -0
- package/dist/cjs/log.js +30 -0
- package/dist/cjs/package.json +1 -0
- package/dist/cjs/register.d.ts +1 -0
- package/dist/cjs/register.js +18 -0
- package/dist/cjs/sanitizer/bodies.d.ts +55 -0
- package/dist/cjs/sanitizer/bodies.js +710 -0
- package/dist/cjs/sanitizer/formats.d.ts +26 -0
- package/dist/cjs/sanitizer/formats.js +110 -0
- package/dist/cjs/sanitizer/graphql.d.ts +23 -0
- package/dist/cjs/sanitizer/graphql.js +478 -0
- package/dist/cjs/sanitizer/hostpat.d.ts +15 -0
- package/dist/cjs/sanitizer/hostpat.js +129 -0
- package/dist/cjs/sanitizer/json.d.ts +60 -0
- package/dist/cjs/sanitizer/json.js +460 -0
- package/dist/cjs/sanitizer/policy.d.ts +224 -0
- package/dist/cjs/sanitizer/policy.js +820 -0
- package/dist/cjs/sanitizer/sanitizer.d.ts +149 -0
- package/dist/cjs/sanitizer/sanitizer.js +1473 -0
- package/dist/cjs/sanitizer/utf8.d.ts +22 -0
- package/dist/cjs/sanitizer/utf8.js +180 -0
- package/dist/cjs/sanitizer/vectors.d.ts +22 -0
- package/dist/cjs/sanitizer/vectors.js +130 -0
- package/dist/cjs/version.d.ts +2 -0
- package/dist/cjs/version.js +5 -0
- package/dist/cjs/worker/dir.d.ts +1 -0
- package/dist/cjs/worker/dir.js +8 -0
- package/dist/cjs/worker/pool.d.ts +47 -0
- package/dist/cjs/worker/pool.js +179 -0
- package/dist/cjs/worker/protocol.d.ts +37 -0
- package/dist/cjs/worker/protocol.js +3 -0
- package/dist/cjs/worker/sanitizer-worker.d.ts +1 -0
- package/dist/cjs/worker/sanitizer-worker.js +53 -0
- package/dist/esm/agent.d.ts +143 -0
- package/dist/esm/agent.js +921 -0
- package/dist/esm/capture/body.d.ts +11 -0
- package/dist/esm/capture/body.js +69 -0
- package/dist/esm/capture/common.d.ts +75 -0
- package/dist/esm/capture/common.js +166 -0
- package/dist/esm/capture/context.d.ts +28 -0
- package/dist/esm/capture/context.js +84 -0
- package/dist/esm/capture/inbound.d.ts +12 -0
- package/dist/esm/capture/inbound.js +187 -0
- package/dist/esm/capture/outbound-http.d.ts +4 -0
- package/dist/esm/capture/outbound-http.js +193 -0
- package/dist/esm/capture/outbound-undici.d.ts +2 -0
- package/dist/esm/capture/outbound-undici.js +236 -0
- package/dist/esm/capture/wire.d.ts +27 -0
- package/dist/esm/capture/wire.js +65 -0
- package/dist/esm/export/event.d.ts +46 -0
- package/dist/esm/export/event.js +85 -0
- package/dist/esm/export/exporter.d.ts +50 -0
- package/dist/esm/export/exporter.js +188 -0
- package/dist/esm/export/pipeline.d.ts +20 -0
- package/dist/esm/export/pipeline.js +37 -0
- package/dist/esm/fleet/asn1.d.ts +44 -0
- package/dist/esm/fleet/asn1.js +128 -0
- package/dist/esm/fleet/client.d.ts +84 -0
- package/dist/esm/fleet/client.js +204 -0
- package/dist/esm/fleet/http.d.ts +17 -0
- package/dist/esm/fleet/http.js +73 -0
- package/dist/esm/fleet/store.d.ts +18 -0
- package/dist/esm/fleet/store.js +110 -0
- package/dist/esm/index.d.ts +14 -0
- package/dist/esm/index.js +48 -0
- package/dist/esm/log.d.ts +4 -0
- package/dist/esm/log.js +25 -0
- package/dist/esm/package.json +1 -0
- package/dist/esm/register.d.ts +1 -0
- package/dist/esm/register.js +16 -0
- package/dist/esm/sanitizer/bodies.d.ts +55 -0
- package/dist/esm/sanitizer/bodies.js +694 -0
- package/dist/esm/sanitizer/formats.d.ts +26 -0
- package/dist/esm/sanitizer/formats.js +101 -0
- package/dist/esm/sanitizer/graphql.d.ts +23 -0
- package/dist/esm/sanitizer/graphql.js +472 -0
- package/dist/esm/sanitizer/hostpat.d.ts +15 -0
- package/dist/esm/sanitizer/hostpat.js +122 -0
- package/dist/esm/sanitizer/json.d.ts +60 -0
- package/dist/esm/sanitizer/json.js +447 -0
- package/dist/esm/sanitizer/policy.d.ts +224 -0
- package/dist/esm/sanitizer/policy.js +800 -0
- package/dist/esm/sanitizer/sanitizer.d.ts +149 -0
- package/dist/esm/sanitizer/sanitizer.js +1459 -0
- package/dist/esm/sanitizer/utf8.d.ts +22 -0
- package/dist/esm/sanitizer/utf8.js +166 -0
- package/dist/esm/sanitizer/vectors.d.ts +22 -0
- package/dist/esm/sanitizer/vectors.js +125 -0
- package/dist/esm/version.d.ts +2 -0
- package/dist/esm/version.js +2 -0
- package/dist/esm/worker/dir.d.ts +1 -0
- package/dist/esm/worker/dir.js +5 -0
- package/dist/esm/worker/pool.d.ts +47 -0
- package/dist/esm/worker/pool.js +175 -0
- package/dist/esm/worker/protocol.d.ts +37 -0
- package/dist/esm/worker/protocol.js +2 -0
- package/dist/esm/worker/sanitizer-worker.d.ts +1 -0
- package/dist/esm/worker/sanitizer-worker.js +51 -0
- package/package.json +75 -4
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Decodes strictly; returns null when the bytes are not valid UTF-8. */
|
|
2
|
+
export declare function decodeUtf8(b: Uint8Array): string | null;
|
|
3
|
+
/** True when the JS string has no lone surrogates (so it has a UTF-8 encoding). */
|
|
4
|
+
export declare function isWellFormed(s: string): boolean;
|
|
5
|
+
/** UTF-8 length of a well-formed string. */
|
|
6
|
+
export declare function utf8Length(s: string): number;
|
|
7
|
+
export declare function utf8(s: string): Buffer;
|
|
8
|
+
/** ASCII-only lowercase (never touches non-ASCII characters). */
|
|
9
|
+
export declare function asciiLower(s: string): string;
|
|
10
|
+
/** ASCII-only uppercase. */
|
|
11
|
+
export declare function asciiUpper(s: string): string;
|
|
12
|
+
/** Lowercase ASCII bytes. */
|
|
13
|
+
export declare function asciiLowerBytes(b: Buffer): Buffer;
|
|
14
|
+
/** Strips OWS (space and horizontal tab) on both sides. */
|
|
15
|
+
export declare function trimOws(b: Buffer): Buffer;
|
|
16
|
+
export declare function trimOwsStr(s: string): string;
|
|
17
|
+
/** Splits bytes on a single byte value (keeps empty parts). */
|
|
18
|
+
export declare function splitBytes(b: Buffer, sep: number): Buffer[];
|
|
19
|
+
/** Percent-decoding of sanitizer.md 4.1 (optionally "+" -> space first, 8.3). */
|
|
20
|
+
export declare function pctDecode(b: Buffer, plusIsSpace: boolean): Buffer;
|
|
21
|
+
/** Byte-order comparison of two strings by their UTF-8 encoding (code point order). */
|
|
22
|
+
export declare function cmpUtf8(a: string, b: string): number;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
// Strict UTF-8 (RFC 3629, sanitizer.md 0.2) helpers.
|
|
2
|
+
//
|
|
3
|
+
// The WHATWG decoder in fatal mode rejects overlong forms, encoded surrogates and code
|
|
4
|
+
// points above U+10FFFF. ignoreBOM keeps a leading U+FEFF as a character instead of
|
|
5
|
+
// silently removing it.
|
|
6
|
+
const strict = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
|
|
7
|
+
/** Decodes strictly; returns null when the bytes are not valid UTF-8. */
|
|
8
|
+
export function decodeUtf8(b) {
|
|
9
|
+
if (b.length === 0)
|
|
10
|
+
return '';
|
|
11
|
+
// Fast path: pure ASCII.
|
|
12
|
+
let ascii = true;
|
|
13
|
+
for (let i = 0; i < b.length; i++) {
|
|
14
|
+
if (b[i] >= 0x80) {
|
|
15
|
+
ascii = false;
|
|
16
|
+
break;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
if (ascii)
|
|
20
|
+
return Buffer.from(b.buffer, b.byteOffset, b.length).toString('latin1');
|
|
21
|
+
try {
|
|
22
|
+
return strict.decode(b);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const LONE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]/;
|
|
29
|
+
/** True when the JS string has no lone surrogates (so it has a UTF-8 encoding). */
|
|
30
|
+
export function isWellFormed(s) {
|
|
31
|
+
const f = s.isWellFormed;
|
|
32
|
+
if (typeof f === 'function')
|
|
33
|
+
return f.call(s);
|
|
34
|
+
return !LONE.test(s);
|
|
35
|
+
}
|
|
36
|
+
/** UTF-8 length of a well-formed string. */
|
|
37
|
+
export function utf8Length(s) {
|
|
38
|
+
return Buffer.byteLength(s, 'utf8');
|
|
39
|
+
}
|
|
40
|
+
export function utf8(s) {
|
|
41
|
+
return Buffer.from(s, 'utf8');
|
|
42
|
+
}
|
|
43
|
+
/** ASCII-only lowercase (never touches non-ASCII characters). */
|
|
44
|
+
export function asciiLower(s) {
|
|
45
|
+
let out = '';
|
|
46
|
+
let changed = false;
|
|
47
|
+
for (let i = 0; i < s.length; i++) {
|
|
48
|
+
const c = s.charCodeAt(i);
|
|
49
|
+
if (c >= 0x41 && c <= 0x5a) {
|
|
50
|
+
changed = true;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (!changed)
|
|
55
|
+
return s;
|
|
56
|
+
for (let i = 0; i < s.length; i++) {
|
|
57
|
+
const c = s.charCodeAt(i);
|
|
58
|
+
out += c >= 0x41 && c <= 0x5a ? String.fromCharCode(c + 32) : s[i];
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
/** ASCII-only uppercase. */
|
|
63
|
+
export function asciiUpper(s) {
|
|
64
|
+
let out = '';
|
|
65
|
+
for (let i = 0; i < s.length; i++) {
|
|
66
|
+
const c = s.charCodeAt(i);
|
|
67
|
+
out += c >= 0x61 && c <= 0x7a ? String.fromCharCode(c - 32) : s[i];
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
/** Lowercase ASCII bytes. */
|
|
72
|
+
export function asciiLowerBytes(b) {
|
|
73
|
+
const out = Buffer.from(b);
|
|
74
|
+
for (let i = 0; i < out.length; i++) {
|
|
75
|
+
const c = out[i];
|
|
76
|
+
if (c >= 0x41 && c <= 0x5a)
|
|
77
|
+
out[i] = c + 32;
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
/** Strips OWS (space and horizontal tab) on both sides. */
|
|
82
|
+
export function trimOws(b) {
|
|
83
|
+
let s = 0;
|
|
84
|
+
let e = b.length;
|
|
85
|
+
while (s < e && (b[s] === 0x20 || b[s] === 0x09))
|
|
86
|
+
s++;
|
|
87
|
+
while (e > s && (b[e - 1] === 0x20 || b[e - 1] === 0x09))
|
|
88
|
+
e--;
|
|
89
|
+
return s === 0 && e === b.length ? b : b.subarray(s, e);
|
|
90
|
+
}
|
|
91
|
+
export function trimOwsStr(s) {
|
|
92
|
+
let a = 0;
|
|
93
|
+
let e = s.length;
|
|
94
|
+
while (a < e && (s.charCodeAt(a) === 0x20 || s.charCodeAt(a) === 0x09))
|
|
95
|
+
a++;
|
|
96
|
+
while (e > a && (s.charCodeAt(e - 1) === 0x20 || s.charCodeAt(e - 1) === 0x09))
|
|
97
|
+
e--;
|
|
98
|
+
return s.slice(a, e);
|
|
99
|
+
}
|
|
100
|
+
/** Splits bytes on a single byte value (keeps empty parts). */
|
|
101
|
+
export function splitBytes(b, sep) {
|
|
102
|
+
const out = [];
|
|
103
|
+
let start = 0;
|
|
104
|
+
for (let i = 0; i < b.length; i++) {
|
|
105
|
+
if (b[i] === sep) {
|
|
106
|
+
out.push(b.subarray(start, i));
|
|
107
|
+
start = i + 1;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
out.push(b.subarray(start));
|
|
111
|
+
return out;
|
|
112
|
+
}
|
|
113
|
+
function hexVal(c) {
|
|
114
|
+
if (c >= 0x30 && c <= 0x39)
|
|
115
|
+
return c - 0x30;
|
|
116
|
+
if (c >= 0x41 && c <= 0x46)
|
|
117
|
+
return c - 0x37;
|
|
118
|
+
if (c >= 0x61 && c <= 0x66)
|
|
119
|
+
return c - 0x57;
|
|
120
|
+
return -1;
|
|
121
|
+
}
|
|
122
|
+
/** Percent-decoding of sanitizer.md 4.1 (optionally "+" -> space first, 8.3). */
|
|
123
|
+
export function pctDecode(b, plusIsSpace) {
|
|
124
|
+
let needs = false;
|
|
125
|
+
for (let i = 0; i < b.length; i++) {
|
|
126
|
+
if (b[i] === 0x25 || (plusIsSpace && b[i] === 0x2b)) {
|
|
127
|
+
needs = true;
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
if (!needs)
|
|
132
|
+
return b;
|
|
133
|
+
const out = Buffer.allocUnsafe(b.length);
|
|
134
|
+
let o = 0;
|
|
135
|
+
for (let i = 0; i < b.length; i++) {
|
|
136
|
+
const c = b[i];
|
|
137
|
+
if (c === 0x25 && i + 2 < b.length) {
|
|
138
|
+
const h = hexVal(b[i + 1]);
|
|
139
|
+
const l = hexVal(b[i + 2]);
|
|
140
|
+
if (h >= 0 && l >= 0) {
|
|
141
|
+
out[o++] = h * 16 + l;
|
|
142
|
+
i += 2;
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
out[o++] = plusIsSpace && c === 0x2b ? 0x20 : c;
|
|
147
|
+
}
|
|
148
|
+
return out.subarray(0, o);
|
|
149
|
+
}
|
|
150
|
+
/** Byte-order comparison of two strings by their UTF-8 encoding (code point order). */
|
|
151
|
+
export function cmpUtf8(a, b) {
|
|
152
|
+
const n = Math.min(a.length, b.length);
|
|
153
|
+
for (let i = 0; i < n; i++) {
|
|
154
|
+
let x = a.charCodeAt(i);
|
|
155
|
+
let y = b.charCodeAt(i);
|
|
156
|
+
if (x === y)
|
|
157
|
+
continue;
|
|
158
|
+
// Surrogates (supplementary code points) sort after U+E000..U+FFFF in UTF-8.
|
|
159
|
+
if (x >= 0xd800 && x <= 0xdfff)
|
|
160
|
+
x += 0x10000;
|
|
161
|
+
if (y >= 0xd800 && y <= 0xdfff)
|
|
162
|
+
y += 0x10000;
|
|
163
|
+
return x < y ? -1 : 1;
|
|
164
|
+
}
|
|
165
|
+
return a.length === b.length ? 0 : a.length < b.length ? -1 : 1;
|
|
166
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type Plain } from './json.js';
|
|
2
|
+
import { type Exchange } from './sanitizer.js';
|
|
3
|
+
export type VectorResult = {
|
|
4
|
+
outcome: 'event';
|
|
5
|
+
canonical: string;
|
|
6
|
+
} | {
|
|
7
|
+
outcome: 'not_sampled';
|
|
8
|
+
sampled_by: string;
|
|
9
|
+
} | {
|
|
10
|
+
outcome: 'refused';
|
|
11
|
+
code: string;
|
|
12
|
+
};
|
|
13
|
+
type Obj = {
|
|
14
|
+
[k: string]: Plain;
|
|
15
|
+
};
|
|
16
|
+
/** Converts the `input` object of a vector into a sanitizer exchange. */
|
|
17
|
+
export declare function vectorExchange(inp: Obj): Exchange;
|
|
18
|
+
/** Parses a vector file (bytes) with the lexeme-preserving parser. */
|
|
19
|
+
export declare function parseVector(bytes: Uint8Array): Obj;
|
|
20
|
+
/** Runs one parsed vector. `root` is the conformance/ directory (for policy_ref). */
|
|
21
|
+
export declare function runVector(vec: Obj, root: string): VectorResult;
|
|
22
|
+
export {};
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
// Executes one conformance vector (conformance/format.md, "Executing a vector").
|
|
2
|
+
// This is the test function every source must expose (sanitizer.md 16).
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { Lex, nodeToPlain, parseJson } from './json.js';
|
|
6
|
+
import { Policy, Refused, checkAcceptance, ed25519PublicKey, parseKeyRing, parseTime, selectKey, validateBodySchema, verifyEnvelope, } from './policy.js';
|
|
7
|
+
import { Sanitizer } from './sanitizer.js';
|
|
8
|
+
function decodeBytes(v) {
|
|
9
|
+
if (v === undefined || v === null)
|
|
10
|
+
return Buffer.alloc(0);
|
|
11
|
+
if (typeof v === 'string')
|
|
12
|
+
return Buffer.from(v, 'utf8');
|
|
13
|
+
if (typeof v === 'object' && !Array.isArray(v) && !(v instanceof Lex) && typeof v.b64 === 'string') {
|
|
14
|
+
return Buffer.from(v.b64, 'base64');
|
|
15
|
+
}
|
|
16
|
+
throw new Error('bad bytes value');
|
|
17
|
+
}
|
|
18
|
+
function num(v) {
|
|
19
|
+
return v instanceof Lex ? Number(v.lex) : undefined;
|
|
20
|
+
}
|
|
21
|
+
function headers(v) {
|
|
22
|
+
if (!Array.isArray(v))
|
|
23
|
+
return [];
|
|
24
|
+
return v.map((pair) => {
|
|
25
|
+
const p = pair;
|
|
26
|
+
return [p[0], decodeBytes(p[1])];
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
function total(v) {
|
|
30
|
+
if (v === 'unknown')
|
|
31
|
+
return 'unknown';
|
|
32
|
+
return num(v);
|
|
33
|
+
}
|
|
34
|
+
function bool(v) {
|
|
35
|
+
return typeof v === 'boolean' ? v : undefined;
|
|
36
|
+
}
|
|
37
|
+
/** Converts the `input` object of a vector into a sanitizer exchange. */
|
|
38
|
+
export function vectorExchange(inp) {
|
|
39
|
+
const ex = {
|
|
40
|
+
dir: inp.dir,
|
|
41
|
+
method: inp.method ?? '',
|
|
42
|
+
raw_path: decodeBytes(inp.raw_path),
|
|
43
|
+
raw_query: inp.raw_query === undefined ? undefined : decodeBytes(inp.raw_query),
|
|
44
|
+
req_headers: headers(inp.req_headers),
|
|
45
|
+
req_body: decodeBytes(inp.req_body),
|
|
46
|
+
req_body_total: total(inp.req_body_total),
|
|
47
|
+
req_body_consumed: bool(inp.req_body_consumed),
|
|
48
|
+
req_body_decoded: bool(inp.req_body_decoded),
|
|
49
|
+
resp_headers: headers(inp.resp_headers),
|
|
50
|
+
resp_body: decodeBytes(inp.resp_body),
|
|
51
|
+
resp_body_total: total(inp.resp_body_total),
|
|
52
|
+
resp_body_consumed: bool(inp.resp_body_consumed),
|
|
53
|
+
resp_body_decoded: bool(inp.resp_body_decoded),
|
|
54
|
+
};
|
|
55
|
+
if (typeof inp.service_name === 'string')
|
|
56
|
+
ex.service_name = inp.service_name;
|
|
57
|
+
if (typeof inp.url_base === 'string')
|
|
58
|
+
ex.url_base = inp.url_base;
|
|
59
|
+
if (typeof inp.framework_route === 'string')
|
|
60
|
+
ex.framework_route = inp.framework_route;
|
|
61
|
+
if (typeof inp.framework === 'string')
|
|
62
|
+
ex.framework = inp.framework;
|
|
63
|
+
if (typeof inp.trace_id === 'string')
|
|
64
|
+
ex.trace_id = inp.trace_id;
|
|
65
|
+
if (typeof inp.inherited === 'boolean')
|
|
66
|
+
ex.inherited = inp.inherited;
|
|
67
|
+
const st = num(inp.status);
|
|
68
|
+
if (st !== undefined)
|
|
69
|
+
ex.status = st;
|
|
70
|
+
if (typeof inp.net_error === 'string')
|
|
71
|
+
ex.net_error = inp.net_error;
|
|
72
|
+
return ex;
|
|
73
|
+
}
|
|
74
|
+
/** Parses a vector file (bytes) with the lexeme-preserving parser. */
|
|
75
|
+
export function parseVector(bytes) {
|
|
76
|
+
return nodeToPlain(parseJson(bytes), false);
|
|
77
|
+
}
|
|
78
|
+
/** Runs one parsed vector. `root` is the conformance/ directory (for policy_ref). */
|
|
79
|
+
export function runVector(vec, root) {
|
|
80
|
+
const now = typeof vec.now === 'string' ? parseTime(vec.now) : undefined;
|
|
81
|
+
let policy;
|
|
82
|
+
let sanitizer;
|
|
83
|
+
try {
|
|
84
|
+
let body;
|
|
85
|
+
if (vec.policy_envelope !== undefined) {
|
|
86
|
+
const keys = new Map();
|
|
87
|
+
const fk = vec.fleet_keys;
|
|
88
|
+
for (const kid of Object.keys(fk ?? {})) {
|
|
89
|
+
const k = ed25519PublicKey(fk[kid]);
|
|
90
|
+
if (k)
|
|
91
|
+
keys.set(kid, k);
|
|
92
|
+
}
|
|
93
|
+
body = verifyEnvelope(vec.policy_envelope, keys, true).body;
|
|
94
|
+
}
|
|
95
|
+
else if (typeof vec.policy_ref === 'string') {
|
|
96
|
+
body = validateBodySchema(nodeToPlain(parseJson(readFileSync(join(root, vec.policy_ref))), true));
|
|
97
|
+
}
|
|
98
|
+
else if (vec.policy !== undefined) {
|
|
99
|
+
body = validateBodySchema(vec.policy);
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
throw new Refused('no_policy');
|
|
103
|
+
}
|
|
104
|
+
policy = new Policy(body);
|
|
105
|
+
checkAcceptance(policy, {
|
|
106
|
+
group: typeof vec.source_group === 'string' ? vec.source_group : '',
|
|
107
|
+
now: now ?? NaN,
|
|
108
|
+
lastIssuedSeq: num(vec.last_issued_seq) ?? 0,
|
|
109
|
+
minIssuedSeq: num(vec.min_issued_seq) ?? 0,
|
|
110
|
+
});
|
|
111
|
+
const ring = parseKeyRing(typeof vec.hmac_key === 'string' ? vec.hmac_key : undefined);
|
|
112
|
+
sanitizer = new Sanitizer(policy, selectKey(ring, policy));
|
|
113
|
+
}
|
|
114
|
+
catch (e) {
|
|
115
|
+
if (e instanceof Refused)
|
|
116
|
+
return { outcome: 'refused', code: e.code };
|
|
117
|
+
throw e;
|
|
118
|
+
}
|
|
119
|
+
const r = sanitizer.run(vectorExchange(vec.input), now);
|
|
120
|
+
if (r.outcome === 'not_sampled')
|
|
121
|
+
return { outcome: 'not_sampled', sampled_by: r.sampledBy };
|
|
122
|
+
if (r.outcome === 'too_large')
|
|
123
|
+
return { outcome: 'refused', code: 'event_too_large' };
|
|
124
|
+
return { outcome: 'event', canonical: r.canonical };
|
|
125
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function workerDir(): string;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { Sanitizer } from '../sanitizer/sanitizer.js';
|
|
2
|
+
import type { Job } from './protocol.js';
|
|
3
|
+
interface Batch {
|
|
4
|
+
n: number;
|
|
5
|
+
bytes: number;
|
|
6
|
+
}
|
|
7
|
+
export interface PoolOptions {
|
|
8
|
+
size: number;
|
|
9
|
+
/** A batch came back: `stale` when it was posted before a discard(); recs/errs are then empty. */
|
|
10
|
+
onOut: (b: Batch & {
|
|
11
|
+
recs: string[];
|
|
12
|
+
errs: string[];
|
|
13
|
+
stale: boolean;
|
|
14
|
+
}) => void;
|
|
15
|
+
/** A worker died with `lost` batches in flight. */
|
|
16
|
+
onLost: (lost: Batch) => void;
|
|
17
|
+
}
|
|
18
|
+
export declare class SanitizerPool {
|
|
19
|
+
private readonly o;
|
|
20
|
+
private workers;
|
|
21
|
+
private epoch;
|
|
22
|
+
private seq;
|
|
23
|
+
private nextGen;
|
|
24
|
+
private readonly gens;
|
|
25
|
+
private closed;
|
|
26
|
+
constructor(o: PoolOptions);
|
|
27
|
+
/** True when at least one worker is up. */
|
|
28
|
+
get usable(): boolean;
|
|
29
|
+
/** Starts the first worker (idempotent). Never throws. */
|
|
30
|
+
warm(): void;
|
|
31
|
+
/** The policy of this sanitizer can be shipped to a worker. */
|
|
32
|
+
canHandle(san: Sanitizer): boolean;
|
|
33
|
+
private spawn;
|
|
34
|
+
private onMessage;
|
|
35
|
+
private pick;
|
|
36
|
+
/** Posts a batch of jobs. False (nothing posted) when no worker is ready. */
|
|
37
|
+
post(items: Array<{
|
|
38
|
+
san: Sanitizer;
|
|
39
|
+
job: Job;
|
|
40
|
+
}>, bytes: number, sourceId: string): boolean;
|
|
41
|
+
/** Results of everything posted so far are ignored when they arrive (kill switch, revocation). */
|
|
42
|
+
discard(): void;
|
|
43
|
+
/** Resolves when every batch posted before this call has come back. */
|
|
44
|
+
sync(): Promise<void>;
|
|
45
|
+
terminate(): void;
|
|
46
|
+
}
|
|
47
|
+
export {};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// Main-thread side of the sanitizer worker pool: a small number of worker threads that turn
|
|
2
|
+
// wire exchanges into OTLP records off the request path. Bounded and drop-on-pressure by
|
|
3
|
+
// construction (the agent charges every posted job to its pending limits and drops when
|
|
4
|
+
// they are exceeded); fail-open for the app: a dead worker just means "sanitize inline".
|
|
5
|
+
import { Worker } from 'node:worker_threads';
|
|
6
|
+
import { join } from 'node:path';
|
|
7
|
+
import { debug } from '../log.js';
|
|
8
|
+
import { workerDir } from './dir.js';
|
|
9
|
+
export class SanitizerPool {
|
|
10
|
+
o;
|
|
11
|
+
workers = [];
|
|
12
|
+
epoch = 0;
|
|
13
|
+
seq = 0;
|
|
14
|
+
nextGen = 1;
|
|
15
|
+
gens = new WeakMap();
|
|
16
|
+
closed = false;
|
|
17
|
+
constructor(o) {
|
|
18
|
+
this.o = o;
|
|
19
|
+
}
|
|
20
|
+
/** True when at least one worker is up. */
|
|
21
|
+
get usable() {
|
|
22
|
+
return !this.closed && this.workers.some((w) => w.ready && !w.dead);
|
|
23
|
+
}
|
|
24
|
+
/** Starts the first worker (idempotent). Never throws. */
|
|
25
|
+
warm() {
|
|
26
|
+
if (this.closed || this.workers.length > 0)
|
|
27
|
+
return;
|
|
28
|
+
this.spawn();
|
|
29
|
+
}
|
|
30
|
+
/** The policy of this sanitizer can be shipped to a worker. */
|
|
31
|
+
canHandle(san) {
|
|
32
|
+
if (!san.policy.bodyBytes)
|
|
33
|
+
return false;
|
|
34
|
+
const g = this.gens.get(san);
|
|
35
|
+
return g === undefined || g > this.nextGen - 12;
|
|
36
|
+
}
|
|
37
|
+
spawn() {
|
|
38
|
+
try {
|
|
39
|
+
const worker = new Worker(join(workerDir(), 'sanitizer-worker.js'), {
|
|
40
|
+
// The parent's --import/--require preloads must not run again in the worker.
|
|
41
|
+
execArgv: [],
|
|
42
|
+
env: { TT_DISABLED: '1' },
|
|
43
|
+
workerData: { ttSanitizer: true },
|
|
44
|
+
resourceLimits: { maxOldGenerationSizeMb: 192 },
|
|
45
|
+
});
|
|
46
|
+
const w = { worker, ready: false, dead: false, gens: new Set(), batches: [], syncs: new Map() };
|
|
47
|
+
worker.unref();
|
|
48
|
+
worker.on('message', (m) => this.onMessage(w, m));
|
|
49
|
+
const die = () => {
|
|
50
|
+
if (w.dead)
|
|
51
|
+
return;
|
|
52
|
+
w.dead = true;
|
|
53
|
+
w.ready = false;
|
|
54
|
+
let n = 0;
|
|
55
|
+
let bytes = 0;
|
|
56
|
+
for (const b of w.batches) {
|
|
57
|
+
n += b.n;
|
|
58
|
+
bytes += b.bytes;
|
|
59
|
+
}
|
|
60
|
+
w.batches = [];
|
|
61
|
+
for (const r of w.syncs.values())
|
|
62
|
+
r();
|
|
63
|
+
w.syncs.clear();
|
|
64
|
+
if (n > 0)
|
|
65
|
+
this.o.onLost({ n, bytes });
|
|
66
|
+
debug('sanitizer_worker', 'down');
|
|
67
|
+
};
|
|
68
|
+
worker.on('error', die);
|
|
69
|
+
worker.on('exit', die);
|
|
70
|
+
this.workers.push(w);
|
|
71
|
+
return w;
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
onMessage(w, m) {
|
|
78
|
+
if (m.t === 'ready') {
|
|
79
|
+
w.ready = true;
|
|
80
|
+
}
|
|
81
|
+
else if (m.t === 'out') {
|
|
82
|
+
const b = w.batches.shift();
|
|
83
|
+
if (!b)
|
|
84
|
+
return;
|
|
85
|
+
const stale = m.epoch !== this.epoch;
|
|
86
|
+
this.o.onOut({ ...b, recs: stale ? [] : m.recs, errs: stale ? [] : m.errs, stale });
|
|
87
|
+
}
|
|
88
|
+
else if (m.t === 'synced') {
|
|
89
|
+
const f = w.syncs.get(m.seq);
|
|
90
|
+
if (f) {
|
|
91
|
+
w.syncs.delete(m.seq);
|
|
92
|
+
f();
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
pick() {
|
|
97
|
+
let best = null;
|
|
98
|
+
for (const w of this.workers) {
|
|
99
|
+
if (!w.ready || w.dead)
|
|
100
|
+
continue;
|
|
101
|
+
if (!best || w.batches.length < best.batches.length)
|
|
102
|
+
best = w;
|
|
103
|
+
}
|
|
104
|
+
// Grow (up to the configured size) only when the least busy worker already has a backlog.
|
|
105
|
+
if ((!best || best.batches.length >= 2) && this.workers.filter((w) => !w.dead).length < this.o.size)
|
|
106
|
+
this.spawn();
|
|
107
|
+
return best;
|
|
108
|
+
}
|
|
109
|
+
/** Posts a batch of jobs. False (nothing posted) when no worker is ready. */
|
|
110
|
+
post(items, bytes, sourceId) {
|
|
111
|
+
const w = this.pick();
|
|
112
|
+
if (!w)
|
|
113
|
+
return false;
|
|
114
|
+
try {
|
|
115
|
+
const jobs = [];
|
|
116
|
+
for (const { san, job } of items) {
|
|
117
|
+
let g = this.gens.get(san);
|
|
118
|
+
if (g === undefined) {
|
|
119
|
+
g = this.nextGen++;
|
|
120
|
+
this.gens.set(san, g);
|
|
121
|
+
}
|
|
122
|
+
job.gen = g;
|
|
123
|
+
jobs.push(job);
|
|
124
|
+
if (!w.gens.has(g)) {
|
|
125
|
+
const p = san.policy;
|
|
126
|
+
const msg = {
|
|
127
|
+
t: 'policy',
|
|
128
|
+
gen: g,
|
|
129
|
+
body: Uint8Array.from(p.bodyBytes),
|
|
130
|
+
hash: p.hash,
|
|
131
|
+
kid: san.key.kid,
|
|
132
|
+
key: Uint8Array.from(san.key.key),
|
|
133
|
+
rateOverrideBp: san.options.rateOverrideBp ?? null,
|
|
134
|
+
};
|
|
135
|
+
w.worker.postMessage(msg);
|
|
136
|
+
w.gens.add(g);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
w.worker.postMessage({ t: 'run', epoch: this.epoch, sourceId, jobs });
|
|
140
|
+
w.batches.push({ n: jobs.length, bytes });
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/** Results of everything posted so far are ignored when they arrive (kill switch, revocation). */
|
|
148
|
+
discard() {
|
|
149
|
+
this.epoch++;
|
|
150
|
+
}
|
|
151
|
+
/** Resolves when every batch posted before this call has come back. */
|
|
152
|
+
sync() {
|
|
153
|
+
const ps = [];
|
|
154
|
+
for (const w of this.workers) {
|
|
155
|
+
if (w.dead || !w.ready || w.batches.length === 0)
|
|
156
|
+
continue;
|
|
157
|
+
const seq = ++this.seq;
|
|
158
|
+
ps.push(new Promise((r) => {
|
|
159
|
+
w.syncs.set(seq, r);
|
|
160
|
+
w.worker.postMessage({ t: 'sync', seq });
|
|
161
|
+
}));
|
|
162
|
+
}
|
|
163
|
+
return Promise.all(ps).then(() => undefined);
|
|
164
|
+
}
|
|
165
|
+
terminate() {
|
|
166
|
+
this.closed = true;
|
|
167
|
+
for (const w of this.workers) {
|
|
168
|
+
w.dead = true;
|
|
169
|
+
for (const r of w.syncs.values())
|
|
170
|
+
r();
|
|
171
|
+
void w.worker.terminate().catch(() => undefined);
|
|
172
|
+
}
|
|
173
|
+
this.workers = [];
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { WireExchange } from '../capture/wire.js';
|
|
2
|
+
import type { JobMeta } from '../export/pipeline.js';
|
|
3
|
+
import type { Decision } from '../sanitizer/sanitizer.js';
|
|
4
|
+
export interface Job {
|
|
5
|
+
gen: number;
|
|
6
|
+
decision: Decision;
|
|
7
|
+
meta: JobMeta;
|
|
8
|
+
wire: WireExchange;
|
|
9
|
+
}
|
|
10
|
+
export type ToWorker = {
|
|
11
|
+
t: 'policy';
|
|
12
|
+
gen: number;
|
|
13
|
+
body: Uint8Array;
|
|
14
|
+
hash: string;
|
|
15
|
+
kid: string;
|
|
16
|
+
key: Uint8Array;
|
|
17
|
+
rateOverrideBp: number | null;
|
|
18
|
+
} | {
|
|
19
|
+
t: 'run';
|
|
20
|
+
epoch: number;
|
|
21
|
+
sourceId: string;
|
|
22
|
+
jobs: Job[];
|
|
23
|
+
} | {
|
|
24
|
+
t: 'sync';
|
|
25
|
+
seq: number;
|
|
26
|
+
};
|
|
27
|
+
export type FromWorker = {
|
|
28
|
+
t: 'ready';
|
|
29
|
+
} | {
|
|
30
|
+
t: 'out';
|
|
31
|
+
epoch: number;
|
|
32
|
+
recs: string[];
|
|
33
|
+
errs: string[];
|
|
34
|
+
} | {
|
|
35
|
+
t: 'synced';
|
|
36
|
+
seq: number;
|
|
37
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Sanitizer worker thread: receives wire exchanges, returns ready-made OTLP logRecords.
|
|
2
|
+
// It runs exactly the code the main thread would run inline (export/pipeline.ts), so the
|
|
3
|
+
// events are byte-identical. It holds the HMAC key like the main thread does and has no
|
|
4
|
+
// network, file or Fleet access of its own: it only reads messages and posts results.
|
|
5
|
+
import { parentPort } from 'node:worker_threads';
|
|
6
|
+
import { sanitizeToRecord } from '../export/pipeline.js';
|
|
7
|
+
import { nodeToPlain, parseJson } from '../sanitizer/json.js';
|
|
8
|
+
import { Policy } from '../sanitizer/policy.js';
|
|
9
|
+
import { Sanitizer } from '../sanitizer/sanitizer.js';
|
|
10
|
+
const MAX_GENS = 16;
|
|
11
|
+
const sans = new Map();
|
|
12
|
+
const port = parentPort;
|
|
13
|
+
if (port) {
|
|
14
|
+
const send = (m) => port.postMessage(m);
|
|
15
|
+
port.on('message', (m) => {
|
|
16
|
+
try {
|
|
17
|
+
if (m.t === 'policy') {
|
|
18
|
+
const p = new Policy(nodeToPlain(parseJson(m.body), true));
|
|
19
|
+
p.hash = m.hash;
|
|
20
|
+
sans.set(m.gen, new Sanitizer(p, { kid: m.kid, key: Buffer.from(m.key) }, { rateOverrideBp: m.rateOverrideBp }));
|
|
21
|
+
while (sans.size > MAX_GENS)
|
|
22
|
+
sans.delete(sans.keys().next().value);
|
|
23
|
+
}
|
|
24
|
+
else if (m.t === 'run') {
|
|
25
|
+
const recs = [];
|
|
26
|
+
const errs = [];
|
|
27
|
+
const now = Date.now();
|
|
28
|
+
for (const j of m.jobs) {
|
|
29
|
+
const san = sans.get(j.gen);
|
|
30
|
+
const r = san ? sanitizeToRecord(san, j.wire, j.decision, j.meta, m.sourceId, now) : { err: 'sanitizer_error' };
|
|
31
|
+
if (r === null)
|
|
32
|
+
continue;
|
|
33
|
+
if ('record' in r)
|
|
34
|
+
recs.push(r.record);
|
|
35
|
+
else
|
|
36
|
+
errs.push(r.err);
|
|
37
|
+
}
|
|
38
|
+
send({ t: 'out', epoch: m.epoch, recs, errs });
|
|
39
|
+
}
|
|
40
|
+
else if (m.t === 'sync') {
|
|
41
|
+
send({ t: 'synced', seq: m.seq });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
// A malformed message must not kill the worker; the jobs it carried are lost.
|
|
46
|
+
if (m.t === 'run')
|
|
47
|
+
send({ t: 'out', epoch: m.epoch, recs: [], errs: m.jobs.map(() => 'sanitizer_error') });
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
send({ t: 'ready' });
|
|
51
|
+
}
|