@traffic-twin/sdk 0.0.0-stage → 0.1.0

package/dist/cjs/sanitizer/utf8.d.ts

@@ -0,0 +1,22 @@
+/** Decodes strictly; returns null when the bytes are not valid UTF-8. */
+export declare function decodeUtf8(b: Uint8Array): string | null;
+/** True when the JS string has no lone surrogates (so it has a UTF-8 encoding). */
+export declare function isWellFormed(s: string): boolean;
+/** UTF-8 length of a well-formed string. */
+export declare function utf8Length(s: string): number;
+export declare function utf8(s: string): Buffer;
+/** ASCII-only lowercase (never touches non-ASCII characters). */
+export declare function asciiLower(s: string): string;
+/** ASCII-only uppercase. */
+export declare function asciiUpper(s: string): string;
+/** Lowercase ASCII bytes. */
+export declare function asciiLowerBytes(b: Buffer): Buffer;
+/** Strips OWS (space and horizontal tab) on both sides. */
+export declare function trimOws(b: Buffer): Buffer;
+export declare function trimOwsStr(s: string): string;
+/** Splits bytes on a single byte value (keeps empty parts). */
+export declare function splitBytes(b: Buffer, sep: number): Buffer[];
+/** Percent-decoding of sanitizer.md 4.1 (optionally "+" -> space first, 8.3). */
+export declare function pctDecode(b: Buffer, plusIsSpace: boolean): Buffer;
+/** Byte-order comparison of two strings by their UTF-8 encoding (code point order). */
+export declare function cmpUtf8(a: string, b: string): number;

package/dist/cjs/sanitizer/utf8.js

@@ -0,0 +1,180 @@
+"use strict";
+// Strict UTF-8 (RFC 3629, sanitizer.md 0.2) helpers.
+//
+// The WHATWG decoder in fatal mode rejects overlong forms, encoded surrogates and code
+// points above U+10FFFF. ignoreBOM keeps a leading U+FEFF as a character instead of
+// silently removing it.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeUtf8 = decodeUtf8;
+exports.isWellFormed = isWellFormed;
+exports.utf8Length = utf8Length;
+exports.utf8 = utf8;
+exports.asciiLower = asciiLower;
+exports.asciiUpper = asciiUpper;
+exports.asciiLowerBytes = asciiLowerBytes;
+exports.trimOws = trimOws;
+exports.trimOwsStr = trimOwsStr;
+exports.splitBytes = splitBytes;
+exports.pctDecode = pctDecode;
+exports.cmpUtf8 = cmpUtf8;
+const strict = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true });
+/** Decodes strictly; returns null when the bytes are not valid UTF-8. */
+function decodeUtf8(b) {
+    if (b.length === 0)
+        return '';
+    // Fast path: pure ASCII.
+    let ascii = true;
+    for (let i = 0; i < b.length; i++) {
+        if (b[i] >= 0x80) {
+            ascii = false;
+            break;
+        }
+    }
+    if (ascii)
+        return Buffer.from(b.buffer, b.byteOffset, b.length).toString('latin1');
+    try {
+        return strict.decode(b);
+    }
+    catch {
+        return null;
+    }
+}
+const LONE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]/;
+/** True when the JS string has no lone surrogates (so it has a UTF-8 encoding). */
+function isWellFormed(s) {
+    const f = s.isWellFormed;
+    if (typeof f === 'function')
+        return f.call(s);
+    return !LONE.test(s);
+}
+/** UTF-8 length of a well-formed string. */
+function utf8Length(s) {
+    return Buffer.byteLength(s, 'utf8');
+}
+function utf8(s) {
+    return Buffer.from(s, 'utf8');
+}
+/** ASCII-only lowercase (never touches non-ASCII characters). */
+function asciiLower(s) {
+    let out = '';
+    let changed = false;
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        if (c >= 0x41 && c <= 0x5a) {
+            changed = true;
+            break;
+        }
+    }
+    if (!changed)
+        return s;
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        out += c >= 0x41 && c <= 0x5a ? String.fromCharCode(c + 32) : s[i];
+    }
+    return out;
+}
+/** ASCII-only uppercase. */
+function asciiUpper(s) {
+    let out = '';
+    for (let i = 0; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        out += c >= 0x61 && c <= 0x7a ? String.fromCharCode(c - 32) : s[i];
+    }
+    return out;
+}
+/** Lowercase ASCII bytes. */
+function asciiLowerBytes(b) {
+    const out = Buffer.from(b);
+    for (let i = 0; i < out.length; i++) {
+        const c = out[i];
+        if (c >= 0x41 && c <= 0x5a)
+            out[i] = c + 32;
+    }
+    return out;
+}
+/** Strips OWS (space and horizontal tab) on both sides. */
+function trimOws(b) {
+    let s = 0;
+    let e = b.length;
+    while (s < e && (b[s] === 0x20 || b[s] === 0x09))
+        s++;
+    while (e > s && (b[e - 1] === 0x20 || b[e - 1] === 0x09))
+        e--;
+    return s === 0 && e === b.length ? b : b.subarray(s, e);
+}
+function trimOwsStr(s) {
+    let a = 0;
+    let e = s.length;
+    while (a < e && (s.charCodeAt(a) === 0x20 || s.charCodeAt(a) === 0x09))
+        a++;
+    while (e > a && (s.charCodeAt(e - 1) === 0x20 || s.charCodeAt(e - 1) === 0x09))
+        e--;
+    return s.slice(a, e);
+}
+/** Splits bytes on a single byte value (keeps empty parts). */
+function splitBytes(b, sep) {
+    const out = [];
+    let start = 0;
+    for (let i = 0; i < b.length; i++) {
+        if (b[i] === sep) {
+            out.push(b.subarray(start, i));
+            start = i + 1;
+        }
+    }
+    out.push(b.subarray(start));
+    return out;
+}
+function hexVal(c) {
+    if (c >= 0x30 && c <= 0x39)
+        return c - 0x30;
+    if (c >= 0x41 && c <= 0x46)
+        return c - 0x37;
+    if (c >= 0x61 && c <= 0x66)
+        return c - 0x57;
+    return -1;
+}
+/** Percent-decoding of sanitizer.md 4.1 (optionally "+" -> space first, 8.3). */
+function pctDecode(b, plusIsSpace) {
+    let needs = false;
+    for (let i = 0; i < b.length; i++) {
+        if (b[i] === 0x25 || (plusIsSpace && b[i] === 0x2b)) {
+            needs = true;
+            break;
+        }
+    }
+    if (!needs)
+        return b;
+    const out = Buffer.allocUnsafe(b.length);
+    let o = 0;
+    for (let i = 0; i < b.length; i++) {
+        const c = b[i];
+        if (c === 0x25 && i + 2 < b.length) {
+            const h = hexVal(b[i + 1]);
+            const l = hexVal(b[i + 2]);
+            if (h >= 0 && l >= 0) {
+                out[o++] = h * 16 + l;
+                i += 2;
+                continue;
+            }
+        }
+        out[o++] = plusIsSpace && c === 0x2b ? 0x20 : c;
+    }
+    return out.subarray(0, o);
+}
+/** Byte-order comparison of two strings by their UTF-8 encoding (code point order). */
+function cmpUtf8(a, b) {
+    const n = Math.min(a.length, b.length);
+    for (let i = 0; i < n; i++) {
+        let x = a.charCodeAt(i);
+        let y = b.charCodeAt(i);
+        if (x === y)
+            continue;
+        // Surrogates (supplementary code points) sort after U+E000..U+FFFF in UTF-8.
+        if (x >= 0xd800 && x <= 0xdfff)
+            x += 0x10000;
+        if (y >= 0xd800 && y <= 0xdfff)
+            y += 0x10000;
+        return x < y ? -1 : 1;
+    }
+    return a.length === b.length ? 0 : a.length < b.length ? -1 : 1;
+}

package/dist/cjs/sanitizer/vectors.d.ts

@@ -0,0 +1,22 @@
+import { type Plain } from './json.js';
+import { type Exchange } from './sanitizer.js';
+export type VectorResult = {
+    outcome: 'event';
+    canonical: string;
+} | {
+    outcome: 'not_sampled';
+    sampled_by: string;
+} | {
+    outcome: 'refused';
+    code: string;
+};
+type Obj = {
+    [k: string]: Plain;
+};
+/** Converts the `input` object of a vector into a sanitizer exchange. */
+export declare function vectorExchange(inp: Obj): Exchange;
+/** Parses a vector file (bytes) with the lexeme-preserving parser. */
+export declare function parseVector(bytes: Uint8Array): Obj;
+/** Runs one parsed vector. `root` is the conformance/ directory (for policy_ref). */
+export declare function runVector(vec: Obj, root: string): VectorResult;
+export {};

package/dist/cjs/sanitizer/vectors.js

@@ -0,0 +1,130 @@
+"use strict";
+// Executes one conformance vector (conformance/format.md, "Executing a vector").
+// This is the test function every source must expose (sanitizer.md 16).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.vectorExchange = vectorExchange;
+exports.parseVector = parseVector;
+exports.runVector = runVector;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const json_js_1 = require("./json.js");
+const policy_js_1 = require("./policy.js");
+const sanitizer_js_1 = require("./sanitizer.js");
+function decodeBytes(v) {
+    if (v === undefined || v === null)
+        return Buffer.alloc(0);
+    if (typeof v === 'string')
+        return Buffer.from(v, 'utf8');
+    if (typeof v === 'object' && !Array.isArray(v) && !(v instanceof json_js_1.Lex) && typeof v.b64 === 'string') {
+        return Buffer.from(v.b64, 'base64');
+    }
+    throw new Error('bad bytes value');
+}
+function num(v) {
+    return v instanceof json_js_1.Lex ? Number(v.lex) : undefined;
+}
+function headers(v) {
+    if (!Array.isArray(v))
+        return [];
+    return v.map((pair) => {
+        const p = pair;
+        return [p[0], decodeBytes(p[1])];
+    });
+}
+function total(v) {
+    if (v === 'unknown')
+        return 'unknown';
+    return num(v);
+}
+function bool(v) {
+    return typeof v === 'boolean' ? v : undefined;
+}
+/** Converts the `input` object of a vector into a sanitizer exchange. */
+function vectorExchange(inp) {
+    const ex = {
+        dir: inp.dir,
+        method: inp.method ?? '',
+        raw_path: decodeBytes(inp.raw_path),
+        raw_query: inp.raw_query === undefined ? undefined : decodeBytes(inp.raw_query),
+        req_headers: headers(inp.req_headers),
+        req_body: decodeBytes(inp.req_body),
+        req_body_total: total(inp.req_body_total),
+        req_body_consumed: bool(inp.req_body_consumed),
+        req_body_decoded: bool(inp.req_body_decoded),
+        resp_headers: headers(inp.resp_headers),
+        resp_body: decodeBytes(inp.resp_body),
+        resp_body_total: total(inp.resp_body_total),
+        resp_body_consumed: bool(inp.resp_body_consumed),
+        resp_body_decoded: bool(inp.resp_body_decoded),
+    };
+    if (typeof inp.service_name === 'string')
+        ex.service_name = inp.service_name;
+    if (typeof inp.url_base === 'string')
+        ex.url_base = inp.url_base;
+    if (typeof inp.framework_route === 'string')
+        ex.framework_route = inp.framework_route;
+    if (typeof inp.framework === 'string')
+        ex.framework = inp.framework;
+    if (typeof inp.trace_id === 'string')
+        ex.trace_id = inp.trace_id;
+    if (typeof inp.inherited === 'boolean')
+        ex.inherited = inp.inherited;
+    const st = num(inp.status);
+    if (st !== undefined)
+        ex.status = st;
+    if (typeof inp.net_error === 'string')
+        ex.net_error = inp.net_error;
+    return ex;
+}
+/** Parses a vector file (bytes) with the lexeme-preserving parser. */
+function parseVector(bytes) {
+    return (0, json_js_1.nodeToPlain)((0, json_js_1.parseJson)(bytes), false);
+}
+/** Runs one parsed vector. `root` is the conformance/ directory (for policy_ref). */
+function runVector(vec, root) {
+    const now = typeof vec.now === 'string' ? (0, policy_js_1.parseTime)(vec.now) : undefined;
+    let policy;
+    let sanitizer;
+    try {
+        let body;
+        if (vec.policy_envelope !== undefined) {
+            const keys = new Map();
+            const fk = vec.fleet_keys;
+            for (const kid of Object.keys(fk ?? {})) {
+                const k = (0, policy_js_1.ed25519PublicKey)(fk[kid]);
+                if (k)
+                    keys.set(kid, k);
+            }
+            body = (0, policy_js_1.verifyEnvelope)(vec.policy_envelope, keys, true).body;
+        }
+        else if (typeof vec.policy_ref === 'string') {
+            body = (0, policy_js_1.validateBodySchema)((0, json_js_1.nodeToPlain)((0, json_js_1.parseJson)((0, node_fs_1.readFileSync)((0, node_path_1.join)(root, vec.policy_ref))), true));
+        }
+        else if (vec.policy !== undefined) {
+            body = (0, policy_js_1.validateBodySchema)(vec.policy);
+        }
+        else {
+            throw new policy_js_1.Refused('no_policy');
+        }
+        policy = new policy_js_1.Policy(body);
+        (0, policy_js_1.checkAcceptance)(policy, {
+            group: typeof vec.source_group === 'string' ? vec.source_group : '',
+            now: now ?? NaN,
+            lastIssuedSeq: num(vec.last_issued_seq) ?? 0,
+            minIssuedSeq: num(vec.min_issued_seq) ?? 0,
+        });
+        const ring = (0, policy_js_1.parseKeyRing)(typeof vec.hmac_key === 'string' ? vec.hmac_key : undefined);
+        sanitizer = new sanitizer_js_1.Sanitizer(policy, (0, policy_js_1.selectKey)(ring, policy));
+    }
+    catch (e) {
+        if (e instanceof policy_js_1.Refused)
+            return { outcome: 'refused', code: e.code };
+        throw e;
+    }
+    const r = sanitizer.run(vectorExchange(vec.input), now);
+    if (r.outcome === 'not_sampled')
+        return { outcome: 'not_sampled', sampled_by: r.sampledBy };
+    if (r.outcome === 'too_large')
+        return { outcome: 'refused', code: 'event_too_large' };
+    return { outcome: 'event', canonical: r.canonical };
+}

package/dist/cjs/version.d.ts

@@ -0,0 +1,2 @@
+export declare const SDK_VERSION = "0.1.0";
+export declare const SOURCE_KIND = "sdk-node";

package/dist/cjs/version.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SOURCE_KIND = exports.SDK_VERSION = void 0;
+exports.SDK_VERSION = '0.1.0';
+exports.SOURCE_KIND = 'sdk-node';

package/dist/cjs/worker/dir.d.ts

@@ -0,0 +1 @@
+export declare function workerDir(): string;

package/dist/cjs/worker/dir.js

@@ -0,0 +1,8 @@
+"use strict";
+// Directory of the compiled worker files. This source is the CommonJS flavour; scripts/build.mjs
+// replaces dist/esm/worker/dir.js with an import.meta.url based one (tsc cannot emit both).
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.workerDir = workerDir;
+function workerDir() {
+    return __dirname;
+}

package/dist/cjs/worker/pool.d.ts

@@ -0,0 +1,47 @@
+import type { Sanitizer } from '../sanitizer/sanitizer.js';
+import type { Job } from './protocol.js';
+interface Batch {
+    n: number;
+    bytes: number;
+}
+export interface PoolOptions {
+    size: number;
+    /** A batch came back: `stale` when it was posted before a discard(); recs/errs are then empty. */
+    onOut: (b: Batch & {
+        recs: string[];
+        errs: string[];
+        stale: boolean;
+    }) => void;
+    /** A worker died with `lost` batches in flight. */
+    onLost: (lost: Batch) => void;
+}
+export declare class SanitizerPool {
+    private readonly o;
+    private workers;
+    private epoch;
+    private seq;
+    private nextGen;
+    private readonly gens;
+    private closed;
+    constructor(o: PoolOptions);
+    /** True when at least one worker is up. */
+    get usable(): boolean;
+    /** Starts the first worker (idempotent). Never throws. */
+    warm(): void;
+    /** The policy of this sanitizer can be shipped to a worker. */
+    canHandle(san: Sanitizer): boolean;
+    private spawn;
+    private onMessage;
+    private pick;
+    /** Posts a batch of jobs. False (nothing posted) when no worker is ready. */
+    post(items: Array<{
+        san: Sanitizer;
+        job: Job;
+    }>, bytes: number, sourceId: string): boolean;
+    /** Results of everything posted so far are ignored when they arrive (kill switch, revocation). */
+    discard(): void;
+    /** Resolves when every batch posted before this call has come back. */
+    sync(): Promise<void>;
+    terminate(): void;
+}
+export {};

package/dist/cjs/worker/pool.js

@@ -0,0 +1,179 @@
+"use strict";
+// Main-thread side of the sanitizer worker pool: a small number of worker threads that turn
+// wire exchanges into OTLP records off the request path. Bounded and drop-on-pressure by
+// construction (the agent charges every posted job to its pending limits and drops when
+// they are exceeded); fail-open for the app: a dead worker just means "sanitize inline".
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SanitizerPool = void 0;
+const node_worker_threads_1 = require("node:worker_threads");
+const node_path_1 = require("node:path");
+const log_js_1 = require("../log.js");
+const dir_js_1 = require("./dir.js");
+class SanitizerPool {
+    o;
+    workers = [];
+    epoch = 0;
+    seq = 0;
+    nextGen = 1;
+    gens = new WeakMap();
+    closed = false;
+    constructor(o) {
+        this.o = o;
+    }
+    /** True when at least one worker is up. */
+    get usable() {
+        return !this.closed && this.workers.some((w) => w.ready && !w.dead);
+    }
+    /** Starts the first worker (idempotent). Never throws. */
+    warm() {
+        if (this.closed || this.workers.length > 0)
+            return;
+        this.spawn();
+    }
+    /** The policy of this sanitizer can be shipped to a worker. */
+    canHandle(san) {
+        if (!san.policy.bodyBytes)
+            return false;
+        const g = this.gens.get(san);
+        return g === undefined || g > this.nextGen - 12;
+    }
+    spawn() {
+        try {
+            const worker = new node_worker_threads_1.Worker((0, node_path_1.join)((0, dir_js_1.workerDir)(), 'sanitizer-worker.js'), {
+                // The parent's --import/--require preloads must not run again in the worker.
+                execArgv: [],
+                env: { TT_DISABLED: '1' },
+                workerData: { ttSanitizer: true },
+                resourceLimits: { maxOldGenerationSizeMb: 192 },
+            });
+            const w = { worker, ready: false, dead: false, gens: new Set(), batches: [], syncs: new Map() };
+            worker.unref();
+            worker.on('message', (m) => this.onMessage(w, m));
+            const die = () => {
+                if (w.dead)
+                    return;
+                w.dead = true;
+                w.ready = false;
+                let n = 0;
+                let bytes = 0;
+                for (const b of w.batches) {
+                    n += b.n;
+                    bytes += b.bytes;
+                }
+                w.batches = [];
+                for (const r of w.syncs.values())
+                    r();
+                w.syncs.clear();
+                if (n > 0)
+                    this.o.onLost({ n, bytes });
+                (0, log_js_1.debug)('sanitizer_worker', 'down');
+            };
+            worker.on('error', die);
+            worker.on('exit', die);
+            this.workers.push(w);
+            return w;
+        }
+        catch {
+            return null;
+        }
+    }
+    onMessage(w, m) {
+        if (m.t === 'ready') {
+            w.ready = true;
+        }
+        else if (m.t === 'out') {
+            const b = w.batches.shift();
+            if (!b)
+                return;
+            const stale = m.epoch !== this.epoch;
+            this.o.onOut({ ...b, recs: stale ? [] : m.recs, errs: stale ? [] : m.errs, stale });
+        }
+        else if (m.t === 'synced') {
+            const f = w.syncs.get(m.seq);
+            if (f) {
+                w.syncs.delete(m.seq);
+                f();
+            }
+        }
+    }
+    pick() {
+        let best = null;
+        for (const w of this.workers) {
+            if (!w.ready || w.dead)
+                continue;
+            if (!best || w.batches.length < best.batches.length)
+                best = w;
+        }
+        // Grow (up to the configured size) only when the least busy worker already has a backlog.
+        if ((!best || best.batches.length >= 2) && this.workers.filter((w) => !w.dead).length < this.o.size)
+            this.spawn();
+        return best;
+    }
+    /** Posts a batch of jobs. False (nothing posted) when no worker is ready. */
+    post(items, bytes, sourceId) {
+        const w = this.pick();
+        if (!w)
+            return false;
+        try {
+            const jobs = [];
+            for (const { san, job } of items) {
+                let g = this.gens.get(san);
+                if (g === undefined) {
+                    g = this.nextGen++;
+                    this.gens.set(san, g);
+                }
+                job.gen = g;
+                jobs.push(job);
+                if (!w.gens.has(g)) {
+                    const p = san.policy;
+                    const msg = {
+                        t: 'policy',
+                        gen: g,
+                        body: Uint8Array.from(p.bodyBytes),
+                        hash: p.hash,
+                        kid: san.key.kid,
+                        key: Uint8Array.from(san.key.key),
+                        rateOverrideBp: san.options.rateOverrideBp ?? null,
+                    };
+                    w.worker.postMessage(msg);
+                    w.gens.add(g);
+                }
+            }
+            w.worker.postMessage({ t: 'run', epoch: this.epoch, sourceId, jobs });
+            w.batches.push({ n: jobs.length, bytes });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /** Results of everything posted so far are ignored when they arrive (kill switch, revocation). */
+    discard() {
+        this.epoch++;
+    }
+    /** Resolves when every batch posted before this call has come back. */
+    sync() {
+        const ps = [];
+        for (const w of this.workers) {
+            if (w.dead || !w.ready || w.batches.length === 0)
+                continue;
+            const seq = ++this.seq;
+            ps.push(new Promise((r) => {
+                w.syncs.set(seq, r);
+                w.worker.postMessage({ t: 'sync', seq });
+            }));
+        }
+        return Promise.all(ps).then(() => undefined);
+    }
+    terminate() {
+        this.closed = true;
+        for (const w of this.workers) {
+            w.dead = true;
+            for (const r of w.syncs.values())
+                r();
+            void w.worker.terminate().catch(() => undefined);
+        }
+        this.workers = [];
+    }
+}
+exports.SanitizerPool = SanitizerPool;

package/dist/cjs/worker/protocol.d.ts

@@ -0,0 +1,37 @@
+import type { WireExchange } from '../capture/wire.js';
+import type { JobMeta } from '../export/pipeline.js';
+import type { Decision } from '../sanitizer/sanitizer.js';
+export interface Job {
+    gen: number;
+    decision: Decision;
+    meta: JobMeta;
+    wire: WireExchange;
+}
+export type ToWorker = {
+    t: 'policy';
+    gen: number;
+    body: Uint8Array;
+    hash: string;
+    kid: string;
+    key: Uint8Array;
+    rateOverrideBp: number | null;
+} | {
+    t: 'run';
+    epoch: number;
+    sourceId: string;
+    jobs: Job[];
+} | {
+    t: 'sync';
+    seq: number;
+};
+export type FromWorker = {
+    t: 'ready';
+} | {
+    t: 'out';
+    epoch: number;
+    recs: string[];
+    errs: string[];
+} | {
+    t: 'synced';
+    seq: number;
+};

package/dist/cjs/worker/protocol.js

@@ -0,0 +1,3 @@
+"use strict";
+// Messages between the main thread and a sanitizer worker (structured-clone friendly).
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/worker/sanitizer-worker.d.ts

@@ -0,0 +1 @@
+export {};