@traffic-twin/sdk 0.0.0-stage → 0.1.0

package/dist/esm/export/exporter.d.ts

@@ -0,0 +1,50 @@
+import type https from 'node:https';
+import { type QueuedEvent, type Resource } from './event.js';
+export interface ExportCounters {
+    events_sent: number;
+    events_dropped_queue: number;
+    events_rejected: number;
+    send_errors: number;
+}
+export interface ExporterOptions {
+    url: string;
+    agent: https.Agent;
+    resource: Resource;
+    counters: ExportCounters;
+    onError: (code: string) => void;
+    maxEvents?: number;
+    maxBytes?: number;
+    batchEvents?: number;
+    batchBytes?: number;
+    flushMs?: number;
+    maxRetries?: number;
+    baseBackoffMs?: number;
+    maxBackoffMs?: number;
+}
+export declare class Exporter {
+    private queue;
+    private head;
+    private bytes;
+    private inFlight;
+    private timer;
+    private retryTimer;
+    private stopped;
+    /** Bumped by clear() (kill switch, revocation): a batch built under an older generation is never sent. */
+    private gen;
+    private readonly o;
+    constructor(opts: ExporterOptions);
+    get size(): {
+        events: number;
+        bytes: number;
+    };
+    setTarget(url: string, agent: https.Agent): void;
+    enqueue(e: QueuedEvent): void;
+    /** Drops everything queued (kill switch). */
+    clear(): void;
+    stop(): void;
+    /** Sends what is queued; resolves when the queue is empty or a send failed. */
+    flush(): Promise<void>;
+    private kick;
+    private takeBatch;
+    private sendOne;
+}

package/dist/esm/export/exporter.js

@@ -0,0 +1,188 @@
+// OTLP/HTTP JSON exporter (event.md 4): bounded queue, batches of at most 512 events or
+// 4 MiB, retry with exponential backoff and jitter on 429/502/503/504 and network errors
+// (at most 5 times, Retry-After honoured), drop-oldest under pressure. The app never waits.
+import { otlpBody } from './event.js';
+import { request } from '../fleet/http.js';
+import { debug } from '../log.js';
+const sizeOf = (e) => e.size ?? e.json.length;
+const RETRYABLE = new Set([429, 502, 503, 504]);
+const DROP = new Set([400, 401, 403, 413]);
+export class Exporter {
+    queue = [];
+    head = 0;
+    bytes = 0;
+    inFlight = false;
+    timer;
+    retryTimer;
+    stopped = false;
+    /** Bumped by clear() (kill switch, revocation): a batch built under an older generation is never sent. */
+    gen = 0;
+    o;
+    constructor(opts) {
+        this.o = {
+            maxEvents: 10_000,
+            maxBytes: 16 << 20,
+            batchEvents: 512,
+            batchBytes: 4 << 20,
+            flushMs: 1000,
+            maxRetries: 5,
+            baseBackoffMs: 1000,
+            maxBackoffMs: 30_000,
+            ...opts,
+        };
+        this.timer = setInterval(() => this.kick(), this.o.flushMs);
+        this.timer.unref();
+    }
+    get size() {
+        return { events: this.queue.length - this.head, bytes: this.bytes };
+    }
+    setTarget(url, agent) {
+        this.o.url = url;
+        this.o.agent = agent;
+    }
+    enqueue(e) {
+        if (this.stopped)
+            return;
+        this.queue.push(e);
+        this.bytes += sizeOf(e);
+        while (this.queue.length - this.head > this.o.maxEvents || this.bytes > this.o.maxBytes) {
+            const old = this.queue[this.head++];
+            this.bytes -= sizeOf(old);
+            this.o.counters.events_dropped_queue++;
+        }
+        if (this.head > 4096 && this.head * 2 > this.queue.length) {
+            this.queue = this.queue.slice(this.head);
+            this.head = 0;
+        }
+        if (this.queue.length - this.head >= this.o.batchEvents)
+            setImmediate(() => this.kick());
+    }
+    /** Drops everything queued (kill switch). */
+    clear() {
+        this.gen++;
+        this.o.counters.events_dropped_queue += this.queue.length - this.head;
+        this.queue = [];
+        this.head = 0;
+        this.bytes = 0;
+    }
+    stop() {
+        this.stopped = true;
+        if (this.timer)
+            clearInterval(this.timer);
+        if (this.retryTimer)
+            clearTimeout(this.retryTimer);
+    }
+    /** Sends what is queued; resolves when the queue is empty or a send failed. */
+    async flush() {
+        for (let i = 0; i < 1000 && this.queue.length - this.head > 0 && !this.stopped; i++) {
+            if (this.inFlight) {
+                await new Promise((r) => setTimeout(r, 20));
+                continue;
+            }
+            const ok = await this.sendOne(false);
+            if (!ok)
+                return;
+        }
+    }
+    kick() {
+        if (this.inFlight || this.stopped || this.retryTimer || this.queue.length - this.head === 0)
+            return;
+        void this.sendOne(true);
+    }
+    takeBatch() {
+        const batch = [];
+        let size = 0;
+        while (this.head < this.queue.length && batch.length < this.o.batchEvents) {
+            const e = this.queue[this.head];
+            const sz = sizeOf(e);
+            if (batch.length > 0 && size + sz > this.o.batchBytes)
+                break;
+            batch.push(e);
+            size += sz;
+            this.head++;
+            this.bytes -= sz;
+        }
+        return batch;
+    }
+    async sendOne(retryLater) {
+        this.inFlight = true;
+        const batch = this.takeBatch();
+        const gen = this.gen;
+        try {
+            for (let attempt = 0;; attempt++) {
+                // Kill switch or revocation while this batch was held (first attempt or between
+                // retries, Retry-After included): it is dropped, never sent.
+                if (gen !== this.gen) {
+                    this.o.counters.events_dropped_queue += batch.length;
+                    return false;
+                }
+                let res;
+                let netErr = false;
+                try {
+                    res = await request({
+                        method: 'POST',
+                        url: this.o.url,
+                        agent: this.o.agent,
+                        headers: { 'content-type': 'application/json' },
+                        body: otlpBody(batch, this.o.resource, Date.now()),
+                        timeoutMs: 15_000,
+                        maxResponseBytes: 64 << 10,
+                    });
+                }
+                catch {
+                    netErr = true;
+                }
+                if (res && res.status >= 200 && res.status < 300) {
+                    let rejected = 0;
+                    try {
+                        const j = JSON.parse(res.body.toString('utf8') || '{}');
+                        rejected = Number(j.partialSuccess?.rejectedLogRecords ?? 0) || 0;
+                    }
+                    catch {
+                        rejected = 0;
+                    }
+                    rejected = Math.min(Math.max(0, rejected), batch.length);
+                    this.o.counters.events_sent += batch.length - rejected;
+                    this.o.counters.events_rejected += rejected;
+                    if (rejected)
+                        this.o.onError('intake_partial_reject');
+                    return true;
+                }
+                const retryable = netErr || (res !== undefined && RETRYABLE.has(res.status));
+                if (!retryable || attempt >= this.o.maxRetries || this.stopped) {
+                    if (res && DROP.has(res.status))
+                        this.o.counters.events_rejected += batch.length;
+                    else
+                        this.o.counters.events_dropped_queue += batch.length;
+                    this.o.counters.send_errors++;
+                    this.o.onError(res ? 'intake_http_' + res.status : 'intake_network');
+                    debug('export_drop', res ? res.status : 'network');
+                    return false;
+                }
+                this.o.counters.send_errors++;
+                let delay = Math.min(this.o.maxBackoffMs, this.o.baseBackoffMs * 2 ** attempt);
+                delay = delay / 2 + Math.random() * (delay / 2);
+                const ra = res?.headers['retry-after'];
+                if (typeof ra === 'string') {
+                    const secs = /^[0-9]+$/.test(ra.trim()) ? Number(ra) * 1000 : Date.parse(ra) - Date.now();
+                    if (Number.isFinite(secs) && secs > 0)
+                        delay = Math.min(Math.max(delay, secs), 300_000);
+                }
+                await new Promise((r) => {
+                    this.retryTimer = setTimeout(() => {
+                        this.retryTimer = undefined;
+                        r();
+                    }, delay);
+                    this.retryTimer.unref();
+                });
+                if (this.stopped)
+                    return false;
+                void retryLater;
+                // (the generation is checked again at the top of the loop)
+            }
+        }
+        finally {
+            this.inFlight = false;
+        }
+    }
+}

package/dist/esm/export/pipeline.d.ts

@@ -0,0 +1,20 @@
+import { type WireExchange } from '../capture/wire.js';
+import type { Decision, NetError, Sanitizer } from '../sanitizer/sanitizer.js';
+/** Per-exchange facts the source fills in (event.md 1). */
+export interface JobMeta {
+    dir: 'inbound' | 'outbound';
+    tsMs: number;
+    latencyMs: number;
+    status?: number;
+    netError?: NetError;
+    traceId?: string;
+    spanId?: string;
+    parentSpanId?: string;
+}
+export type JobResult = {
+    record: string;
+} | {
+    err: 'event_too_large' | 'event_invalid_or_too_large' | 'sanitizer_error';
+} | null;
+/** Null when the exchange turned out not to be sampled. Never throws. */
+export declare function sanitizeToRecord(san: Sanitizer, wire: WireExchange, decision: Decision, meta: JobMeta, sourceId: string, nowMs: number): JobResult;

package/dist/esm/export/pipeline.js

@@ -0,0 +1,37 @@
+// The off-path half of one exchange: wire form -> sanitizer -> canonical event -> OTLP
+// logRecord. Used both inline (main thread) and by the sanitizer worker threads, so the
+// output is the same bytes either way.
+import { wireToExchange } from '../capture/wire.js';
+import { buildEvent, makeRecord, serializedParts } from './event.js';
+/** Null when the exchange turned out not to be sampled. Never throws. */
+export function sanitizeToRecord(san, wire, decision, meta, sourceId, nowMs) {
+    try {
+        const r = san.run(wireToExchange(wire), nowMs, decision);
+        if (r.outcome === 'not_sampled')
+            return null;
+        if (r.outcome === 'too_large')
+            return { err: 'event_too_large' };
+        const p = san.policy;
+        const json = buildEvent(serializedParts(r.parts), {
+            policyId: p.policyId,
+            policyVersion: p.version,
+            policyHash: p.hash,
+            sourceId,
+            dir: meta.dir,
+            tsMs: meta.tsMs,
+            latencyMs: meta.latencyMs,
+            status: meta.status,
+            netError: meta.netError,
+            traceId: meta.traceId,
+            spanId: meta.spanId,
+            parentSpanId: meta.parentSpanId,
+        });
+        if (json === null)
+            return { err: 'event_invalid_or_too_large' };
+        return { record: makeRecord({ json, tsMs: meta.tsMs, traceId: meta.traceId, spanId: meta.spanId, policyHash: p.hash }, Date.now()) };
+    }
+    catch {
+        // 0: an internal error never produces an event.
+        return { err: 'sanitizer_error' };
+    }
+}

package/dist/esm/fleet/asn1.d.ts

@@ -0,0 +1,44 @@
+import { type KeyObject } from 'node:crypto';
+export declare function tlv(tag: number, ...content: Buffer[]): Buffer;
+export declare const seq: (...c: Buffer[]) => Buffer<ArrayBufferLike>;
+export declare const set: (...c: Buffer[]) => Buffer<ArrayBufferLike>;
+export declare const ctx: (n: number, ...c: Buffer[]) => Buffer<ArrayBufferLike>;
+export declare const octets: (b: Buffer) => Buffer<ArrayBufferLike>;
+export declare const bitString: (b: Buffer) => Buffer<ArrayBufferLike>;
+export declare const utf8String: (s: string) => Buffer<ArrayBufferLike>;
+export declare const bool: (v: boolean) => Buffer<ArrayBufferLike>;
+export declare const nul: () => Buffer<ArrayBuffer>;
+export declare function int(v: number | Buffer): Buffer;
+export declare function oid(dotted: string): Buffer;
+export declare const OID: {
+    commonName: string;
+    ecdsaWithSHA256: string;
+    ed25519: string;
+};
+export declare function name(cn: string): Buffer;
+export interface DerNode {
+    tag: number;
+    start: number;
+    header: number;
+    length: number;
+    raw: Buffer;
+    content: Buffer;
+}
+/** Reads one TLV at `off`. */
+export declare function readTlv(buf: Buffer, off?: number): DerNode;
+export declare function children(node: DerNode): DerNode[];
+export declare function pem(label: string, der: Buffer): string;
+export declare function pemBlocks(text: string, label: string): Buffer[];
+/** Signs DER `data` with `key` and returns the AlgorithmIdentifier and the signature bits. */
+export declare function signDer(data: Buffer, key: KeyObject): {
+    alg: Buffer;
+    sig: Buffer;
+};
+export interface GeneratedCsr {
+    csrPem: string;
+    keyPem: string;
+}
+/** Generates a key pair (default ECDSA P-256) and a PKCS#10 request with CN = commonName. */
+export declare function generateCsr(commonName: string, kind?: 'ec' | 'ed25519'): GeneratedCsr;
+/** base64url(SHA-256(DER)), the CA pin format of the enrollment token. */
+export declare function pinOf(der: Buffer): string;

package/dist/esm/fleet/asn1.js

@@ -0,0 +1,128 @@
+// Tiny DER writer/reader, enough for a PKCS#10 certificate request (RFC 2986) with an
+// ECDSA P-256 or Ed25519 key, without any dependency.
+import { createHash, generateKeyPairSync, sign } from 'node:crypto';
+function len(n) {
+    if (n < 0x80)
+        return Buffer.from([n]);
+    const bytes = [];
+    while (n > 0) {
+        bytes.unshift(n & 0xff);
+        n = Math.floor(n / 256);
+    }
+    return Buffer.from([0x80 | bytes.length, ...bytes]);
+}
+export function tlv(tag, ...content) {
+    const c = Buffer.concat(content);
+    return Buffer.concat([Buffer.from([tag]), len(c.length), c]);
+}
+export const seq = (...c) => tlv(0x30, ...c);
+export const set = (...c) => tlv(0x31, ...c);
+export const ctx = (n, ...c) => tlv(0xa0 + n, ...c);
+export const octets = (b) => tlv(0x04, b);
+export const bitString = (b) => tlv(0x03, Buffer.from([0]), b);
+export const utf8String = (s) => tlv(0x0c, Buffer.from(s, 'utf8'));
+export const bool = (v) => tlv(0x01, Buffer.from([v ? 0xff : 0]));
+export const nul = () => Buffer.from([0x05, 0x00]);
+export function int(v) {
+    let b;
+    if (typeof v === 'number') {
+        let h = v.toString(16);
+        if (h.length % 2)
+            h = '0' + h;
+        b = Buffer.from(h, 'hex');
+    }
+    else {
+        b = v;
+    }
+    while (b.length > 1 && b[0] === 0 && b[1] < 0x80)
+        b = b.subarray(1);
+    if (b[0] >= 0x80)
+        b = Buffer.concat([Buffer.from([0]), b]);
+    return tlv(0x02, b);
+}
+export function oid(dotted) {
+    const parts = dotted.split('.').map(Number);
+    const out = [parts[0] * 40 + parts[1]];
+    for (const p of parts.slice(2)) {
+        const stack = [p & 0x7f];
+        let v = Math.floor(p / 128);
+        while (v > 0) {
+            stack.unshift((v & 0x7f) | 0x80);
+            v = Math.floor(v / 128);
+        }
+        out.push(...stack);
+    }
+    return tlv(0x06, Buffer.from(out));
+}
+export const OID = {
+    commonName: '2.5.4.3',
+    ecdsaWithSHA256: '1.2.840.10045.4.3.2',
+    ed25519: '1.3.101.112',
+};
+export function name(cn) {
+    return seq(set(seq(oid(OID.commonName), utf8String(cn))));
+}
+/** Reads one TLV at `off`. */
+export function readTlv(buf, off = 0) {
+    const tag = buf[off];
+    let l = buf[off + 1];
+    let header = 2;
+    if (l & 0x80) {
+        const n = l & 0x7f;
+        if (n === 0 || n > 4)
+            throw new Error('der length');
+        l = 0;
+        for (let i = 0; i < n; i++)
+            l = l * 256 + buf[off + 2 + i];
+        header += n;
+    }
+    if (off + header + l > buf.length)
+        throw new Error('der overflow');
+    return { tag, start: off, header, length: l, raw: buf.subarray(off, off + header + l), content: buf.subarray(off + header, off + header + l) };
+}
+export function children(node) {
+    const out = [];
+    let off = 0;
+    while (off < node.content.length) {
+        const c = readTlv(node.content, off);
+        out.push(c);
+        off += c.header + c.length;
+    }
+    return out;
+}
+export function pem(label, der) {
+    const b64 = der.toString('base64').replace(/(.{64})/g, '$1\n').replace(/\n$/, '');
+    return `-----BEGIN ${label}-----\n${b64}\n-----END ${label}-----\n`;
+}
+export function pemBlocks(text, label) {
+    const re = new RegExp(`-----BEGIN ${label}-----([\\s\\S]*?)-----END ${label}-----`, 'g');
+    const out = [];
+    let m;
+    while ((m = re.exec(text)))
+        out.push(Buffer.from(m[1].replace(/\s+/g, ''), 'base64'));
+    return out;
+}
+/** Signs DER `data` with `key` and returns the AlgorithmIdentifier and the signature bits. */
+export function signDer(data, key) {
+    if (key.asymmetricKeyType === 'ed25519')
+        return { alg: seq(oid(OID.ed25519)), sig: sign(null, data, key) };
+    return { alg: seq(oid(OID.ecdsaWithSHA256)), sig: sign('sha256', data, key) };
+}
+/** Generates a key pair (default ECDSA P-256) and a PKCS#10 request with CN = commonName. */
+export function generateCsr(commonName, kind = 'ec') {
+    const { publicKey, privateKey } = kind === 'ed25519'
+        ? generateKeyPairSync('ed25519')
+        : generateKeyPairSync('ec', { namedCurve: 'prime256v1' });
+    const spki = publicKey.export({ type: 'spki', format: 'der' });
+    const info = seq(int(0), name(commonName), spki, ctx(0));
+    const { alg, sig } = signDer(info, privateKey);
+    const csr = seq(info, alg, bitString(sig));
+    return {
+        csrPem: pem('CERTIFICATE REQUEST', csr),
+        keyPem: privateKey.export({ type: 'pkcs8', format: 'pem' }),
+    };
+}
+/** base64url(SHA-256(DER)), the CA pin format of the enrollment token. */
+export function pinOf(der) {
+    return createHash('sha256').update(der).digest('base64url');
+}

package/dist/esm/fleet/client.d.ts

@@ -0,0 +1,84 @@
+import https from 'node:https';
+export declare const TOKEN_RE: RegExp;
+export interface SigningKey {
+    kid: string;
+    alg: 'Ed25519';
+    public_key: string;
+}
+export interface Enrollment {
+    source_id: string;
+    group: string;
+    service_name: string;
+    client_cert: string;
+    ca_chain: string;
+    not_after?: string;
+    policy_signing_keys: SigningKey[];
+    intake_url: string;
+    fleet_url: string;
+    policy_max_age_s: number;
+    heartbeat_interval_s: number;
+    policy_poll_interval_s: number;
+    /** Local bookkeeping. */
+    enroll_url: string;
+    pin: string;
+}
+export interface SourceInfo {
+    kind: string;
+    version: string;
+    service_name: string;
+    instance?: string;
+    runtime?: {
+        lang: string;
+        lang_version: string;
+        os: string;
+        arch: string;
+    };
+    hmac_kids?: string[];
+}
+export declare class FleetError extends Error {
+    readonly code: string;
+    readonly status?: number | undefined;
+    constructor(code: string, status?: number | undefined);
+}
+/** Extracts the CA pin from an enrollment token. */
+export declare function tokenPin(token: string): string | null;
+/**
+ * Opens a TLS connection without trusting anything, walks the chain the server presents
+ * to its top (the root, which tt-core must include) and checks it against the pin.
+ * Returns the root as PEM, to be used as the only trust anchor for the real request.
+ */
+export declare function fetchPinnedRoot(url: URL, pin: string, timeoutMs?: number): Promise<string>;
+/** Enrolls with a token. Returns the enrollment and the private key PEM (generated locally when useCsr). */
+export declare function enroll(o: {
+    enrollUrl: string;
+    token: string;
+    source: SourceInfo;
+    useCsr: boolean;
+    timeoutMs?: number;
+}): Promise<{
+    enrollment: Enrollment;
+    keyPem: string;
+}>;
+/** mTLS agent for the intake and Fleet calls. Trust anchors: the CA chain received over the pinned channel. */
+export declare function mtlsAgent(e: Enrollment, keyPem: string): https.Agent;
+export declare class FleetClient {
+    private e;
+    private agent;
+    constructor(e: Enrollment, agent: https.Agent);
+    setAgent(agent: https.Agent): void;
+    private url;
+    heartbeat(body: unknown): Promise<{
+        status: number;
+        json?: Record<string, unknown>;
+    }>;
+    getPolicy(etag?: string): Promise<{
+        status: number;
+        body?: Buffer;
+        etag?: string;
+    }>;
+    renew(useCsr: boolean, commonName: string): Promise<{
+        cert: string;
+        keyPem?: string;
+        notAfter?: string;
+    } | null>;
+}