nixamp 0.16.0 → 0.17.0

package/dist/compression/envelope.js

@@ -0,0 +1,191 @@
+/**
+ * The wire shape of a compressed relay: what one nixamp sends another.
+ *
+ * Not a .ts file and not an HTTP body somebody's proxy might gunzip: a
+ * stream of framed blocks, each one independently decodable, each one
+ * carrying the length and the SHA-256 of the bytes it stands for, ending in
+ * a marker that says the stream finished rather than dropped. The layout is
+ * specified byte by byte in docs/stream-compression.md; this file is that
+ * document as code, and the test vectors there are checked against it.
+ *
+ * Everything is big-endian. Every length is checked against the negotiated
+ * limit before a byte is allocated for it, so a hostile header cannot ask
+ * for a gigabyte.
+ */
+import { createHash } from "node:crypto";
+export const MAGIC = "NXS1";
+export const ENVELOPE_VERSION = 1;
+export const MEDIA_TYPE = "application/vnd.nixamp.stream";
+export const STREAM_HEADER_BYTES = 16;
+export const FRAME_HEADER_BYTES = 48;
+const BOUNDARY_CODE = { source: 0, channel: 1 };
+export const MODE_CODE = { stored: 0, zstd: 1, gzip: 2, "ts-zstd": 3 };
+export const MODES = ["stored", "zstd", "gzip", "ts-zstd"];
+export const FRAME_DATA = 1;
+export const FRAME_END = 2;
+/** The decoded size one frame may claim, unless negotiated otherwise. */
+export const DEFAULT_MAX_FRAME_BYTES = 256 * 1024;
+/** The most a stream header may negotiate, whatever it asks for. */
+export const CEILING_FRAME_BYTES = 16 * 1024 * 1024;
+/**
+ * How much bigger than its original an encoded payload may be. A codec that
+ * cannot beat stored is stored, so anything past a small fixed allowance is
+ * either a bug or an attack.
+ */
+export const EXPANSION_ALLOWANCE = 1024;
+/** A stream that broke one of the rules, and which rule. Never silently. */
+export class RelayError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+        this.name = "RelayError";
+    }
+}
+export function sha256(bytes) {
+    return createHash("sha256").update(bytes).digest();
+}
+export function encodeStreamHeader(header) {
+    if (header.version !== ENVELOPE_VERSION)
+        throw new RelayError("BAD_VERSION", `cannot write version ${header.version}`);
+    if (!(header.boundary in BOUNDARY_CODE))
+        throw new RelayError("BAD_BOUNDARY", `no such boundary: ${header.boundary}`);
+    if (!Number.isInteger(header.generation) || header.generation < 0 || header.generation > 0xffff_ffff) {
+        throw new RelayError("BAD_LIMIT", "generation must fit in 32 bits");
+    }
+    if (!Number.isInteger(header.maxFrameBytes) || header.maxFrameBytes < 1 || header.maxFrameBytes > CEILING_FRAME_BYTES) {
+        throw new RelayError("BAD_LIMIT", `maxFrameBytes must be 1..${CEILING_FRAME_BYTES}`);
+    }
+    const out = Buffer.alloc(STREAM_HEADER_BYTES);
+    out.write(MAGIC, 0, 4, "latin1");
+    out.writeUInt8(header.version, 4);
+    out.writeUInt8(0, 5); // flags, none defined
+    out.writeUInt8(BOUNDARY_CODE[header.boundary], 6);
+    out.writeUInt8(0, 7); // reserved
+    out.writeUInt32BE(header.generation, 8);
+    out.writeUInt32BE(header.maxFrameBytes, 12);
+    return out;
+}
+/** Read a stream header from the front of `bytes`. Throws on anything off. */
+export function decodeStreamHeader(bytes) {
+    if (bytes.length < STREAM_HEADER_BYTES)
+        throw new RelayError("TRUNCATED", "stream header is short");
+    if (bytes.toString("latin1", 0, 4) !== MAGIC)
+        throw new RelayError("BAD_MAGIC", "not a nixamp relay stream");
+    const version = bytes.readUInt8(4);
+    if (version !== ENVELOPE_VERSION)
+        throw new RelayError("BAD_VERSION", `envelope version ${version} is not understood`);
+    const boundaryCode = bytes.readUInt8(6);
+    const boundary = Object.keys(BOUNDARY_CODE).find((b) => BOUNDARY_CODE[b] === boundaryCode);
+    if (!boundary)
+        throw new RelayError("BAD_BOUNDARY", `boundary code ${boundaryCode} is not understood`);
+    const generation = bytes.readUInt32BE(8);
+    const maxFrameBytes = bytes.readUInt32BE(12);
+    if (maxFrameBytes < 1 || maxFrameBytes > CEILING_FRAME_BYTES) {
+        throw new RelayError("BAD_LIMIT", `maxFrameBytes ${maxFrameBytes} is outside 1..${CEILING_FRAME_BYTES}`);
+    }
+    return { version, boundary, generation, maxFrameBytes };
+}
+export function encodeFrameHeader(header) {
+    if (header.type !== FRAME_DATA && header.type !== FRAME_END)
+        throw new RelayError("BAD_FRAME_TYPE", `no such frame type ${header.type}`);
+    if (!(header.mode in MODE_CODE))
+        throw new RelayError("BAD_MODE", `no such mode: ${header.mode}`);
+    if (header.sha256.length !== 32)
+        throw new RelayError("CHECKSUM_MISMATCH", "sha256 must be 32 bytes");
+    for (const [name, value] of [["seq", header.seq], ["originalLength", header.originalLength], ["encodedLength", header.encodedLength]]) {
+        if (!Number.isInteger(value) || value < 0 || value > 0xffff_ffff)
+            throw new RelayError("BAD_LIMIT", `${name} must fit in 32 bits`);
+    }
+    const out = Buffer.alloc(FRAME_HEADER_BYTES);
+    out.writeUInt8(header.type, 0);
+    out.writeUInt8(MODE_CODE[header.mode], 1);
+    out.writeUInt16BE(0, 2); // reserved
+    out.writeUInt32BE(header.seq, 4);
+    out.writeUInt32BE(header.originalLength, 8);
+    out.writeUInt32BE(header.encodedLength, 12);
+    header.sha256.copy(out, 16);
+    return out;
+}
+/** A data frame: header then payload, as one buffer. */
+export function encodeDataFrame(seq, mode, original, encoded) {
+    return Buffer.concat([dataFrameHeader(seq, mode, original, encoded.length), encoded]);
+}
+/** Just the header of a data frame, for a payload that is shared between listeners. */
+export function dataFrameHeader(seq, mode, original, encodedLength, digest) {
+    return encodeFrameHeader({
+        type: FRAME_DATA,
+        mode,
+        seq,
+        originalLength: original.length,
+        encodedLength,
+        sha256: digest ?? sha256(original),
+    });
+}
+/**
+ * The end marker: no payload, the generation's total original byte count in
+ * the two length fields (high word, low word) and the SHA-256 of every
+ * original byte in order. A stream that stops without one was cut off.
+ */
+export function encodeEndFrame(seq, totalOriginalBytes, digest) {
+    if (!Number.isSafeInteger(totalOriginalBytes) || totalOriginalBytes < 0)
+        throw new RelayError("BAD_LIMIT", "total must be a non-negative integer");
+    return encodeFrameHeader({
+        type: FRAME_END,
+        mode: "stored",
+        seq,
+        originalLength: Math.floor(totalOriginalBytes / 0x1_0000_0000),
+        encodedLength: totalOriginalBytes % 0x1_0000_0000,
+        sha256: digest,
+    });
+}
+/** The total an end frame carries, from its two halves. */
+export function endFrameTotal(header) {
+    return header.originalLength * 0x1_0000_0000 + header.encodedLength;
+}
+/**
+ * Read a frame header, checking every field against the limits before the
+ * caller allocates anything for the payload.
+ */
+export function decodeFrameHeader(bytes, limits) {
+    if (bytes.length < FRAME_HEADER_BYTES)
+        throw new RelayError("TRUNCATED", "frame header is short");
+    const type = bytes.readUInt8(0);
+    if (type !== FRAME_DATA && type !== FRAME_END)
+        throw new RelayError("BAD_FRAME_TYPE", `frame type ${type} is not understood`);
+    const modeCode = bytes.readUInt8(1);
+    const mode = MODES.find((m) => MODE_CODE[m] === modeCode);
+    if (!mode)
+        throw new RelayError("BAD_MODE", `mode code ${modeCode} is not understood`);
+    const seq = bytes.readUInt32BE(4);
+    if (seq !== limits.expectSeq)
+        throw new RelayError("BAD_SEQUENCE", `expected frame ${limits.expectSeq}, got ${seq}`);
+    const originalLength = bytes.readUInt32BE(8);
+    const encodedLength = bytes.readUInt32BE(12);
+    const sha = Buffer.from(bytes.subarray(16, 48));
+    if (type === FRAME_END) {
+        return { type: FRAME_END, mode: "stored", seq, originalLength, encodedLength, sha256: sha };
+    }
+    if (!limits.modes.has(mode))
+        throw new RelayError("UNSUPPORTED_MODE", `this decoder cannot undo ${mode}`);
+    if (originalLength > limits.maxFrameBytes) {
+        throw new RelayError("FRAME_TOO_LARGE", `frame claims ${originalLength} original bytes, limit ${limits.maxFrameBytes}`);
+    }
+    if (encodedLength > originalLength + EXPANSION_ALLOWANCE) {
+        throw new RelayError("EXPANSION_BUDGET", `frame carries ${encodedLength} bytes for ${originalLength} original`);
+    }
+    if (mode === "stored" && encodedLength !== originalLength) {
+        throw new RelayError("LENGTH_MISMATCH", "a stored frame must carry exactly its original bytes");
+    }
+    return { type: FRAME_DATA, mode, seq, originalLength, encodedLength, sha256: sha };
+}
+/** The codec names a peer lists in a negotiation header, kept to the ones we know. */
+export function parseModes(header) {
+    const out = new Set();
+    for (const raw of (header ?? "").split(",")) {
+        const name = raw.trim().toLowerCase();
+        if (MODES.includes(name))
+            out.add(name);
+    }
+    return out;
+}

package/dist/compression/jobs.d.ts

@@ -0,0 +1,59 @@
+import type { Analysis } from "./analyze.ts";
+export type JobStatus = "queued" | "running" | "done" | "failed" | "cancelled";
+export interface Job {
+    id: string;
+    /** What is being analysed, for the listing. */
+    subject: string;
+    /** Who may see it: the scope that started it. */
+    owner: string;
+    status: JobStatus;
+    createdAt: number;
+    startedAt?: number;
+    finishedAt?: number;
+    progress: {
+        bytes: number;
+        ms: number;
+    };
+    result?: Analysis;
+    error?: string;
+}
+export interface JobsOptions {
+    concurrency?: number;
+    /** How long a finished job is kept. */
+    ttlMs?: number;
+    /** How many jobs, in any state, may exist at once. */
+    maxJobs?: number;
+}
+export type Runner = (signal: AbortSignal, progress: (bytes: number, ms: number) => void) => Promise<Analysis>;
+export declare class AnalysisJobs {
+    private readonly slots;
+    private readonly queue;
+    private running;
+    private readonly concurrency;
+    private readonly ttlMs;
+    private readonly maxJobs;
+    constructor(options?: JobsOptions);
+    /**
+     * Start a job, or return the one already doing the same thing. `key`
+     * names the thing: the same key while a job is queued or running is the
+     * same job. Null when the server has all the jobs it will hold.
+     */
+    start(key: string, subject: string, owner: string, run: Runner): {
+        job: Job;
+        existing: boolean;
+    } | null;
+    /** A job, if it exists and `owner` may see it. */
+    get(id: string, owner: string): Job | null;
+    cancel(id: string, owner: string): boolean;
+    list(owner: string): Job[];
+    get counts(): {
+        running: number;
+        queued: number;
+        kept: number;
+    };
+    /** Stop everything, for a server going down. */
+    stopAll(): void;
+    private pump;
+    /** Forget finished jobs older than the TTL. */
+    private sweep;
+}

package/dist/compression/jobs.js

@@ -0,0 +1,120 @@
+/**
+ * Diagnostic jobs: bounded, deduplicated, cancellable, forgotten in time.
+ *
+ * An analysis reads up to thirty seconds or twenty-five megabytes of a
+ * channel and runs every codec over it, which is real work. So there is a
+ * ceiling on how many run at once, a second request for the same thing
+ * while the first is still going is handed the first, and a finished
+ * result is kept for a while and then dropped rather than for ever.
+ */
+import { randomBytes } from "node:crypto";
+export class AnalysisJobs {
+    slots = new Map();
+    queue = [];
+    running = 0;
+    concurrency;
+    ttlMs;
+    maxJobs;
+    constructor(options = {}) {
+        this.concurrency = Math.max(1, options.concurrency ?? 1);
+        this.ttlMs = options.ttlMs ?? 10 * 60_000;
+        this.maxJobs = options.maxJobs ?? 32;
+    }
+    /**
+     * Start a job, or return the one already doing the same thing. `key`
+     * names the thing: the same key while a job is queued or running is the
+     * same job. Null when the server has all the jobs it will hold.
+     */
+    start(key, subject, owner, run) {
+        this.sweep();
+        for (const slot of this.slots.values()) {
+            if (slot.key === key && (slot.job.status === "queued" || slot.job.status === "running"))
+                return { job: slot.job, existing: true };
+        }
+        if (this.slots.size >= this.maxJobs)
+            return null;
+        const job = {
+            id: `a${randomBytes(6).toString("hex")}`,
+            subject,
+            owner,
+            status: "queued",
+            createdAt: Date.now(),
+            progress: { bytes: 0, ms: 0 },
+        };
+        const slot = { job, key, controller: new AbortController(), run };
+        this.slots.set(job.id, slot);
+        this.queue.push(slot);
+        this.pump();
+        return { job, existing: false };
+    }
+    /** A job, if it exists and `owner` may see it. */
+    get(id, owner) {
+        this.sweep();
+        const slot = this.slots.get(id);
+        if (!slot || slot.job.owner !== owner)
+            return null;
+        return slot.job;
+    }
+    cancel(id, owner) {
+        const slot = this.slots.get(id);
+        if (!slot || slot.job.owner !== owner)
+            return false;
+        if (slot.job.status !== "queued" && slot.job.status !== "running")
+            return false;
+        slot.controller.abort();
+        if (slot.job.status === "queued") {
+            const at = this.queue.indexOf(slot);
+            if (at !== -1)
+                this.queue.splice(at, 1);
+            slot.job.status = "cancelled";
+            slot.job.finishedAt = Date.now();
+        }
+        return true;
+    }
+    list(owner) {
+        this.sweep();
+        return [...this.slots.values()].map((slot) => slot.job).filter((job) => job.owner === owner);
+    }
+    get counts() {
+        return { running: this.running, queued: this.queue.length, kept: this.slots.size };
+    }
+    /** Stop everything, for a server going down. */
+    stopAll() {
+        for (const slot of this.slots.values())
+            slot.controller.abort();
+        this.queue.length = 0;
+    }
+    pump() {
+        while (this.running < this.concurrency && this.queue.length > 0) {
+            const slot = this.queue.shift();
+            this.running += 1;
+            slot.job.status = "running";
+            slot.job.startedAt = Date.now();
+            void slot
+                .run(slot.controller.signal, (bytes, ms) => {
+                slot.job.progress = { bytes, ms };
+            })
+                .then((result) => {
+                slot.job.result = result;
+                slot.job.status = slot.controller.signal.aborted ? "cancelled" : "done";
+            })
+                .catch((error) => {
+                slot.job.status = slot.controller.signal.aborted ? "cancelled" : "failed";
+                slot.job.error = error.message;
+            })
+                .finally(() => {
+                slot.job.finishedAt = Date.now();
+                this.running -= 1;
+                this.pump();
+            });
+        }
+    }
+    /** Forget finished jobs older than the TTL. */
+    sweep() {
+        const cutoff = Date.now() - this.ttlMs;
+        for (const [id, slot] of this.slots) {
+            if (slot.job.finishedAt !== undefined && slot.job.finishedAt < cutoff)
+                this.slots.delete(id);
+        }
+    }
+}

package/dist/compression/metrics.d.ts

@@ -0,0 +1,62 @@
+/**
+ * What was measured, per channel, and only what was measured.
+ *
+ * Input bytes are what the channel produced. Wire bytes are what went to
+ * relay listeners, headers included, counted once per listener because
+ * each listener is a separate cost on the network. Neither is a saving
+ * until it is compared with the other, and a block that was stored saved
+ * nothing -- these counters say so rather than counting what compression
+ * might have done.
+ */
+export type FallbackReason = "compression is off" | "already efficiently compressed" | "receiver does not support this format" | "processing budget exceeded" | "original source bytes are unavailable" | "slow listener";
+export interface ChannelMetricsSnapshot {
+    generation: number;
+    /** Bytes in from the channel since this generation began. */
+    inputBytes: number;
+    /** Payload bytes out, before fan-out: what one listener would receive. */
+    representationBytes: number;
+    /** Payload plus headers, summed over every listener that was sent it. */
+    wireBytes: number;
+    blocks: number;
+    storedBlocks: number;
+    compressedBlocks: number;
+    /** Which codec produced the most recent compressed block, or stored. */
+    activeMode: string;
+    bypassed: boolean;
+    bypassUntil: number | null;
+    /** From the first byte of a block entering to its being ready, milliseconds. */
+    latencyMs: {
+        last: number;
+        p50: number;
+        p95: number;
+        max: number;
+        samples: number;
+    };
+    queueBytes: number;
+    listeners: number;
+    droppedListeners: number;
+    codecFailures: number;
+    fallbackReason: FallbackReason | null;
+}
+export declare class ChannelMetrics {
+    generation: number;
+    inputBytes: number;
+    representationBytes: number;
+    wireBytes: number;
+    blocks: number;
+    storedBlocks: number;
+    compressedBlocks: number;
+    activeMode: string;
+    bypassed: boolean;
+    bypassUntil: number | null;
+    queueBytes: number;
+    listeners: number;
+    droppedListeners: number;
+    codecFailures: number;
+    fallbackReason: FallbackReason | null;
+    private latencies;
+    latency(ms: number): void;
+    /** A new run of the source: the per-generation counters start over. */
+    reset(generation: number): void;
+    snapshot(): ChannelMetricsSnapshot;
+}

package/dist/compression/metrics.js

@@ -0,0 +1,64 @@
+const LATENCY_WINDOW = 512;
+export class ChannelMetrics {
+    generation = 0;
+    inputBytes = 0;
+    representationBytes = 0;
+    wireBytes = 0;
+    blocks = 0;
+    storedBlocks = 0;
+    compressedBlocks = 0;
+    activeMode = "stored";
+    bypassed = false;
+    bypassUntil = null;
+    queueBytes = 0;
+    listeners = 0;
+    droppedListeners = 0;
+    codecFailures = 0;
+    fallbackReason = null;
+    latencies = [];
+    latency(ms) {
+        this.latencies.push(ms);
+        if (this.latencies.length > LATENCY_WINDOW)
+            this.latencies.shift();
+    }
+    /** A new run of the source: the per-generation counters start over. */
+    reset(generation) {
+        this.generation = generation;
+        this.inputBytes = 0;
+        this.representationBytes = 0;
+        this.wireBytes = 0;
+        this.blocks = 0;
+        this.storedBlocks = 0;
+        this.compressedBlocks = 0;
+        this.queueBytes = 0;
+        this.latencies = [];
+    }
+    snapshot() {
+        const sorted = [...this.latencies].sort((a, b) => a - b);
+        const at = (q) => (sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]);
+        return {
+            generation: this.generation,
+            inputBytes: this.inputBytes,
+            representationBytes: this.representationBytes,
+            wireBytes: this.wireBytes,
+            blocks: this.blocks,
+            storedBlocks: this.storedBlocks,
+            compressedBlocks: this.compressedBlocks,
+            activeMode: this.activeMode,
+            bypassed: this.bypassed,
+            bypassUntil: this.bypassUntil,
+            latencyMs: {
+                last: sorted.length === 0 ? 0 : this.latencies[this.latencies.length - 1],
+                p50: at(0.5),
+                p95: at(0.95),
+                max: sorted.length === 0 ? 0 : sorted[sorted.length - 1],
+                samples: sorted.length,
+            },
+            queueBytes: this.queueBytes,
+            listeners: this.listeners,
+            droppedListeners: this.droppedListeners,
+            codecFailures: this.codecFailures,
+            fallbackReason: this.fallbackReason,
+        };
+    }
+}

package/dist/compression/policy.d.ts

@@ -0,0 +1,76 @@
+/**
+ * What an operator can set, and what a block has to achieve to be sent
+ * compressed.
+ *
+ * Three settings that must never be confused: whether to squeeze the bytes
+ * losslessly, how to package HLS, and whether to re-encode the picture.
+ * Turning one on never turns another on. The first is the only one this
+ * module measures; the other two are carried here so that one document
+ * describes a channel, and so that the API can refuse a change that mixes
+ * them up.
+ */
+import { type Boundary } from "./envelope.ts";
+export type PolicyMode = "off" | "auto" | "zstd";
+export type HlsPackaging = "mpegts" | "fmp4";
+/** `source` is the media as it came. Anything else re-encodes and says so. */
+export type QualityProfile = "source";
+export interface LosslessPolicy {
+    mode: PolicyMode;
+    boundary: Boundary;
+    zstdLevel: number;
+    /** A block is sent compressed only if it saves at least this much, both ways. */
+    minSavingsPercent: number;
+    minSavingsBytes: number;
+    maxBlockBytes: number;
+    /** From the first byte entering a block to its being flushed, at most. */
+    maxHoldMs: number;
+    /** After `auto` has given up on a source, how long before it tries again. */
+    resampleAfterMs: number;
+    /** Bytes the shared compressor may hold for a channel before it stops that relay. */
+    maxChannelQueueBytes: number;
+    /** Unsent bytes one relay listener may fall behind by before it is cut off. */
+    maxListenerQueueBytes: number;
+    /** The experimental transport-stream transform. Off unless asked for. */
+    tsAware: boolean;
+}
+export interface ChannelPolicy {
+    losslessCompression: LosslessPolicy;
+    hlsPackaging: HlsPackaging;
+    qualityProfile: QualityProfile;
+    /** Bumped on every accepted change; a conditional update names the one it saw. */
+    version: number;
+}
+export declare const DEFAULT_LOSSLESS: LosslessPolicy;
+export declare const DEFAULT_POLICY: ChannelPolicy;
+/** Fresh copies, so nobody edits the defaults in place. */
+export declare function defaultPolicy(): ChannelPolicy;
+export type Normalized = {
+    ok: true;
+    policy: ChannelPolicy;
+} | {
+    ok: false;
+    errors: string[];
+};
+/**
+ * A change, applied over what is there, checked field by field.
+ *
+ * Unknown keys are errors rather than ignored: a typo that is silently
+ * dropped is a setting the operator believes is on and is not. The version
+ * is not settable here; the store bumps it when it accepts the result.
+ */
+export declare function normalizePolicy(input: unknown, base: ChannelPolicy): Normalized;
+/**
+ * Whether a compressed block earns its place.
+ *
+ * Both representations carry the same frame header, so the comparison of
+ * complete representations reduces to the payloads: the saving is what the
+ * original would have cost stored, less what the encoding costs. Both
+ * thresholds must hold. A tiny block can meet the percentage and still not
+ * be worth the decode; a huge one can save half a kilobyte and still be
+ * nothing.
+ */
+export declare function eligible(originalLength: number, encodedLength: number, policy: Pick<LosslessPolicy, "minSavingsPercent" | "minSavingsBytes">): boolean;
+/** What a block costs on the wire in each form, header included. */
+export declare function wireBytes(payloadLength: number): number;
+/** Whether two policies would produce the same bytes for the same input. */
+export declare function variantOf(policy: LosslessPolicy): string;