nixamp 0.17.1 → 0.18.0

package/dist/channels.d.ts

@@ -64,6 +64,13 @@ export interface ChannelInfo {
      * a member may have on at once.
      */
     startedBy?: string;
+    /**
+     * Set when nixamp reads the source itself and hands ffmpeg the bytes down
+     * a pipe, so the original bytes pass through this process and can be
+     * relayed exactly as they came. Only a transport stream from a file or a
+     * plain URL is read this way, and only when a policy asks for it.
+     */
+    teed?: boolean;
 }
 /** Where a pulled source is picked up from, and whether it can be at all. */
 export interface PullResume {
@@ -72,6 +79,22 @@ export interface PullResume {
     /** Seconds into a film to start from. */
     position: number;
 }
+/**
+ * A source read by us rather than by ffmpeg: what to tell ffmpeg it is,
+ * and how to open it. Opened once per dial; the signal is pulled when that
+ * dial is over.
+ */
+export interface PullThrough {
+    /** ffmpeg's name for the container, e.g. `mpegts`. */
+    format: string;
+    open(signal: AbortSignal): Promise<AsyncIterable<Uint8Array>>;
+}
+/**
+ * Asked at every dial whether this source should be read through us. Null
+ * means ffmpeg dials it as it always has. `from` is where a film is being
+ * picked up from; a source read through us cannot be joined mid-way.
+ */
+export type ThroughProvider = (info: ChannelInfo, from: number, input: string[], audio: string) => PullThrough | null;
 /**
  * How far back of the saved place a film is picked up from, in seconds. The
  * place is written down every so often and a restart lands between two
@@ -144,6 +167,8 @@ export interface ChannelOptions {
     maxListenerQueueBytes?: number;
     /** How long a rate is measured over before the backlog is sized off it. Tests shorten it. */
     rateWindowMs?: number;
+    /** Whether a pulled source is read here and piped to ffmpeg. See `Channels.setThrough`. */
+    through?: ThroughProvider;
 }
 /**
  * One live source, and its audience.
@@ -192,6 +217,10 @@ export declare class Channel {
      */
     ephemeral: boolean;
     private idle;
+    /** Whoever wants the source's own bytes, when the source is read through us. */
+    private readonly sourceTaps;
+    /** Pulls the plug on the current read-through, when there is one. */
+    private throughAbort;
     constructor(info: ChannelInfo, options: ChannelOptions, onGone: (id: string) => void);
     start(format: string): void;
     /**
@@ -230,6 +259,8 @@ export declare class Channel {
      * new beginning as it is written, rather than a stale one first.
      */
     private startOver;
+    /** Everybody tapping the source is told it ended. */
+    private endTaps;
     /** Expect output within STALL, or treat the source as gone and dial again. */
     private rearm;
     /** End everybody listening; the stream they were on is over. */
@@ -298,6 +329,21 @@ export declare class Channel {
      * own ffmpeg is dialled again, and a newcomer gets the new beginning.
      */
     rollover(): void;
+    /**
+     * Hear the source's own bytes, as they go down the pipe to ffmpeg. Only
+     * a channel read through us has any; for the rest this attaches nothing
+     * and returns null. Ended, like a listener, when the source starts over.
+     */
+    tapSource(tap: Listener): (() => void) | null;
+    private tap;
+    /**
+     * Read the source and write it to this ffmpeg's stdin, at the rate ffmpeg
+     * takes it. Every chunk is handed to the taps first, so a tap sees exactly
+     * the bytes ffmpeg does. When the source ends, stdin is ended, ffmpeg
+     * finishes, and its close handler dials again -- the same path a source
+     * that ffmpeg read itself takes when it drops.
+     */
+    private feedThrough;
     listen(listener: Listener): () => void;
     /** Stay up with nobody watching: no longer on demand. */
     keep(): void;
@@ -337,7 +383,7 @@ export declare class Channels {
      * about it is the same too, which is the point -- a re-stream stops being a
      * special case and becomes one more thing that is on.
      */
-    pull(id: string, name: string, source: string, encode: string[], kind: "audio" | "video", paced?: boolean, stall?: number, input?: string[], audio?: string, resume?: PullResume): Channel | null;
+    pull(id: string, name: string, source: string, encode: string[], kind: "audio" | "video", paced?: boolean, stall?: number, input?: string[], audio?: string, resume?: PullResume, codecs?: ChannelInfo["codecs"]): Channel | null;
     /**
      * Dial a pulled channel's source again, now. False for a channel that is
      * not there or is not ours to dial: a publisher's stream restarts at the
@@ -382,6 +428,13 @@ export declare class Channels {
     relayIn(id: string, name: string, kind: "audio" | "video", source: string): Channel | null;
     /** What a new listener would be written first, for a relay's preface. */
     opening(id: string): Buffer[];
+    /** Hear a channel's source bytes, when it is read through us. Null otherwise. */
+    tapSource(id: string, tap: Listener): (() => void) | null;
+    /**
+     * Who decides whether a pulled source is read here and piped to ffmpeg.
+     * Set once by whoever owns the policies; asked at every dial.
+     */
+    setThrough(provider: ThroughProvider | null): void;
     /** The kind of a channel, for a relay to say what it is carrying. */
     kindOf(id: string): "audio" | "video" | undefined;
     stop(id: string): boolean;

package/dist/channels.js

@@ -163,6 +163,10 @@ export class Channel {
      */
     ephemeral = false;
     idle = null;
+    /** Whoever wants the source's own bytes, when the source is read through us. */
+    sourceTaps = new Set();
+    /** Pulls the plug on the current read-through, when there is one. */
+    throughAbort = null;
     constructor(info, options, onGone) {
         this.info = info;
         this.options = options;
@@ -236,6 +240,15 @@ export class Channel {
             // and a film that has barely started is started.
             const from = resume.live ? 0 : Math.max(0, Math.floor((this.info.position ?? 0) - REWIND));
             const seek = from > 0 ? ["-ss", String(from)] : [];
+            // Read the source here rather than in ffmpeg, when a policy wants the
+            // original bytes and the source is the kind that can be. ffmpeg then
+            // reads a pipe, and every byte that goes down it is also handed to
+            // whoever has tapped the source. Asked again at every dial, so a
+            // policy set after the channel started applies at its next restart.
+            this.throughAbort?.abort();
+            this.throughAbort = null;
+            const through = this.options.through?.(this.info, from, input, audio) ?? null;
+            this.info.teed = through !== null;
             const child = spawn(command, [
                 ...prefix,
                 "-hide_banner",
@@ -245,33 +258,52 @@ export class Channel {
                 // to. Not stderr, which is for what went wrong.
                 "-progress", "pipe:3",
                 "-stats_period", "1",
-                // A dropped source is normal over hours, and a channel that dies
-                // the first time a CDN hiccups is not a channel anybody can rely
-                // on. ffmpeg redials on its own before we have to.
-                // A connection that stops answering is an error after this long,
-                // and an error is a thing the reconnect knows what to do with.
-                // Without it a silent socket is waited on for ever. In
-                // microseconds, as ffmpeg wants it.
-                ...remoteArgs,
-                // Real time, always. A file read as fast as the disk allows is an
-                // hour of film in ninety seconds and a room that cannot be in it
-                // together; a live source is already paced and loses nothing.
-                ...(paced ? ["-re"] : []),
-                // What the source's site expects on the request: a user agent, a
-                // referer, a cookie. A link resolved by yt-dlp comes with these,
-                // and a CDN that got them from yt-dlp and not from us answers 403.
-                ...input,
-                ...seek,
-                "-i", source,
-                // The sound, when the site keeps it apart from the picture: a
-                // second input, dialled the same way, that the encode maps in.
-                ...(audio ? [...remoteArgs, ...(paced ? ["-re"] : []), ...input, ...seek, "-i", audio] : []),
+                ...(through
+                    ? [
+                        // Stated, as for a publisher: ffmpeg mis-probes a pipe. Paced
+                        // the same way, since a pipe is read as fast as it is written.
+                        // The source's own input tuning still applies -- a transport
+                        // stream read from a pipe needs the same probe depth and
+                        // generated timestamps it would off a socket -- but request
+                        // headers, which only a source read through us leaves out,
+                        // are no use to a pipe and are not here (a source that needs
+                        // them is not read through us in the first place).
+                        ...input,
+                        "-f", through.format,
+                        ...(paced ? ["-re"] : []),
+                        "-i", "pipe:0",
+                    ]
+                    : [
+                        // A dropped source is normal over hours, and a channel that dies
+                        // the first time a CDN hiccups is not a channel anybody can rely
+                        // on. ffmpeg redials on its own before we have to.
+                        // A connection that stops answering is an error after this long,
+                        // and an error is a thing the reconnect knows what to do with.
+                        // Without it a silent socket is waited on for ever. In
+                        // microseconds, as ffmpeg wants it.
+                        ...remoteArgs,
+                        // Real time, always. A file read as fast as the disk allows is an
+                        // hour of film in ninety seconds and a room that cannot be in it
+                        // together; a live source is already paced and loses nothing.
+                        ...(paced ? ["-re"] : []),
+                        // What the source's site expects on the request: a user agent, a
+                        // referer, a cookie. A link resolved by yt-dlp comes with these,
+                        // and a CDN that got them from yt-dlp and not from us answers 403.
+                        ...input,
+                        ...seek,
+                        "-i", source,
+                        // The sound, when the site keeps it apart from the picture: a
+                        // second input, dialled the same way, that the encode maps in.
+                        ...(audio ? [...remoteArgs, ...(paced ? ["-re"] : []), ...input, ...seek, "-i", audio] : []),
+                    ]),
                 ...encode,
                 "pipe:1",
-            ], { stdio: ["ignore", "pipe", "pipe", "pipe"] });
+            ], { stdio: [through ? "pipe" : "ignore", "pipe", "pipe", "pipe"] });
             let sent = false;
             this.child = child;
             this.rearm(child);
+            if (through)
+                void this.feedThrough(child, through);
             // ffmpeg's progress: key=value lines, out_time_us being how much it
             // has written, from where it was told to start. Read whole lines,
             // since a chunk can end mid-number. Drained whatever it says, for
@@ -363,6 +395,22 @@ export class Channel {
         this.rateBytes = 0;
         this.rate = 0;
         this.hangUp();
+        // The source's own bytes start over too: a new dial is a new stream
+        // from its beginning, and whoever was tapping it must not be handed the
+        // new beginning after the old middle.
+        this.endTaps();
+    }
+    /** Everybody tapping the source is told it ended. */
+    endTaps() {
+        for (const tap of this.sourceTaps) {
+            try {
+                tap.end();
+            }
+            catch {
+                // Gone already.
+            }
+        }
+        this.sourceTaps.clear();
     }
     /** Expect output within STALL, or treat the source as gone and dial again. */
     rearm(child) {
@@ -572,6 +620,75 @@ export class Channel {
         this.info.redials = (this.info.redials ?? 0) + 1;
         this.startOver();
     }
+    /**
+     * Hear the source's own bytes, as they go down the pipe to ffmpeg. Only
+     * a channel read through us has any; for the rest this attaches nothing
+     * and returns null. Ended, like a listener, when the source starts over.
+     */
+    tapSource(tap) {
+        if (!this.info.teed || this.closing)
+            return null;
+        this.sourceTaps.add(tap);
+        return () => {
+            this.sourceTaps.delete(tap);
+        };
+    }
+    tap(chunk) {
+        for (const tap of this.sourceTaps) {
+            try {
+                tap.write(chunk);
+            }
+            catch {
+                this.sourceTaps.delete(tap);
+            }
+        }
+    }
+    /**
+     * Read the source and write it to this ffmpeg's stdin, at the rate ffmpeg
+     * takes it. Every chunk is handed to the taps first, so a tap sees exactly
+     * the bytes ffmpeg does. When the source ends, stdin is ended, ffmpeg
+     * finishes, and its close handler dials again -- the same path a source
+     * that ffmpeg read itself takes when it drops.
+     */
+    async feedThrough(child, through) {
+        const controller = new AbortController();
+        this.throughAbort = controller;
+        const stdin = child.stdin;
+        if (!stdin)
+            return;
+        stdin.on("error", () => undefined);
+        try {
+            const body = await through.open(controller.signal);
+            for await (const raw of body) {
+                if (this.child !== child || this.closing || controller.signal.aborted)
+                    break;
+                const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(raw);
+                this.tap(chunk);
+                if (!stdin.write(chunk)) {
+                    // Wait for ffmpeg to take it, or for the pipe to go: a pipe that
+                    // closed never drains, and waiting on it would hold the read open.
+                    await new Promise((done) => {
+                        stdin.once("drain", done);
+                        stdin.once("close", done);
+                    });
+                }
+            }
+        }
+        catch (error) {
+            if (this.child === child && !controller.signal.aborted)
+                this.info.error = error.message;
+        }
+        finally {
+            try {
+                stdin.end();
+            }
+            catch {
+                // Already gone.
+            }
+            if (this.throughAbort === controller)
+                this.throughAbort = null;
+        }
+    }
     listen(listener) {
         // What the stream is, before any of what it is currently saying. Without
         // this a listener who arrives after the first second gets fragments that
@@ -645,6 +762,9 @@ export class Channel {
             this.info.error = said;
         const child = this.child;
         this.child = null;
+        this.throughAbort?.abort();
+        this.throughAbort = null;
+        this.endTaps();
         try {
             child?.stdin?.end();
         }
@@ -731,7 +851,7 @@ export class Channels {
      * about it is the same too, which is the point -- a re-stream stops being a
      * special case and becomes one more thing that is on.
      */
-    pull(id, name, source, encode, kind, paced = true, stall = STALL, input = [], audio = "", resume = { live: true, position: 0 }) {
+    pull(id, name, source, encode, kind, paced = true, stall = STALL, input = [], audio = "", resume = { live: true, position: 0 }, codecs) {
         if (this.open.has(id))
             return null;
         const channel = new Channel({
@@ -744,6 +864,9 @@ export class Channels {
             listeners: 0,
             kind,
             source,
+            // Known before the first dial, so whether to read the source here
+            // can be decided from what it holds.
+            ...(codecs ? { codecs } : {}),
         }, this.options, (gone) => this.open.delete(gone));
         this.open.set(id, channel);
         channel.pull(source, encode, paced, stall, input, audio, resume);
@@ -841,6 +964,20 @@ export class Channels {
     opening(id) {
         return this.open.get(id)?.opening() ?? [];
     }
+    /** Hear a channel's source bytes, when it is read through us. Null otherwise. */
+    tapSource(id, tap) {
+        return this.open.get(id)?.tapSource(tap) ?? null;
+    }
+    /**
+     * Who decides whether a pulled source is read here and piped to ffmpeg.
+     * Set once by whoever owns the policies; asked at every dial.
+     */
+    setThrough(provider) {
+        if (provider)
+            this.options.through = provider;
+        else
+            delete this.options.through;
+    }
     /** The kind of a channel, for a relay to say what it is carrying. */
     kindOf(id) {
         return this.open.get(id)?.info.kind;

package/dist/compression/benchmark.d.ts

@@ -0,0 +1,97 @@
+import { type BenchRow } from "./analyze.ts";
+/** The report schema version, bumped when the shape below changes. */
+export declare const REPORT_SCHEMA = 1;
+export interface Sample {
+    name: string;
+    /** synthetic bytes we generated, or a real file the runner supplied. */
+    kind: "synthetic" | "real";
+    bytes: Buffer;
+    /** A word on what it is and why it is in the corpus. */
+    notes: string;
+}
+export interface SampleResult {
+    sample: string;
+    kind: Sample["kind"];
+    inputBytes: number;
+    sha256: string;
+    container: string;
+    rows: BenchRow[];
+    /** The mode the policy would pick for this sample, and why. */
+    recommendation: {
+        mode: string;
+        level: number;
+        reason: string;
+    };
+}
+export interface BenchmarkReport {
+    schema: number;
+    spec: "openstream";
+    specVersion: string;
+    generatedAt: string;
+    implementation: {
+        name: string;
+        version: string;
+    };
+    environment: {
+        runtime: string;
+        zstd: string;
+        zlib: string;
+        os: string;
+        arch: string;
+        cpu: string;
+        cores: number;
+        memoryGiB: number;
+    };
+    envelope: {
+        magic: string;
+        streamHeaderBytes: number;
+        frameHeaderBytes: number;
+    };
+    policy: {
+        minSavingsPercent: number;
+        minSavingsBytes: number;
+        maxBlockBytes: number;
+        zstdLevels: number[];
+    };
+    samples: SampleResult[];
+    /** Per mode, summed across the corpus: the honest aggregate. */
+    summary: {
+        corpusBytes: number;
+        byMode: {
+            mode: string;
+            level: number;
+            wireBytes: number;
+            savingsPercent: number;
+            roundTrip: boolean;
+            encodeMs: number;
+            decodeMs: number;
+        }[];
+    };
+    caveats: string[];
+}
+/** The default corpus: bytes only, no ffmpeg, deterministic. */
+export declare function syntheticCorpus(): Sample[];
+/** Real files a runner points us at, each read whole (capped) as a sample. */
+export declare function fileCorpus(dir: string, capBytes?: number): Sample[];
+export interface RunOptions {
+    implementationVersion: string;
+    zstdLevels?: number[];
+    blockBytes?: number;
+    signal?: AbortSignal;
+}
+/** Run the corpus and build the report. */
+export declare function runBenchmark(corpus: Sample[], options: RunOptions): Promise<BenchmarkReport>;
+/** The report as Markdown, for a human and for the reports page. */
+export declare function reportMarkdown(report: BenchmarkReport): string;
+/**
+ * Whether every applicable codec restored exactly: the gate a release must
+ * pass. A row with a note is a mode that did not apply to that sample, not a
+ * corrupted round trip, so it does not fail the gate.
+ */
+export declare function reportPasses(report: BenchmarkReport): boolean;
+/** The rows that are a real exactness failure: ran, and did not restore. */
+export declare function exactnessFailures(report: BenchmarkReport): {
+    sample: string;
+    mode: string;
+    level: number;
+}[];