nixamp 0.7.34 → 0.7.35

package/dist/channels.d.ts

@@ -11,11 +11,23 @@ export interface ChannelInfo {
     /** The container it is sending, e.g. webm from a browser, flv over RTMP. */
     format: string;
     /** How it arrived. */
-    via: "http" | "rtmp";
+    via: "http" | "rtmp" | "pull";
     startedAt: number;
     bytes: number;
     listeners: number;
+    /**
+     * Whether there is a picture, which decides what a listener is sent and
+     * what the response calls it. A channel that says audio/mpeg while sending
+     * MP4 plays as nothing at all.
+     */
+    kind?: "audio" | "video";
+    /** For a channel we pull ourselves: where from. Never shown to a listener. */
+    source?: string;
 }
+/** How long to wait before dialling a dropped source again. */
+export declare const REDIAL = 2000;
+/** How many times in a row a source may fail without ever sending anything. */
+export declare const GIVE_UP = 5;
 /** A name that can sit in a URL and be read back in a list. */
 export declare function cleanId(value: unknown, fallback?: string): string;
 export interface ChannelOptions {
@@ -36,8 +48,44 @@ export declare class Channel {
     readonly listeners: Set<Listener>;
     private child;
     private closing;
+    /** Set for a channel that carries pictures, which cannot be joined blind. */
+    private fragments;
+    /** For a pulled channel: what to run, and how many times it has failed. */
+    private redial;
+    private failures;
+    private timer;
     constructor(info: ChannelInfo, options: ChannelOptions, onGone: (id: string) => void);
     start(format: string): void;
+    /**
+     * Fetch a source ourselves, rather than waiting to be sent one.
+     *
+     * This is what makes a re-stream a channel instead of a track. A track is
+     * played by the one player a server has, so a second one is a second thing
+     * that server cannot do at the same time; a channel is its own process with
+     * its own audience, and a server can carry as many as it can decode. Two
+     * channels means two tabs, or two panels of a multiview.
+     *
+     * It keeps running with nobody listening. Live television does not pause
+     * because you looked away, and a room where the picture depends on who is
+     * in it is not a room anybody can be invited to.
+     */
+    pull(source: string, encode: string[], paced?: boolean): void;
+    /**
+     * A source that stopped. Try it again, unless it never worked at all.
+     *
+     * The difference matters: a channel that ran for six hours and dropped is
+     * worth dialling again, and a URL that has never once produced a byte is a
+     * mistake somebody made, and retrying it for ever helps nobody.
+     */
+    private dropped;
+    /**
+     * Out to the audience, whole boxes at a time when there are boxes.
+     *
+     * Video listeners are only ever sent complete boxes, so that a new one can
+     * be given the opening boxes and then join at the next fragment and have it
+     * make sense.
+     */
+    private emit;
     /** Feed the source. */
     write(chunk: Buffer): boolean;
     pump(body: Readable): Promise<void>;
@@ -75,6 +123,16 @@ export declare class Channels {
      * *different* channel is exactly what this class exists for.
      */
     publish(id: string, name: string, format: string, via: ChannelInfo["via"]): Channel | null;
+    /**
+     * Carry a source of our own: a re-stream, or a film on this disk shown live.
+     *
+     * Null when that channel is taken, the same as publishing. Everything else
+     * about it is the same too, which is the point -- a re-stream stops being a
+     * special case and becomes one more thing that is on.
+     */
+    pull(id: string, name: string, source: string, encode: string[], kind: "audio" | "video", paced?: boolean): Channel | null;
+    /** What a listener should be told this channel is. */
+    contentType(id: string): string;
     /** Attach a listener, or null when nothing is playing on that channel. */
     listen(id: string, listener: Listener): (() => void) | null;
     /** Feed a channel that already exists, for a publisher sending chunks. */

package/dist/channels.js

@@ -16,6 +16,11 @@
  */
 import { spawn } from "node:child_process";
 import { randomBytes } from "node:crypto";
+import { Fragments } from "./fragments.js";
+/** How long to wait before dialling a dropped source again. */
+export const REDIAL = 2000;
+/** How many times in a row a source may fail without ever sending anything. */
+export const GIVE_UP = 5;
 /** A name that can sit in a URL and be read back in a list. */
 export function cleanId(value, fallback = "main") {
     if (typeof value !== "string")
@@ -36,6 +41,12 @@ export class Channel {
     listeners = new Set();
     child = null;
     closing = false;
+    /** Set for a channel that carries pictures, which cannot be joined blind. */
+    fragments = null;
+    /** For a pulled channel: what to run, and how many times it has failed. */
+    redial = null;
+    failures = 0;
+    timer = null;
     constructor(info, options, onGone) {
         this.info = info;
         this.options = options;
@@ -70,6 +81,97 @@ export class Channel {
         this.child = child;
         this.options.onStart?.(this.info);
     }
+    /**
+     * Fetch a source ourselves, rather than waiting to be sent one.
+     *
+     * This is what makes a re-stream a channel instead of a track. A track is
+     * played by the one player a server has, so a second one is a second thing
+     * that server cannot do at the same time; a channel is its own process with
+     * its own audience, and a server can carry as many as it can decode. Two
+     * channels means two tabs, or two panels of a multiview.
+     *
+     * It keeps running with nobody listening. Live television does not pause
+     * because you looked away, and a room where the picture depends on who is
+     * in it is not a room anybody can be invited to.
+     */
+    pull(source, encode, paced = true) {
+        if (this.info.kind === "video")
+            this.fragments = new Fragments();
+        const [command, ...prefix] = this.options.ffmpeg;
+        const remote = /^https?:\/\//i.test(source);
+        const dial = () => {
+            if (this.closing)
+                return;
+            const child = spawn(command, [
+                ...prefix,
+                "-hide_banner",
+                "-loglevel", "error",
+                // A dropped source is normal over hours, and a channel that dies
+                // the first time a CDN hiccups is not a channel anybody can rely
+                // on. ffmpeg redials on its own before we have to.
+                ...(remote ? ["-reconnect", "1", "-reconnect_streamed", "1", "-reconnect_delay_max", "5"] : []),
+                // Real time, always. A file read as fast as the disk allows is an
+                // hour of film in ninety seconds and a room that cannot be in it
+                // together; a live source is already paced and loses nothing.
+                ...(paced ? ["-re"] : []),
+                "-i", source,
+                ...encode,
+                "pipe:1",
+            ], { stdio: ["ignore", "pipe", "pipe"] });
+            let sent = false;
+            child.stdout?.on("data", (chunk) => {
+                sent = true;
+                this.info.bytes += chunk.byteLength;
+                this.emit(chunk);
+            });
+            child.stdout?.on("error", () => undefined);
+            child.on("error", () => this.dropped(sent));
+            child.on("close", () => this.dropped(sent));
+            this.child = child;
+        };
+        this.redial = dial;
+        dial();
+        this.options.onStart?.(this.info);
+    }
+    /**
+     * A source that stopped. Try it again, unless it never worked at all.
+     *
+     * The difference matters: a channel that ran for six hours and dropped is
+     * worth dialling again, and a URL that has never once produced a byte is a
+     * mistake somebody made, and retrying it for ever helps nobody.
+     */
+    dropped(sent) {
+        if (this.closing || !this.redial)
+            return;
+        this.child = null;
+        this.failures = sent ? 0 : this.failures + 1;
+        if (this.failures >= GIVE_UP) {
+            this.close();
+            return;
+        }
+        const dial = this.redial;
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            dial();
+        }, REDIAL);
+        // A redial is not a reason to keep the process alive at exit.
+        this.timer.unref?.();
+    }
+    /**
+     * Out to the audience, whole boxes at a time when there are boxes.
+     *
+     * Video listeners are only ever sent complete boxes, so that a new one can
+     * be given the opening boxes and then join at the next fragment and have it
+     * make sense.
+     */
+    emit(chunk) {
+        if (!this.fragments) {
+            this.send(chunk);
+            return;
+        }
+        for (const box of this.fragments.push(chunk))
+            this.send(box);
+    }
     /** Feed the source. */
     write(chunk) {
         return this.child?.stdin?.write(chunk) ?? false;
@@ -106,6 +208,17 @@ export class Channel {
         this.info.listeners = this.listeners.size;
     }
     listen(listener) {
+        // What the stream is, before any of what it is currently saying. Without
+        // this a listener who arrives after the first second gets fragments that
+        // reference tracks they were never told about: a blank panel, no error.
+        if (this.fragments?.ready) {
+            try {
+                listener.write(this.fragments.header);
+            }
+            catch {
+                // Gone before it began; the detach below still tidies up.
+            }
+        }
         this.listeners.add(listener);
         this.info.listeners = this.listeners.size;
         return () => {
@@ -117,6 +230,10 @@ export class Channel {
         if (this.closing)
             return;
         this.closing = true;
+        this.redial = null;
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = null;
         const child = this.child;
         this.child = null;
         try {
@@ -194,6 +311,35 @@ export class Channels {
         channel.start(format);
         return channel;
     }
+    /**
+     * Carry a source of our own: a re-stream, or a film on this disk shown live.
+     *
+     * Null when that channel is taken, the same as publishing. Everything else
+     * about it is the same too, which is the point -- a re-stream stops being a
+     * special case and becomes one more thing that is on.
+     */
+    pull(id, name, source, encode, kind, paced = true) {
+        if (this.open.has(id))
+            return null;
+        const channel = new Channel({
+            id,
+            name: name || source,
+            format: kind === "video" ? "mp4" : "mp3",
+            via: "pull",
+            startedAt: Date.now(),
+            bytes: 0,
+            listeners: 0,
+            kind,
+            source,
+        }, this.options, (gone) => this.open.delete(gone));
+        this.open.set(id, channel);
+        channel.pull(source, encode, paced);
+        return channel;
+    }
+    /** What a listener should be told this channel is. */
+    contentType(id) {
+        return this.open.get(id)?.info.kind === "video" ? "video/mp4" : "audio/mpeg";
+    }
     /** Attach a listener, or null when nothing is playing on that channel. */
     listen(id, listener) {
         const channel = this.open.get(id);

package/dist/fragments.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Splitting a fragmented MP4 into the pieces a late arrival needs.
+ *
+ * MP3 can be joined halfway through because every frame says what it is: a
+ * player finds the next frame boundary and carries on. Fragmented MP4 cannot.
+ * It opens with an `ftyp` and a `moov` that describe the tracks -- how many,
+ * which codecs, what timescale -- and everything after that is a `moof` and an
+ * `mdat` that mean nothing without them. Hand somebody the middle of that
+ * stream and their browser has no idea what it is holding, which is a black
+ * panel and no error.
+ *
+ * So the opening boxes are kept, and a listener who arrives an hour late is
+ * given them before the live bytes. Fragments written with `frag_keyframe`
+ * each begin at a keyframe, so the picture starts at the first one rather than
+ * with a screen of blocks catching up.
+ *
+ * This also means listeners are only ever written whole boxes. A chunk from a
+ * pipe ends wherever the pipe felt like ending it, and half a `moof` is not
+ * something to send anybody.
+ */
+export interface Box {
+    type: string;
+    bytes: Buffer;
+}
+/**
+ * The first whole box in a buffer, or null when it has not all arrived.
+ *
+ * Null is also the answer for anything malformed, because the difference does
+ * not matter to a caller who can only wait or give up, and guessing at a
+ * broken length walks off the end of the stream.
+ */
+export declare function firstBox(buffer: Buffer): {
+    box: Box;
+    rest: Buffer;
+} | null;
+/** The boxes that describe the stream rather than carry it. */
+export declare function isOpening(type: string): boolean;
+/**
+ * A fragmented MP4 arriving in pieces, handed back a box at a time.
+ *
+ * Keeps the opening boxes so they can be replayed to whoever turns up later.
+ * If the bytes turn out not to be an MP4 at all -- a source that failed, a
+ * format nobody expected -- it stops trying to parse and passes them through,
+ * on the grounds that a stream somebody might be able to play beats a stream
+ * nobody can.
+ */
+export declare class Fragments {
+    private held;
+    private opening;
+    private confused;
+    /** The `ftyp` and `moov` seen so far, ready to send to a new listener. */
+    get header(): Buffer;
+    /** Whether enough has arrived to describe the stream to somebody new. */
+    get ready(): boolean;
+    /** Feed bytes in; get whole boxes out, in order. */
+    push(chunk: Buffer): Buffer[];
+}

package/dist/fragments.js

@@ -0,0 +1,109 @@
+/**
+ * Splitting a fragmented MP4 into the pieces a late arrival needs.
+ *
+ * MP3 can be joined halfway through because every frame says what it is: a
+ * player finds the next frame boundary and carries on. Fragmented MP4 cannot.
+ * It opens with an `ftyp` and a `moov` that describe the tracks -- how many,
+ * which codecs, what timescale -- and everything after that is a `moof` and an
+ * `mdat` that mean nothing without them. Hand somebody the middle of that
+ * stream and their browser has no idea what it is holding, which is a black
+ * panel and no error.
+ *
+ * So the opening boxes are kept, and a listener who arrives an hour late is
+ * given them before the live bytes. Fragments written with `frag_keyframe`
+ * each begin at a keyframe, so the picture starts at the first one rather than
+ * with a screen of blocks catching up.
+ *
+ * This also means listeners are only ever written whole boxes. A chunk from a
+ * pipe ends wherever the pipe felt like ending it, and half a `moof` is not
+ * something to send anybody.
+ */
+/** The header of an MP4 box: four bytes of length, four of name. */
+const HEADER = 8;
+/** A length of 1 means the real one is the eight bytes that follow. */
+const BIG = 16;
+/**
+ * The first whole box in a buffer, or null when it has not all arrived.
+ *
+ * Null is also the answer for anything malformed, because the difference does
+ * not matter to a caller who can only wait or give up, and guessing at a
+ * broken length walks off the end of the stream.
+ */
+export function firstBox(buffer) {
+    if (buffer.length < HEADER)
+        return null;
+    const stated = buffer.readUInt32BE(0);
+    const type = buffer.toString("latin1", 4, HEADER);
+    // A name is four printable characters. Anything else means we are not
+    // looking at a box header, and no length read from here can be trusted.
+    if (!/^[\x20-\x7e]{4}$/.test(type))
+        return null;
+    let size = stated;
+    let header = HEADER;
+    if (stated === 1) {
+        if (buffer.length < BIG)
+            return null;
+        const large = buffer.readBigUInt64BE(HEADER);
+        if (large > BigInt(Number.MAX_SAFE_INTEGER))
+            return null;
+        size = Number(large);
+        header = BIG;
+    }
+    // Zero means "to the end of the file", which a live stream does not have.
+    if (size < header)
+        return null;
+    if (buffer.length < size)
+        return null;
+    return { box: { type, bytes: buffer.subarray(0, size) }, rest: buffer.subarray(size) };
+}
+/** The boxes that describe the stream rather than carry it. */
+export function isOpening(type) {
+    return type === "ftyp" || type === "moov";
+}
+/**
+ * A fragmented MP4 arriving in pieces, handed back a box at a time.
+ *
+ * Keeps the opening boxes so they can be replayed to whoever turns up later.
+ * If the bytes turn out not to be an MP4 at all -- a source that failed, a
+ * format nobody expected -- it stops trying to parse and passes them through,
+ * on the grounds that a stream somebody might be able to play beats a stream
+ * nobody can.
+ */
+export class Fragments {
+    held = Buffer.alloc(0);
+    opening = [];
+    confused = false;
+    /** The `ftyp` and `moov` seen so far, ready to send to a new listener. */
+    get header() {
+        return this.opening.length === 0 ? Buffer.alloc(0) : Buffer.concat(this.opening);
+    }
+    /** Whether enough has arrived to describe the stream to somebody new. */
+    get ready() {
+        return this.opening.length > 0;
+    }
+    /** Feed bytes in; get whole boxes out, in order. */
+    push(chunk) {
+        if (this.confused)
+            return [chunk];
+        this.held = this.held.length === 0 ? chunk : Buffer.concat([this.held, chunk]);
+        const out = [];
+        for (;;) {
+            const next = firstBox(this.held);
+            if (!next)
+                break;
+            this.held = next.rest;
+            if (isOpening(next.box.type))
+                this.opening.push(next.box.bytes);
+            out.push(next.box.bytes);
+        }
+        // Nothing parses and the buffer keeps growing: this is not an MP4. Let it
+        // through rather than swallowing a stream into memory for ever.
+        if (this.opening.length === 0 && this.held.length > 4 * 1024 * 1024) {
+            this.confused = true;
+            const everything = this.held;
+            this.held = Buffer.alloc(0);
+            return [everything];
+        }
+        return out;
+    }
+}

package/dist/server.js

@@ -19,7 +19,7 @@ import { readFileSync } from "node:fs";
 import { Connections } from "./connections.js";
 import { Broadcaster, DEFAULT_ENCODER, PRESETS, redact, } from "./broadcast.js";
 import { Ingest, normaliseFormat } from "./ingest.js";
-import { Channels, cleanId } from "./channels.js";
+import { Channels, cleanId, generatedId } from "./channels.js";
 import { RtmpListeners } from "./rtmp-in.js";
 import { Accounts, clearedCookie, sessionCookie, tokenFrom } from "./accounts.js";
 import { anonymousHandle, Handles } from "./handles.js";
@@ -1810,17 +1810,32 @@ export function createHandler(engine, options) {
             if (action === undefined && request.method === "GET") {
                 // Listening. The response is the fan-out target: whatever ffmpeg
                 // produces for this channel is written to it until one end goes away.
-                const detach = channels.listen(id, response);
-                if (detach === null) {
+                if (!channels.has(id)) {
                     json(response, 404, { error: "nothing is playing on that channel" });
                     return;
                 }
                 watch(request, response, "stream", id);
+                // Headers first, and then the listener.
+                //
+                // Attaching first was fine while a channel only ever wrote future
+                // bytes. A video channel writes the opening boxes to a new listener
+                // the moment it joins, and those went out before this response had
+                // any headers at all -- so it committed as a bare 200 with no
+                // content-type, ended immediately, and the picture was one kilobyte
+                // long. Whether it happened depended on whether ffmpeg had produced
+                // its header yet, which is why it looked intermittent.
                 response.writeHead(200, {
                     ...CORS,
-                    "content-type": "audio/mpeg",
+                    // Asked of the channel rather than assumed: a channel carrying
+                    // pictures that calls itself audio/mpeg plays as nothing at all.
+                    "content-type": channels.contentType(id),
                     "cache-control": "no-store",
                 });
+                const detach = channels.listen(id, response);
+                if (detach === null) {
+                    response.end();
+                    return;
+                }
                 const leave = () => detach();
                 request.on("close", leave);
                 response.on("close", leave);
@@ -1837,6 +1852,64 @@ export function createHandler(engine, options) {
                 json(response, 405, { error: "GET, POST or DELETE" });
                 return;
             }
+            /**
+             * Carry a source of our own, rather than waiting to be sent one.
+             *
+             * A re-stream used to be added to the playlist, where it became one
+             * more track -- and a server plays one track at a time, so the second
+             * channel you added sat there saying "stopped". Two channels are two
+             * processes with two audiences and two addresses, which is what lets
+             * one person watch the baseball while another watches the news, in two
+             * tabs or in two panels of the same multiview.
+             */
+            if (action === "pull") {
+                let source = "";
+                let called = "";
+                try {
+                    const body = JSON.parse(await readBody(request));
+                    called = String(body.name ?? "").replace(/[\u0000-\u001f\u007f]/g, " ").trim().slice(0, 80);
+                    // A track number rather than a path: it is the server's own library
+                    // either way, and a number cannot name a file outside it.
+                    if (typeof body.at === "number" && Number.isInteger(body.at) && body.at >= 0) {
+                        source = engine.trackPath(body.at) ?? "";
+                        if (source === "") {
+                            json(response, 404, { error: "no track there" });
+                            return;
+                        }
+                    }
+                    else {
+                        source = String(body.source ?? "").trim();
+                    }
+                }
+                catch {
+                    json(response, 400, { error: "bad JSON" });
+                    return;
+                }
+                if (source === "") {
+                    json(response, 400, { error: "give a URL to carry, or a track to show" });
+                    return;
+                }
+                const wanted = cleanId(rawId, generatedId());
+                if (channels.has(wanted)) {
+                    json(response, 409, { error: "that channel is already on" });
+                    return;
+                }
+                const probe = options.ffprobe ?? ["ffprobe"];
+                const codecs = await codecsOf({ ffmpeg: [], ffprobe: probe, play: null }, source);
+                const kind = codecs.video === "" ? "audio" : "video";
+                const encode = kind === "video"
+                    ? videoArgs(codecs)
+                    // No picture in it, so none is invented: MP3 is the thing every
+                    // browser plays and the thing a listener can join halfway through.
+                    : ["-vn", "-c:a", "libmp3lame", "-b:a", "192k", "-f", "mp3"];
+                const channel = channels.pull(wanted, called, source, encode, kind);
+                if (!channel) {
+                    json(response, 409, { error: "that channel is already on" });
+                    return;
+                }
+                json(response, 200, { ok: true, channel: channel.info });
+                return;
+            }
             const format = normaliseFormat(url.searchParams.get("format") ?? request.headers["content-type"]);
             if (format === null) {
                 json(response, 415, { error: "give a container ffmpeg knows: webm, ogg, mp4, mp3, wav" });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "nixamp",
-  "version": "0.7.34",
+  "version": "0.7.35",
   "description": "It really whips the terminal's ass. A Winamp-shaped audio player for your terminal.",
   "license": "MIT",
   "type": "module",