nixamp 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/channels.d.ts +54 -1
- package/dist/channels.js +160 -23
- package/dist/compression/benchmark.d.ts +97 -0
- package/dist/compression/benchmark.js +241 -0
- package/dist/compression/cli.js +59 -1
- package/dist/compression/receiver.d.ts +25 -5
- package/dist/compression/receiver.js +54 -5
- package/dist/compression/routes.js +7 -2
- package/dist/compression/service.d.ts +18 -6
- package/dist/compression/service.js +110 -19
- package/dist/compression/source.d.ts +17 -0
- package/dist/compression/source.js +131 -0
- package/dist/links.js +14 -3
- package/dist/live-api.js +1 -0
- package/dist/live-events.js +11 -3
- package/dist/owner.js +10 -0
- package/dist/server.js +118 -10
- package/dist/share.js +6 -0
- package/package.json +1 -1
- package/src/channels.ts +180 -22
- package/src/compression/benchmark.ts +297 -0
- package/src/compression/cli.ts +56 -1
- package/src/compression/receiver.ts +68 -7
- package/src/compression/routes.ts +7 -2
- package/src/compression/service.ts +121 -24
- package/src/compression/source.ts +130 -0
- package/src/links.ts +12 -3
- package/src/live-api.ts +1 -0
- package/src/live-events.ts +14 -3
- package/src/owner.ts +8 -0
- package/src/server.ts +113 -11
- package/src/share.ts +5 -0
- package/web/dist/assets/{hls-3VKVEQE3-eV54kXE3.js → hls-3VKVEQE3-B2kl0-z1.js} +1 -1
- package/web/dist/assets/{index-CurZFzlH.css → index-D1QxpRmE.css} +1 -1
- package/web/dist/assets/index-DF6O2leR.js +1 -0
- package/web/dist/assets/{mpegts-Buc3Odv6.js → mpegts-4pBNK_Yr.js} +1 -1
- package/web/dist/assets/{mpegts-LO6RVLD6-DcDKPB4P.js → mpegts-LO6RVLD6-B4RLM-_e.js} +1 -1
- package/web/dist/index.html +15 -10
- package/web/dist/sw.js +6 -6
- package/web/dist/assets/index-DDzutJ75.js +0 -1
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The OpenStream benchmark: what a codec makes of a defined corpus, on this
|
|
3
|
+
* machine, with every number reproducible and nothing claimed that was not
|
|
4
|
+
* measured.
|
|
5
|
+
*
|
|
6
|
+
* It proves the honest things: that decompression restores every byte, that
|
|
7
|
+
* an incompressible sample costs only the envelope overhead and never more,
|
|
8
|
+
* that a compressible one saves what it says against the complete wire size,
|
|
9
|
+
* and how long each takes. It does not prove a production saving -- a
|
|
10
|
+
* synthetic padded stream compresses to almost nothing, which says more about
|
|
11
|
+
* the padding than the codec, and the report labels it so. Point it at real
|
|
12
|
+
* authorized samples with `--corpus` for numbers that mean something.
|
|
13
|
+
*
|
|
14
|
+
* The corpus is built from bytes alone by default, so anyone can run it with
|
|
15
|
+
* no ffmpeg and no media: random data, zeros, repetitive text, tiny and empty
|
|
16
|
+
* inputs, and hand-built transport-stream packets with a known share of null
|
|
17
|
+
* padding. ffmpeg, when present, adds real encoded media; a directory of your
|
|
18
|
+
* own files replaces the lot.
|
|
19
|
+
*/
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
22
|
+
import { arch, cpus, platform, release, totalmem } from "node:os";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { benchmark } from "./analyze.js";
|
|
25
|
+
import { toolVersions } from "./codec.js";
|
|
26
|
+
import { ENVELOPE_VERSION, FRAME_HEADER_BYTES, MAGIC, STREAM_HEADER_BYTES } from "./envelope.js";
|
|
27
|
+
import { DEFAULT_LOSSLESS } from "./policy.js";
|
|
28
|
+
import { SYNC, TS_PACKET } from "./ts-transform.js";
|
|
29
|
+
/** The report schema version, bumped when the shape below changes. */
|
|
30
|
+
export const REPORT_SCHEMA = 1;
|
|
31
|
+
/**
|
|
32
|
+
* A deterministic, genuinely incompressible fill: a SHA-256 keystream from a
|
|
33
|
+
* fixed seed. Reproducible byte-for-byte across runs (so two reports compare)
|
|
34
|
+
* and uncompressible (so the "floor" sample is really the floor, unlike a
|
|
35
|
+
* linear-congruential stream, whose periodicity a compressor crushes).
|
|
36
|
+
*/
|
|
37
|
+
function keystream(size, seed) {
|
|
38
|
+
const out = Buffer.alloc(size);
|
|
39
|
+
let block = createHash("sha256").update(seed).digest();
|
|
40
|
+
let at = 0;
|
|
41
|
+
while (at < size) {
|
|
42
|
+
const take = Math.min(block.length, size - at);
|
|
43
|
+
block.copy(out, at, 0, take);
|
|
44
|
+
at += take;
|
|
45
|
+
block = createHash("sha256").update(block).digest();
|
|
46
|
+
}
|
|
47
|
+
return out;
|
|
48
|
+
}
|
|
49
|
+
/** Transport-stream packets, a given share of them null padding, payloads incompressible. */
|
|
50
|
+
function fakeTs(packets, nullEvery, seed = "ts") {
|
|
51
|
+
const out = Buffer.alloc(packets * TS_PACKET);
|
|
52
|
+
const fill = keystream(packets * TS_PACKET, seed);
|
|
53
|
+
for (let i = 0; i < packets; i += 1) {
|
|
54
|
+
const at = i * TS_PACKET;
|
|
55
|
+
const isNull = nullEvery > 0 && i % nullEvery === 0;
|
|
56
|
+
const pid = isNull ? 0x1fff : [0x100, 0x101, 0x102][i % 3];
|
|
57
|
+
out[at] = SYNC;
|
|
58
|
+
out[at + 1] = (pid >> 8) & 0x1f;
|
|
59
|
+
out[at + 2] = pid & 0xff;
|
|
60
|
+
out[at + 3] = 0x10 | (i & 0xf);
|
|
61
|
+
for (let j = at + 4; j < at + TS_PACKET; j += 1)
|
|
62
|
+
out[j] = isNull ? 0xff : fill[j];
|
|
63
|
+
}
|
|
64
|
+
return out;
|
|
65
|
+
}
|
|
66
|
+
/** The default corpus: bytes only, no ffmpeg, deterministic. */
|
|
67
|
+
export function syntheticCorpus() {
|
|
68
|
+
return [
|
|
69
|
+
{ name: "random-1mib", kind: "synthetic", bytes: keystream(1024 * 1024, "random"), notes: "incompressible: the floor, where a codec must not grow the data beyond envelope overhead" },
|
|
70
|
+
{ name: "zeros-1mib", kind: "synthetic", bytes: Buffer.alloc(1024 * 1024, 0), notes: "maximally compressible: the ceiling" },
|
|
71
|
+
{ name: "text-repeat-1mib", kind: "synthetic", bytes: Buffer.from("the quick brown fox jumps over the lazy dog\n".repeat(24000)).subarray(0, 1024 * 1024), notes: "repetitive text: ordinary redundancy" },
|
|
72
|
+
{ name: "ts-padded-50pct", kind: "synthetic", bytes: fakeTs(4000, 2), notes: "transport stream, half null packets: padding a copy must keep and a codec removes; the synthetic case that flatters a codec" },
|
|
73
|
+
{ name: "ts-unpadded", kind: "synthetic", bytes: fakeTs(4000, 0), notes: "transport stream, no padding: closer to an efficient real feed" },
|
|
74
|
+
{ name: "tiny-3b", kind: "synthetic", bytes: Buffer.from("abc"), notes: "smaller than a frame header: proves overhead is reported honestly" },
|
|
75
|
+
{ name: "empty", kind: "synthetic", bytes: Buffer.alloc(0), notes: "the empty stream" },
|
|
76
|
+
];
|
|
77
|
+
}
|
|
78
|
+
/** Real files a runner points us at, each read whole (capped) as a sample. */
|
|
79
|
+
export function fileCorpus(dir, capBytes = 64 * 1024 * 1024) {
|
|
80
|
+
const out = [];
|
|
81
|
+
for (const name of readdirSync(dir).sort()) {
|
|
82
|
+
const path = join(dir, name);
|
|
83
|
+
try {
|
|
84
|
+
if (!statSync(path).isFile())
|
|
85
|
+
continue;
|
|
86
|
+
const whole = readFileSync(path);
|
|
87
|
+
out.push({ name, kind: "real", bytes: whole.subarray(0, capBytes), notes: whole.length > capBytes ? `real file, first ${capBytes} bytes of ${whole.length}` : "real file" });
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
// Unreadable entries are skipped, not fatal.
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
/** Run the corpus and build the report. */
|
|
96
|
+
export async function runBenchmark(corpus, options) {
|
|
97
|
+
const zstdLevels = options.zstdLevels ?? [1, 3, 9];
|
|
98
|
+
const blockBytes = options.blockBytes ?? DEFAULT_LOSSLESS.maxBlockBytes;
|
|
99
|
+
const policy = { minSavingsPercent: DEFAULT_LOSSLESS.minSavingsPercent, minSavingsBytes: DEFAULT_LOSSLESS.minSavingsBytes };
|
|
100
|
+
const samples = [];
|
|
101
|
+
for (const sample of corpus) {
|
|
102
|
+
if (options.signal?.aborted)
|
|
103
|
+
break;
|
|
104
|
+
const rows = await benchmark(sample.bytes, { blockBytes, zstdLevels, tsAware: true, policy, ...(options.signal ? { signal: options.signal } : {}) });
|
|
105
|
+
const stored = rows.find((r) => r.mode === "stored");
|
|
106
|
+
const best = rows.filter((r) => r.roundTrip && r.mode !== "stored").sort((a, b) => a.wireBytes - b.wireBytes)[0];
|
|
107
|
+
const beats = stored && best && stored.wireBytes - best.wireBytes >= policy.minSavingsBytes && ((stored.wireBytes - best.wireBytes) * 100) / stored.wireBytes >= policy.minSavingsPercent;
|
|
108
|
+
samples.push({
|
|
109
|
+
sample: sample.name,
|
|
110
|
+
kind: sample.kind,
|
|
111
|
+
inputBytes: sample.bytes.length,
|
|
112
|
+
sha256: createHash("sha256").update(sample.bytes).digest("hex"),
|
|
113
|
+
container: rows.length ? sniff(sample.bytes) : "empty",
|
|
114
|
+
rows,
|
|
115
|
+
recommendation: beats && best
|
|
116
|
+
? { mode: best.mode, level: best.level, reason: `saves ${best.savingsPercent}% of the complete wire size` }
|
|
117
|
+
: { mode: "stored", level: 0, reason: "no codec beat stored by the configured margin" },
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
const byMode = aggregate(samples);
|
|
121
|
+
const cpu = cpus()[0]?.model ?? "unknown";
|
|
122
|
+
return {
|
|
123
|
+
schema: REPORT_SCHEMA,
|
|
124
|
+
spec: "openstream",
|
|
125
|
+
specVersion: MAGIC,
|
|
126
|
+
generatedAt: new Date().toISOString(),
|
|
127
|
+
implementation: { name: "nixamp", version: options.implementationVersion },
|
|
128
|
+
environment: {
|
|
129
|
+
...toolVersions(),
|
|
130
|
+
os: `${platform()} ${release()}`,
|
|
131
|
+
arch: arch(),
|
|
132
|
+
cpu,
|
|
133
|
+
cores: cpus().length,
|
|
134
|
+
memoryGiB: Math.round((totalmem() / 1024 ** 3) * 10) / 10,
|
|
135
|
+
},
|
|
136
|
+
envelope: { magic: MAGIC, streamHeaderBytes: STREAM_HEADER_BYTES, frameHeaderBytes: FRAME_HEADER_BYTES },
|
|
137
|
+
policy: { ...policy, maxBlockBytes: blockBytes, zstdLevels },
|
|
138
|
+
samples,
|
|
139
|
+
summary: { corpusBytes: corpus.reduce((n, s) => n + s.bytes.length, 0), byMode },
|
|
140
|
+
caveats: [
|
|
141
|
+
`Envelope v${ENVELOPE_VERSION}: a ${STREAM_HEADER_BYTES}-byte stream header, a ${FRAME_HEADER_BYTES}-byte header per frame, plus one end frame.`,
|
|
142
|
+
"OpenStream is a framing envelope over Zstandard and gzip, not a new compression algorithm; these numbers are those codecs at the block boundary, honestly framed.",
|
|
143
|
+
"Synthetic samples do not predict production savings. A padded transport stream flatters a codec by its padding; an efficient real feed saves far less. Use --corpus with authorized real samples for numbers that mean something.",
|
|
144
|
+
"Timings are wall-clock on the machine and runtime named in `environment` and do not transfer to other hardware.",
|
|
145
|
+
"roundTrip:false in any row is a failure of exactness and must block a release.",
|
|
146
|
+
],
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
function sniff(bytes) {
|
|
150
|
+
// A light container guess for the report; the full analyzer is elsewhere.
|
|
151
|
+
if (bytes.length >= 8 && bytes.toString("latin1", 4, 8) === "ftyp")
|
|
152
|
+
return "mp4";
|
|
153
|
+
if (bytes.length >= TS_PACKET && bytes[0] === SYNC && bytes[TS_PACKET] === SYNC)
|
|
154
|
+
return "mpegts";
|
|
155
|
+
return "bytes";
|
|
156
|
+
}
|
|
157
|
+
function aggregate(samples) {
|
|
158
|
+
const modes = new Map();
|
|
159
|
+
for (const sample of samples) {
|
|
160
|
+
for (const row of sample.rows) {
|
|
161
|
+
// A row with a note is a mode that did not apply to this sample (ts-zstd
|
|
162
|
+
// on non-transport bytes, say). It is not a saving and not a failure, so
|
|
163
|
+
// it is left out of the aggregate rather than dragging a mode down.
|
|
164
|
+
if (row.note)
|
|
165
|
+
continue;
|
|
166
|
+
const key = `${row.mode}:${row.level}`;
|
|
167
|
+
const acc = modes.get(key) ?? { mode: row.mode, level: row.level, wireBytes: 0, input: 0, roundTrip: true, encodeMs: 0, decodeMs: 0 };
|
|
168
|
+
acc.wireBytes += row.wireBytes;
|
|
169
|
+
acc.input += row.inputBytes;
|
|
170
|
+
acc.roundTrip = acc.roundTrip && row.roundTrip;
|
|
171
|
+
acc.encodeMs += row.encodeMs;
|
|
172
|
+
acc.decodeMs += row.decodeMs;
|
|
173
|
+
modes.set(key, acc);
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
return [...modes.values()].map((m) => ({
|
|
177
|
+
mode: m.mode,
|
|
178
|
+
level: m.level,
|
|
179
|
+
wireBytes: m.wireBytes,
|
|
180
|
+
savingsPercent: m.input === 0 ? 0 : Math.round(((m.input - m.wireBytes) * 100) / m.input * 100) / 100,
|
|
181
|
+
roundTrip: m.roundTrip,
|
|
182
|
+
encodeMs: Math.round(m.encodeMs * 100) / 100,
|
|
183
|
+
decodeMs: Math.round(m.decodeMs * 100) / 100,
|
|
184
|
+
}));
|
|
185
|
+
}
|
|
186
|
+
/** The report as Markdown, for a human and for the reports page. */
|
|
187
|
+
export function reportMarkdown(report) {
|
|
188
|
+
const e = report.environment;
|
|
189
|
+
const lines = [];
|
|
190
|
+
lines.push(`# OpenStream benchmark — ${report.implementation.name} ${report.implementation.version}`);
|
|
191
|
+
lines.push("");
|
|
192
|
+
lines.push(`Generated ${report.generatedAt} · envelope ${report.specVersion} · schema ${report.schema}`);
|
|
193
|
+
lines.push("");
|
|
194
|
+
lines.push(`**Environment.** ${e.runtime}, zstd ${e.zstd}, zlib ${e.zlib}, on ${e.os} ${e.arch}, ${e.cores}× ${e.cpu}, ${e.memoryGiB} GiB.`);
|
|
195
|
+
lines.push("");
|
|
196
|
+
lines.push(`**Policy.** block ${report.policy.maxBlockBytes} B; eligible at ${report.policy.minSavingsPercent}% and ${report.policy.minSavingsBytes} B; zstd levels ${report.policy.zstdLevels.join(", ")}.`);
|
|
197
|
+
lines.push("");
|
|
198
|
+
lines.push("## Corpus");
|
|
199
|
+
lines.push("");
|
|
200
|
+
lines.push("| sample | kind | bytes | container | best mode | saves |");
|
|
201
|
+
lines.push("| --- | --- | ---: | --- | --- | ---: |");
|
|
202
|
+
for (const s of report.samples) {
|
|
203
|
+
const best = s.rows.filter((r) => r.roundTrip && r.mode !== "stored").sort((a, b) => a.wireBytes - b.wireBytes)[0];
|
|
204
|
+
const saves = s.recommendation.mode === "stored" ? "stored" : `${best?.savingsPercent ?? 0}%`;
|
|
205
|
+
lines.push(`| ${s.sample} | ${s.kind} | ${s.inputBytes} | ${s.container} | ${s.recommendation.mode}${s.recommendation.level ? ` L${s.recommendation.level}` : ""} | ${saves} |`);
|
|
206
|
+
}
|
|
207
|
+
lines.push("");
|
|
208
|
+
lines.push("## Aggregate, per mode across the corpus");
|
|
209
|
+
lines.push("");
|
|
210
|
+
lines.push("| mode | wire bytes | saving | enc ms | dec ms | round trip |");
|
|
211
|
+
lines.push("| --- | ---: | ---: | ---: | ---: | --- |");
|
|
212
|
+
for (const m of report.summary.byMode) {
|
|
213
|
+
lines.push(`| ${m.mode}${m.level ? ` L${m.level}` : ""} | ${m.wireBytes} | ${m.savingsPercent >= 0 ? "+" : ""}${m.savingsPercent}% | ${m.encodeMs} | ${m.decodeMs} | ${m.roundTrip ? "ok" : "FAILED"} |`);
|
|
214
|
+
}
|
|
215
|
+
lines.push("");
|
|
216
|
+
lines.push("## Caveats");
|
|
217
|
+
lines.push("");
|
|
218
|
+
for (const c of report.caveats)
|
|
219
|
+
lines.push(`- ${c}`);
|
|
220
|
+
lines.push("");
|
|
221
|
+
return lines.join("\n");
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Whether every applicable codec restored exactly: the gate a release must
|
|
225
|
+
* pass. A row with a note is a mode that did not apply to that sample, not a
|
|
226
|
+
* corrupted round trip, so it does not fail the gate.
|
|
227
|
+
*/
|
|
228
|
+
export function reportPasses(report) {
|
|
229
|
+
return report.samples.every((s) => s.rows.every((r) => r.roundTrip || Boolean(r.note)));
|
|
230
|
+
}
|
|
231
|
+
/** The rows that are a real exactness failure: ran, and did not restore. */
|
|
232
|
+
export function exactnessFailures(report) {
|
|
233
|
+
const out = [];
|
|
234
|
+
for (const s of report.samples) {
|
|
235
|
+
for (const r of s.rows) {
|
|
236
|
+
if (!r.roundTrip && !r.note)
|
|
237
|
+
out.push({ sample: s.sample, mode: r.mode, level: r.level });
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
return out;
|
|
241
|
+
}
|
package/dist/compression/cli.js
CHANGED
|
@@ -16,10 +16,13 @@
|
|
|
16
16
|
* `nixamp admin` finds its target. JSON goes to stdout and only JSON;
|
|
17
17
|
* progress and complaints go to stderr; a failure is a nonzero exit.
|
|
18
18
|
*/
|
|
19
|
-
import { createWriteStream } from "node:fs";
|
|
19
|
+
import { createWriteStream, mkdirSync, writeFileSync } from "node:fs";
|
|
20
|
+
import { join } from "node:path";
|
|
20
21
|
import { resolveTarget } from "../admin.js";
|
|
21
22
|
import { detectTools } from "../audio.js";
|
|
23
|
+
import { version as nixampVersion } from "../meta.js";
|
|
22
24
|
import { KEY_HEADER } from "../share.js";
|
|
25
|
+
import { exactnessFailures, fileCorpus, reportMarkdown, reportPasses, runBenchmark, syntheticCorpus } from "./benchmark.js";
|
|
23
26
|
import { RelayError } from "./envelope.js";
|
|
24
27
|
import { receiveRelay, RelayRefused } from "./receiver.js";
|
|
25
28
|
import { analyzeFile } from "./service.js";
|
|
@@ -33,6 +36,8 @@ const USAGE = `nixamp compression — lossless relay compression: measure it, se
|
|
|
33
36
|
nixamp compression off | on the whole server's switch
|
|
34
37
|
nixamp compression pull --channel ID --from URL [--from-key KEY] [--name NAME]
|
|
35
38
|
nixamp compression fetch URL --out FILE [--key KEY]
|
|
39
|
+
nixamp compression benchmark [--corpus DIR] [--out DIR] [--levels 1,3,9]
|
|
40
|
+
prove the envelope on a corpus and write a report; no server or ffmpeg needed
|
|
36
41
|
|
|
37
42
|
--url U --key K a server other than the local daemon, as for \`nixamp admin\`
|
|
38
43
|
--format json JSON on stdout (the default when stdout is not a terminal)
|
|
@@ -315,6 +320,56 @@ async function fetchRelay(flags) {
|
|
|
315
320
|
return 1;
|
|
316
321
|
}
|
|
317
322
|
}
|
|
323
|
+
/**
|
|
324
|
+
* Run the OpenStream benchmark and write a report. Anyone can run it: the
|
|
325
|
+
* default corpus is bytes we generate, no server and no ffmpeg. `--corpus DIR`
|
|
326
|
+
* benchmarks a directory of real files instead; `--out DIR` writes
|
|
327
|
+
* `openstream-report.json` and `.md` there for publishing.
|
|
328
|
+
*/
|
|
329
|
+
async function bench(flags) {
|
|
330
|
+
const dir = flags.named.get("corpus");
|
|
331
|
+
let corpus;
|
|
332
|
+
try {
|
|
333
|
+
corpus = dir ? fileCorpus(dir) : syntheticCorpus();
|
|
334
|
+
}
|
|
335
|
+
catch (error) {
|
|
336
|
+
console.error(`nixamp: cannot read corpus ${dir}: ${error.message}`);
|
|
337
|
+
return 1;
|
|
338
|
+
}
|
|
339
|
+
if (corpus.length === 0) {
|
|
340
|
+
console.error(`nixamp: no samples${dir ? ` in ${dir}` : ""}`);
|
|
341
|
+
return 1;
|
|
342
|
+
}
|
|
343
|
+
console.error(` Benchmarking ${corpus.length} sample${corpus.length === 1 ? "" : "s"}${dir ? ` from ${dir}` : " (synthetic)"}…`);
|
|
344
|
+
const levels = flags.named.get("levels");
|
|
345
|
+
const report = await runBenchmark(corpus, {
|
|
346
|
+
implementationVersion: nixampVersion(),
|
|
347
|
+
...(levels ? { zstdLevels: levels.split(",").map((n) => Number(n)).filter((n) => Number.isInteger(n)) } : {}),
|
|
348
|
+
});
|
|
349
|
+
const out = flags.named.get("out");
|
|
350
|
+
if (out) {
|
|
351
|
+
try {
|
|
352
|
+
mkdirSync(out, { recursive: true });
|
|
353
|
+
writeFileSync(join(out, "openstream-report.json"), JSON.stringify(report, null, 2));
|
|
354
|
+
writeFileSync(join(out, "openstream-report.md"), reportMarkdown(report));
|
|
355
|
+
console.error(` Wrote openstream-report.json and .md to ${out}`);
|
|
356
|
+
}
|
|
357
|
+
catch (error) {
|
|
358
|
+
console.error(`nixamp: cannot write to ${out}: ${error.message}`);
|
|
359
|
+
return 1;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
if (wantsJson(flags))
|
|
363
|
+
console.log(JSON.stringify(report, null, 2));
|
|
364
|
+
else
|
|
365
|
+
console.log(reportMarkdown(report));
|
|
366
|
+
if (!reportPasses(report)) {
|
|
367
|
+
const bad = exactnessFailures(report).map((f) => `${f.sample}/${f.mode}${f.level ? `L${f.level}` : ""}`).join(", ");
|
|
368
|
+
console.error(`nixamp: these did not restore byte-for-byte, this build must not ship: ${bad}`);
|
|
369
|
+
return 1;
|
|
370
|
+
}
|
|
371
|
+
return 0;
|
|
372
|
+
}
|
|
318
373
|
export async function compression(argv) {
|
|
319
374
|
const flags = parse(argv);
|
|
320
375
|
const verb = flags.positional[0];
|
|
@@ -335,6 +390,9 @@ export async function compression(argv) {
|
|
|
335
390
|
return await pull(flags);
|
|
336
391
|
case "fetch":
|
|
337
392
|
return await fetchRelay(flags);
|
|
393
|
+
case "benchmark":
|
|
394
|
+
case "bench":
|
|
395
|
+
return await bench(flags);
|
|
338
396
|
default:
|
|
339
397
|
console.error(USAGE);
|
|
340
398
|
return verb === undefined || verb === "help" || flags.named.has("help") ? 0 : 2;
|
|
@@ -10,10 +10,33 @@
|
|
|
10
10
|
* which is surfaced as an error naming the reason, never as a stream of
|
|
11
11
|
* something else.
|
|
12
12
|
*/
|
|
13
|
-
import { type Mode, type StreamHeader } from "./envelope.ts";
|
|
13
|
+
import { type Boundary, type Mode, type StreamHeader } from "./envelope.ts";
|
|
14
14
|
export declare const CODECS_HEADER = "x-nixamp-stream-codecs";
|
|
15
15
|
export declare const KIND_HEADER = "x-nixamp-kind";
|
|
16
|
+
export declare const BOUNDARY_HEADER = "x-nixamp-boundary";
|
|
17
|
+
/** What the media inside is, as JSON: video, audio, container, duration. */
|
|
18
|
+
export declare const MEDIA_HEADER = "x-nixamp-codecs";
|
|
16
19
|
export declare const KEY_HEADER = "x-nixamp-key";
|
|
20
|
+
export interface Media {
|
|
21
|
+
video: string;
|
|
22
|
+
audio: string;
|
|
23
|
+
container: string;
|
|
24
|
+
duration?: number;
|
|
25
|
+
}
|
|
26
|
+
/** What a server said it would send, read off its response headers. */
|
|
27
|
+
export interface Accepted {
|
|
28
|
+
codecs: string;
|
|
29
|
+
kind: "audio" | "video" | "";
|
|
30
|
+
boundary: Boundary | "";
|
|
31
|
+
media: Media | null;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Ask a relay what it would send, without taking it: the negotiation
|
|
35
|
+
* headers come back on the response and the body is cancelled at once.
|
|
36
|
+
* Costs one short connection; a receiver needs to know the boundary and
|
|
37
|
+
* the media before it can decide how to carry the stream.
|
|
38
|
+
*/
|
|
39
|
+
export declare function probeRelay(url: string, key: string | null, fetchImpl?: typeof fetch): Promise<Accepted>;
|
|
17
40
|
export interface ReceiveOptions {
|
|
18
41
|
url: string;
|
|
19
42
|
key: string | null;
|
|
@@ -21,10 +44,7 @@ export interface ReceiveOptions {
|
|
|
21
44
|
modes?: Mode[];
|
|
22
45
|
maxFrameBytes?: number;
|
|
23
46
|
/** The server said yes: what it will compress with, and what the channel carries. Before any byte. */
|
|
24
|
-
onStart?: (accepted:
|
|
25
|
-
codecs: string;
|
|
26
|
-
kind: "audio" | "video" | "";
|
|
27
|
-
}) => void;
|
|
47
|
+
onStart?: (accepted: Accepted) => void;
|
|
28
48
|
/** The stream header arrived: the generation this is. */
|
|
29
49
|
onHeader?: (header: StreamHeader) => void;
|
|
30
50
|
onBytes: (bytes: Buffer) => void | Promise<void>;
|
|
@@ -14,7 +14,60 @@ import { MEDIA_TYPE, RelayError } from "./envelope.js";
|
|
|
14
14
|
import { RelayDecoder } from "./relay.js";
|
|
15
15
|
export const CODECS_HEADER = "x-nixamp-stream-codecs";
|
|
16
16
|
export const KIND_HEADER = "x-nixamp-kind";
|
|
17
|
+
export const BOUNDARY_HEADER = "x-nixamp-boundary";
|
|
18
|
+
/** What the media inside is, as JSON: video, audio, container, duration. */
|
|
19
|
+
export const MEDIA_HEADER = "x-nixamp-codecs";
|
|
17
20
|
export const KEY_HEADER = "x-nixamp-key";
|
|
21
|
+
function acceptedFrom(headers) {
|
|
22
|
+
const kindSaid = headers.get(KIND_HEADER);
|
|
23
|
+
const boundarySaid = headers.get(BOUNDARY_HEADER);
|
|
24
|
+
let media = null;
|
|
25
|
+
try {
|
|
26
|
+
const raw = JSON.parse(headers.get(MEDIA_HEADER) ?? "null");
|
|
27
|
+
if (raw && typeof raw.video === "string" && typeof raw.audio === "string" && typeof raw.container === "string") {
|
|
28
|
+
media = { video: raw.video, audio: raw.audio, container: raw.container };
|
|
29
|
+
if (typeof raw.duration === "number")
|
|
30
|
+
media.duration = raw.duration;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// Not JSON: no media description, which the receiver copes with.
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
codecs: headers.get(CODECS_HEADER) ?? "",
|
|
38
|
+
kind: kindSaid === "audio" || kindSaid === "video" ? kindSaid : "",
|
|
39
|
+
boundary: boundarySaid === "source" || boundarySaid === "channel" ? boundarySaid : "",
|
|
40
|
+
media,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Ask a relay what it would send, without taking it: the negotiation
|
|
45
|
+
* headers come back on the response and the body is cancelled at once.
|
|
46
|
+
* Costs one short connection; a receiver needs to know the boundary and
|
|
47
|
+
* the media before it can decide how to carry the stream.
|
|
48
|
+
*/
|
|
49
|
+
export async function probeRelay(url, key, fetchImpl) {
|
|
50
|
+
const headers = { accept: MEDIA_TYPE, [CODECS_HEADER]: "stored,zstd,ts-zstd" };
|
|
51
|
+
if (key)
|
|
52
|
+
headers[KEY_HEADER] = key;
|
|
53
|
+
const response = await (fetchImpl ?? fetch)(url, { headers });
|
|
54
|
+
const type = response.headers.get("content-type") ?? "";
|
|
55
|
+
if (response.status !== 200 || !type.startsWith(MEDIA_TYPE)) {
|
|
56
|
+
let reason = `${response.status}`;
|
|
57
|
+
try {
|
|
58
|
+
const body = (await response.json());
|
|
59
|
+
if (typeof body.error === "string")
|
|
60
|
+
reason = body.error;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// Not JSON; the status is the message.
|
|
64
|
+
}
|
|
65
|
+
throw new RelayRefused(response.status, reason);
|
|
66
|
+
}
|
|
67
|
+
const accepted = acceptedFrom(response.headers);
|
|
68
|
+
await response.body?.cancel().catch(() => undefined);
|
|
69
|
+
return accepted;
|
|
70
|
+
}
|
|
18
71
|
/** The server said no. `status` is what it said it with. */
|
|
19
72
|
export class RelayRefused extends Error {
|
|
20
73
|
status;
|
|
@@ -55,11 +108,7 @@ export async function receiveRelay(options) {
|
|
|
55
108
|
}
|
|
56
109
|
if (!response.body)
|
|
57
110
|
throw new RelayRefused(response.status, "no body");
|
|
58
|
-
|
|
59
|
-
options.onStart?.({
|
|
60
|
-
codecs: response.headers.get(CODECS_HEADER) ?? "",
|
|
61
|
-
kind: kindSaid === "audio" || kindSaid === "video" ? kindSaid : "",
|
|
62
|
-
});
|
|
111
|
+
options.onStart?.(acceptedFrom(response.headers));
|
|
63
112
|
const decoder = new RelayDecoder({
|
|
64
113
|
modes,
|
|
65
114
|
...(options.maxFrameBytes !== undefined ? { maxFrameBytes: options.maxFrameBytes } : {}),
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import { createReadStream } from "node:fs";
|
|
13
13
|
import { MEDIA_TYPE, parseModes } from "./envelope.js";
|
|
14
|
-
import { CODECS_HEADER, KIND_HEADER } from "./receiver.js";
|
|
14
|
+
import { BOUNDARY_HEADER, CODECS_HEADER, KIND_HEADER, MEDIA_HEADER } from "./receiver.js";
|
|
15
15
|
/** Who a diagnostic job belongs to. Every control holder shares them. */
|
|
16
16
|
const CONTROL = "control";
|
|
17
17
|
/** The header a conditional change names the version in, or the body's own field. */
|
|
@@ -115,6 +115,11 @@ export async function handleChannelCompression(request, response, ctx, id, actio
|
|
|
115
115
|
"content-encoding": "identity",
|
|
116
116
|
[CODECS_HEADER]: answer.codecs.join(","),
|
|
117
117
|
[KIND_HEADER]: answer.kind,
|
|
118
|
+
// Which bytes these are, and what is inside them: a receiver of the
|
|
119
|
+
// source boundary hands them to its own ffmpeg and needs to know
|
|
120
|
+
// what ffmpeg will find, without a probe of its own.
|
|
121
|
+
[BOUNDARY_HEADER]: answer.boundary,
|
|
122
|
+
...(answer.media ? { [MEDIA_HEADER]: JSON.stringify(answer.media) } : {}),
|
|
118
123
|
"x-nixamp-generation": String(answer.generation),
|
|
119
124
|
});
|
|
120
125
|
const leave = () => answer.session.leave();
|
|
@@ -135,7 +140,7 @@ export async function handleChannelCompression(request, response, ctx, id, actio
|
|
|
135
140
|
}
|
|
136
141
|
const key = typeof body?.["key"] === "string" ? body["key"] : null;
|
|
137
142
|
const name = typeof body?.["name"] === "string" ? body["name"] : id;
|
|
138
|
-
const started = service.pull(id, from, key, name);
|
|
143
|
+
const started = await service.pull(id, from, key, name);
|
|
139
144
|
if (!started.ok) {
|
|
140
145
|
json(response, started.status, { error: started.error });
|
|
141
146
|
return true;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { type Channels } from "../channels.ts";
|
|
1
|
+
import { type ChannelInfo, type Channels } from "../channels.ts";
|
|
2
2
|
import { type Analysis } from "./analyze.ts";
|
|
3
3
|
import { Pool } from "./codec.ts";
|
|
4
|
-
import { type Mode } from "./envelope.ts";
|
|
4
|
+
import { type Boundary, type Mode } from "./envelope.ts";
|
|
5
5
|
import { AnalysisJobs, type Job } from "./jobs.ts";
|
|
6
6
|
import { type ChannelMetricsSnapshot } from "./metrics.ts";
|
|
7
7
|
import { type ChannelPolicy, type LosslessPolicy } from "./policy.ts";
|
|
@@ -66,6 +66,9 @@ export type RelayAnswer = {
|
|
|
66
66
|
codecs: Mode[];
|
|
67
67
|
generation: number;
|
|
68
68
|
kind: "audio" | "video" | "";
|
|
69
|
+
/** Which bytes the stream carries, and what is inside them. */
|
|
70
|
+
boundary: Boundary;
|
|
71
|
+
media: ChannelInfo["codecs"] | null;
|
|
69
72
|
} | {
|
|
70
73
|
ok: false;
|
|
71
74
|
status: 404 | 406 | 409 | 503;
|
|
@@ -83,6 +86,8 @@ export declare class CompressionService {
|
|
|
83
86
|
private readonly incoming;
|
|
84
87
|
private generation;
|
|
85
88
|
constructor(options: CompressionServiceOptions);
|
|
89
|
+
/** Incoming source-boundary relays, so their channel is read through us. */
|
|
90
|
+
private readonly relaySource;
|
|
86
91
|
private get channels();
|
|
87
92
|
/** What applies to a channel right now, and why that is not what is configured, if it is not. */
|
|
88
93
|
effective(id: string): EffectivePolicy;
|
|
@@ -112,15 +117,22 @@ export declare class CompressionService {
|
|
|
112
117
|
* anything worth sending.
|
|
113
118
|
*/
|
|
114
119
|
relay(id: string, listener: RelayListener, offered: Set<Mode>): RelayAnswer;
|
|
120
|
+
/** Null: the channel ended. False: the source boundary was asked for but is not being read through us. */
|
|
115
121
|
private startEncoder;
|
|
116
|
-
/**
|
|
117
|
-
|
|
122
|
+
/**
|
|
123
|
+
* Start listening to another nixamp's channel as one of ours. Probes the
|
|
124
|
+
* relay first to learn its boundary: a channel-boundary relay is decoded
|
|
125
|
+
* and its bytes are the channel's output directly; a source-boundary relay
|
|
126
|
+
* is the original transport stream, so it is read through ffmpeg here
|
|
127
|
+
* exactly as a pulled source would be, and can be relayed on again.
|
|
128
|
+
*/
|
|
129
|
+
pull(id: string, from: string, key: string | null, name: string): Promise<{
|
|
118
130
|
ok: true;
|
|
119
131
|
} | {
|
|
120
132
|
ok: false;
|
|
121
|
-
status: 409 | 400;
|
|
133
|
+
status: 409 | 400 | 502;
|
|
122
134
|
error: string;
|
|
123
|
-
}
|
|
135
|
+
}>;
|
|
124
136
|
/** Stop an incoming relay, and the channel it feeds. */
|
|
125
137
|
stopPull(id: string): boolean;
|
|
126
138
|
/**
|