@norskvideo/ctl-test-harness 0.1.115 → 0.1.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon.d.ts +9 -1
- package/daemon.js +80 -39
- package/demo/dev-loop.d.ts +7 -5
- package/demo/dev-loop.js +14 -13
- package/demo/run.d.ts +7 -0
- package/demo/run.js +41 -24
- package/media-conversion/local-studio.js +3 -1
- package/package.json +4 -4
- package/sample-media.d.ts +30 -0
- package/sample-media.js +126 -0
- package/source-pump.d.ts +2 -0
- package/source-pump.js +11 -1
package/daemon.d.ts
CHANGED
|
@@ -117,6 +117,10 @@ export declare function startDaemon(storeDir: string, options?: StartDaemonOptio
|
|
|
117
117
|
/** How long one `instance delete` may take before teardown stops waiting for
|
|
118
118
|
* it and reaps the instance's containers itself. A healthy delete is seconds. */
|
|
119
119
|
export declare const INSTANCE_DELETE_TIMEOUT_MS = 20000;
|
|
120
|
+
/** How long `stopDaemon` (`shutdown`: instances DELETE, proxy compose-down, a
|
|
121
|
+
* 5s wait) may take before teardown moves on to the SIGKILL and the container
|
|
122
|
+
* wait, which cope with a daemon that never stopped. */
|
|
123
|
+
export declare const STOP_DAEMON_TIMEOUT_MS = 30000;
|
|
120
124
|
/**
|
|
121
125
|
* Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
|
|
122
126
|
* process, rm store. The teardown actions are injected as pre-bound thunks
|
|
@@ -131,7 +135,8 @@ export declare const INSTANCE_DELETE_TIMEOUT_MS = 20000;
|
|
|
131
135
|
* composition guide's engine running that way for 2.6 hours on 2026-10-03; it
|
|
132
136
|
* grew to 28 GB and the host OOM-killed the next morning's product CI. So each
|
|
133
137
|
* delete is bounded, and whatever an instance leaves behind is then removed by
|
|
134
|
-
* its `norsk-ctl.instance` label, which needs only Docker.
|
|
138
|
+
* its `norsk-ctl.instance` label, which needs only Docker. The daemon stop is
|
|
139
|
+
* bounded the same way (ET-5).
|
|
135
140
|
*/
|
|
136
141
|
export declare function cleanupDaemon(opts: {
|
|
137
142
|
deleteInstance: (id: string) => Promise<unknown>;
|
|
@@ -149,6 +154,9 @@ export declare function cleanupDaemon(opts: {
|
|
|
149
154
|
* foreign endpoint keeps `compose down` from removing the network. */
|
|
150
155
|
leaveNetwork?: (id: string) => void;
|
|
151
156
|
deleteTimeoutMs?: number;
|
|
157
|
+
stopDaemonTimeoutMs?: number;
|
|
158
|
+
/** Told how long each step took, after it finishes or throws. */
|
|
159
|
+
onStep?: (step: string, ms: number) => void;
|
|
152
160
|
}): Promise<void>;
|
|
153
161
|
/** Tear down a direct-API test: stop proxy, remove instances, rm temp dir. */
|
|
154
162
|
export declare function cleanupDirect(opts: {
|
package/daemon.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// consumes these same bodies instead of keeping a third fork.
|
|
10
10
|
import { spawn, spawnSync } from "node:child_process";
|
|
11
11
|
import { createHash } from "node:crypto";
|
|
12
|
-
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
12
|
+
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
13
13
|
import { createServer } from "node:net";
|
|
14
14
|
import { join } from "node:path";
|
|
15
15
|
import { cliCommand } from "./cli-command.js";
|
|
@@ -17,6 +17,7 @@ import { currentInstanceNetworks } from "./container-net.js";
|
|
|
17
17
|
import { rememberDaemonStore } from "./daemon-stores.js";
|
|
18
18
|
import { pollUntil } from "./poll.js";
|
|
19
19
|
import { reapInstance } from "./reap.js";
|
|
20
|
+
import { sampleMediaCacheDir } from "./sample-media.js";
|
|
20
21
|
import { makeStoreDir, makeTempDir, TEST_TMP_BASE } from "./temp-dir.js";
|
|
21
22
|
export const HEALTH_TIMEOUT_MS = 120_000;
|
|
22
23
|
// The daemon's readiness probe (/api/ready) cannot answer until `serve` has
|
|
@@ -242,6 +243,7 @@ export function daemonSpawnEnv(o) {
|
|
|
242
243
|
// otherwise be inherited straight through this env spread and quietly open
|
|
243
244
|
// it (RFC 0003 D10b).
|
|
244
245
|
const { NORSK_CTL_UNGUARDED_TEST_DAEMON: _unguarded, ...inherited } = o.inherited;
|
|
246
|
+
const cacheEnv = { ...inherited, ...o.overrides };
|
|
245
247
|
const id = `${o.port}-${createHash("sha256").update(o.storeDir).digest("hex").slice(0, 8)}`;
|
|
246
248
|
return {
|
|
247
249
|
NORSK_CTL_PROXY_CONTAINER: `norsk-proxy-${id}`,
|
|
@@ -252,6 +254,12 @@ export function daemonSpawnEnv(o) {
|
|
|
252
254
|
// sets it explicitly, with NORSK_CTL_MOQ_RELAY_CONTAINER and its ports
|
|
253
255
|
// (ProductHarness's `moqRelay` does, and removes it at teardown).
|
|
254
256
|
NORSK_CTL_MOQ_RELAY_ENABLE: "0",
|
|
257
|
+
// One sample-media dir for every daemon sharing the cache, outside every store, so
|
|
258
|
+
// a fresh store does not re-download it and no teardown can remove it. Only
|
|
259
|
+
// when the environment names the cache: the daemon bind-mounts this dir, and
|
|
260
|
+
// on a docker-outside-of-docker runner the default under $HOME is invisible
|
|
261
|
+
// to the host daemon. Without it, source-pump seeds each store instead.
|
|
262
|
+
...(cacheEnv.NORSK_ASSET_CACHE ? { NORSK_CTL_SAMPLE_MEDIA_DIR: sampleMediaCacheDir(cacheEnv) } : {}),
|
|
255
263
|
...inherited,
|
|
256
264
|
NORSK_CTL_STORE_DIR: o.storeDir,
|
|
257
265
|
NORSK_CTL_PORT: String(o.port),
|
|
@@ -272,16 +280,21 @@ export function startDaemon(storeDir, options) {
|
|
|
272
280
|
}
|
|
273
281
|
if (options?.seedConfig !== false)
|
|
274
282
|
seedDaemonConfig(storeDir);
|
|
283
|
+
const env = daemonSpawnEnv({
|
|
284
|
+
storeDir,
|
|
285
|
+
port,
|
|
286
|
+
inherited: process.env,
|
|
287
|
+
...(options?.env ? { overrides: options.env } : {}),
|
|
288
|
+
});
|
|
289
|
+
// Ours before the daemon bind-mounts it: dockerd would create it root-owned,
|
|
290
|
+
// and a cache that outlives the run would then refuse every later download.
|
|
291
|
+
if (env.NORSK_CTL_SAMPLE_MEDIA_DIR)
|
|
292
|
+
mkdirSync(env.NORSK_CTL_SAMPLE_MEDIA_DIR, { recursive: true });
|
|
275
293
|
const [cmd, ...cmdArgs] = command;
|
|
276
294
|
const proc = spawn(cmd, [...cmdArgs, "serve"], {
|
|
277
295
|
stdio: "inherit",
|
|
278
296
|
detached: options?.detached === true,
|
|
279
|
-
env
|
|
280
|
-
storeDir,
|
|
281
|
-
port,
|
|
282
|
-
inherited: process.env,
|
|
283
|
-
...(options?.env ? { overrides: options.env } : {}),
|
|
284
|
-
}),
|
|
297
|
+
env,
|
|
285
298
|
});
|
|
286
299
|
const spawnFailed = new Promise((_, reject) => {
|
|
287
300
|
proc.once("error", (e) => reject(new Error(cliUnavailableMessage(command, e.message))));
|
|
@@ -332,6 +345,20 @@ async function awaitContainerShutdown(containers) {
|
|
|
332
345
|
/** How long one `instance delete` may take before teardown stops waiting for
|
|
333
346
|
* it and reaps the instance's containers itself. A healthy delete is seconds. */
|
|
334
347
|
export const INSTANCE_DELETE_TIMEOUT_MS = 20_000;
|
|
348
|
+
/** How long `stopDaemon` (`shutdown`: instances DELETE, proxy compose-down, a
|
|
349
|
+
* 5s wait) may take before teardown moves on to the SIGKILL and the container
|
|
350
|
+
* wait, which cope with a daemon that never stopped. */
|
|
351
|
+
export const STOP_DAEMON_TIMEOUT_MS = 30_000;
|
|
352
|
+
/** Run `step`, then report how long it took — thrown or not. */
|
|
353
|
+
async function timed(name, onStep, step) {
|
|
354
|
+
const started = Date.now();
|
|
355
|
+
try {
|
|
356
|
+
return await step();
|
|
357
|
+
}
|
|
358
|
+
finally {
|
|
359
|
+
onStep?.(name, Date.now() - started);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
335
362
|
/**
|
|
336
363
|
* Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
|
|
337
364
|
* process, rm store. The teardown actions are injected as pre-bound thunks
|
|
@@ -346,58 +373,72 @@ export const INSTANCE_DELETE_TIMEOUT_MS = 20_000;
|
|
|
346
373
|
* composition guide's engine running that way for 2.6 hours on 2026-10-03; it
|
|
347
374
|
* grew to 28 GB and the host OOM-killed the next morning's product CI. So each
|
|
348
375
|
* delete is bounded, and whatever an instance leaves behind is then removed by
|
|
349
|
-
* its `norsk-ctl.instance` label, which needs only Docker.
|
|
376
|
+
* its `norsk-ctl.instance` label, which needs only Docker. The daemon stop is
|
|
377
|
+
* bounded the same way (ET-5).
|
|
350
378
|
*/
|
|
351
379
|
export async function cleanupDaemon(opts) {
|
|
352
380
|
const deleteTimeoutMs = opts.deleteTimeoutMs ?? INSTANCE_DELETE_TIMEOUT_MS;
|
|
381
|
+
const stopDaemonTimeoutMs = opts.stopDaemonTimeoutMs ?? STOP_DAEMON_TIMEOUT_MS;
|
|
353
382
|
const leave = opts.leaveNetwork ?? ((id) => currentInstanceNetworks().leave(id));
|
|
354
383
|
for (const id of opts.instances ?? []) {
|
|
355
384
|
leave(id);
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
385
|
+
await timed(`instance delete ${id}`, opts.onStep, async () => {
|
|
386
|
+
try {
|
|
387
|
+
if ((await settleWithin(opts.deleteInstance(id), deleteTimeoutMs)) === "timeout")
|
|
388
|
+
console.error(`cleanup: delete ${id} gave no answer in ${deleteTimeoutMs}ms`);
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
console.error(`cleanup: delete ${id} failed: ${e}`);
|
|
392
|
+
}
|
|
393
|
+
});
|
|
363
394
|
}
|
|
364
395
|
if (!opts.leaveNetwork)
|
|
365
396
|
currentInstanceNetworks().leaveAll();
|
|
366
397
|
const reap = opts.reapInstance ?? reapInstance;
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
398
|
+
if ((opts.instances ?? []).length > 0)
|
|
399
|
+
await timed("reap instances", opts.onStep, () => {
|
|
400
|
+
for (const id of opts.instances ?? []) {
|
|
401
|
+
const left = reap(id);
|
|
402
|
+
if (left.length > 0)
|
|
403
|
+
console.error(`cleanup: removed ${left.length} container(s) instance ${id} left running`);
|
|
404
|
+
}
|
|
405
|
+
});
|
|
372
406
|
// Stop the proxy unless the caller says it never ran one (`proxy: false`).
|
|
373
407
|
// It used to need `proxy: true`, and three callers passing a real stopProxy
|
|
374
408
|
// omitted it — so `shutdown --daemon-only` left their proxy pair running.
|
|
375
409
|
// On the shared ar-arm-vm runner a leftover pair then answered for the next
|
|
376
410
|
// job's proxy (2026-10-02, norsk-ctl Checks 37048720935).
|
|
377
411
|
if (opts.proxy ?? true)
|
|
412
|
+
await timed("stop proxy", opts.onStep, async () => {
|
|
413
|
+
try {
|
|
414
|
+
await opts.stopProxy();
|
|
415
|
+
}
|
|
416
|
+
catch (e) {
|
|
417
|
+
console.error(`cleanup: proxy stop failed: ${e}`);
|
|
418
|
+
}
|
|
419
|
+
});
|
|
420
|
+
await timed("stop daemon", opts.onStep, async () => {
|
|
378
421
|
try {
|
|
379
|
-
await opts.
|
|
422
|
+
if ((await settleWithin(opts.stopDaemon(), stopDaemonTimeoutMs)) === "timeout")
|
|
423
|
+
console.error(`cleanup: daemon stop gave no answer in ${stopDaemonTimeoutMs}ms`);
|
|
380
424
|
}
|
|
381
425
|
catch (e) {
|
|
382
|
-
console.error(`cleanup:
|
|
383
|
-
}
|
|
384
|
-
try {
|
|
385
|
-
await opts.stopDaemon();
|
|
386
|
-
}
|
|
387
|
-
catch (e) {
|
|
388
|
-
console.error(`cleanup: daemon stop failed: ${e}`);
|
|
389
|
-
}
|
|
390
|
-
if (opts.daemon) {
|
|
391
|
-
opts.daemon.kill();
|
|
392
|
-
// Escalate to SIGKILL after 3s if process hasn't exited
|
|
393
|
-
const race = Promise.race([opts.daemon.exited, sleep(3000).then(() => "timeout")]);
|
|
394
|
-
if ((await race) === "timeout") {
|
|
395
|
-
opts.daemon.kill(9);
|
|
396
|
-
await Promise.race([opts.daemon.exited, sleep(2000)]);
|
|
426
|
+
console.error(`cleanup: daemon stop failed: ${e}`);
|
|
397
427
|
}
|
|
398
|
-
}
|
|
399
|
-
|
|
400
|
-
|
|
428
|
+
});
|
|
429
|
+
const daemon = opts.daemon;
|
|
430
|
+
if (daemon)
|
|
431
|
+
await timed("kill daemon", opts.onStep, async () => {
|
|
432
|
+
daemon.kill();
|
|
433
|
+
// Escalate to SIGKILL after 3s if process hasn't exited
|
|
434
|
+
const race = Promise.race([daemon.exited, sleep(3000).then(() => "timeout")]);
|
|
435
|
+
if ((await race) === "timeout") {
|
|
436
|
+
daemon.kill(9);
|
|
437
|
+
await Promise.race([daemon.exited, sleep(2000)]);
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
await timed("await container shutdown", opts.onStep, () => awaitContainerShutdown(opts.containers));
|
|
441
|
+
await timed("remove store", opts.onStep, () => rmSync(opts.storeDir, { recursive: true, force: true }));
|
|
401
442
|
}
|
|
402
443
|
/** Tear down a direct-API test: stop proxy, remove instances, rm temp dir. */
|
|
403
444
|
export async function cleanupDirect(opts) {
|
package/demo/dev-loop.d.ts
CHANGED
|
@@ -23,17 +23,19 @@ export interface StandaloneSource {
|
|
|
23
23
|
asset: SourceAsset;
|
|
24
24
|
}
|
|
25
25
|
/** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
|
|
26
|
-
* `$f.part
|
|
27
|
-
* as a cached file
|
|
28
|
-
*
|
|
29
|
-
*
|
|
26
|
+
* `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
|
|
27
|
+
* as a cached file and concurrent fetchers sharing the asset cache never write
|
|
28
|
+
* the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
|
|
29
|
+
* whose size disagrees with the remote Content-Length (a truncation from before
|
|
30
|
+
* this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
|
|
31
|
+
* used as it is. No `set -e`. */
|
|
30
32
|
export declare function sampleFetchLines(file: string, at: {
|
|
31
33
|
base: string;
|
|
32
34
|
dir: string;
|
|
33
35
|
}): string[];
|
|
34
36
|
/** The pump script the sources pane runs on Enter: every source in
|
|
35
37
|
* parallel with the host's ffmpeg, into the from-source engine. A preset is
|
|
36
|
-
* fetched into the
|
|
38
|
+
* fetched into the shared asset cache if it is not there yet; a
|
|
37
39
|
* mediaFile that is absent on this box pumps its fallback. */
|
|
38
40
|
export declare function pumpScript(sources: StandaloneSource[], fileExists: (path: string) => boolean): string;
|
|
39
41
|
export declare function runDevLoop(spec: DemoSpec, opts: DevLoopOptions, deps?: DemoDeps): Promise<DevLoopResult>;
|
package/demo/dev-loop.js
CHANGED
|
@@ -8,28 +8,29 @@
|
|
|
8
8
|
// against the same daemon and backend; `down` tears them down and leaves the
|
|
9
9
|
// developer's workdir where it is.
|
|
10
10
|
import { join } from "node:path";
|
|
11
|
+
import { ASSET_BASE_SHELL, ASSET_CACHE_SHELL, SAMPLE_MEDIA_FILES, SAMPLE_MEDIA_KEY_PREFIX } from "../sample-media.js";
|
|
11
12
|
import { DEFAULT_STANDALONE_STUDIO_PORT, DEV_LOOP_SOURCES, DEV_LOOP_STAMP, standaloneDirs, } from "./panes.js";
|
|
12
13
|
import { DemoSession, defaultDemoDeps, demoSlug, exportStandaloneWorkdir, resolveIngestPort, } from "./run.js";
|
|
13
|
-
/** The daemon's
|
|
14
|
-
*
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
|
|
18
|
-
const SAMPLE_MEDIA_DIR = "$HOME/.norsk-ctl/sample-media";
|
|
14
|
+
/** The standalone tier pumps the daemon's preset files with the host's ffmpeg,
|
|
15
|
+
* straight from the shared asset cache (sample-media.ts). */
|
|
16
|
+
const SAMPLE_MEDIA_DIR = `${ASSET_CACHE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
|
|
17
|
+
const SAMPLE_MEDIA_URL = `${ASSET_BASE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
|
|
19
18
|
function generated(pattern) {
|
|
20
19
|
const video = pattern === "bars" ? "smptebars=size=1280x720:rate=25" : "testsrc=size=1280x720:rate=25";
|
|
21
20
|
return `ffmpeg -hide_banner -loglevel warning -re -f lavfi -i "${video}" -f lavfi -i "sine=frequency=440:sample_rate=48000" -c:v libx264 -preset veryfast -tune zerolatency -g 50 -c:a aac`;
|
|
22
21
|
}
|
|
23
22
|
/** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
|
|
24
|
-
* `$f.part
|
|
25
|
-
* as a cached file
|
|
26
|
-
*
|
|
27
|
-
*
|
|
23
|
+
* `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
|
|
24
|
+
* as a cached file and concurrent fetchers sharing the asset cache never write
|
|
25
|
+
* the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
|
|
26
|
+
* whose size disagrees with the remote Content-Length (a truncation from before
|
|
27
|
+
* this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
|
|
28
|
+
* used as it is. No `set -e`. */
|
|
28
29
|
export function sampleFetchLines(file, at) {
|
|
29
30
|
return [
|
|
30
31
|
`f="${at.dir}/${file}"; u="${at.base}/${file}"; get=1`,
|
|
31
32
|
`if [ -s "$f" ]; then get=0; n=$(curl -fsIL "$u" 2>/dev/null | tr -d '\\r' | awk 'tolower($1)=="content-length:"{n=$2} END{print n}'); [ -n "$n" ] && [ "$n" != "$(wc -c < "$f" | tr -d ' ')" ] && get=1; fi`,
|
|
32
|
-
`[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -
|
|
33
|
+
`[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -fsSL --retry 3 -o "$f.part.$$" "$u" && mv -f "$f.part.$$" "$f"; } || rm -f "$f.part.$$"`,
|
|
33
34
|
];
|
|
34
35
|
}
|
|
35
36
|
function pumpLines(asset, target) {
|
|
@@ -44,13 +45,13 @@ function pumpLines(asset, target) {
|
|
|
44
45
|
if (!file)
|
|
45
46
|
throw new Error(`unknown preset '${asset.preset}'; the daemon's presets are ${Object.keys(SAMPLE_MEDIA_FILES).join(", ")}`);
|
|
46
47
|
return [
|
|
47
|
-
...sampleFetchLines(file, { base:
|
|
48
|
+
...sampleFetchLines(file, { base: SAMPLE_MEDIA_URL, dir: SAMPLE_MEDIA_DIR }),
|
|
48
49
|
`ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$f" -c copy -f mpegts "${target}" &`,
|
|
49
50
|
];
|
|
50
51
|
}
|
|
51
52
|
/** The pump script the sources pane runs on Enter: every source in
|
|
52
53
|
* parallel with the host's ffmpeg, into the from-source engine. A preset is
|
|
53
|
-
* fetched into the
|
|
54
|
+
* fetched into the shared asset cache if it is not there yet; a
|
|
54
55
|
* mediaFile that is absent on this box pumps its fallback. */
|
|
55
56
|
export function pumpScript(sources, fileExists) {
|
|
56
57
|
const lines = [
|
package/demo/run.d.ts
CHANGED
|
@@ -92,6 +92,9 @@ export interface DemoDeps {
|
|
|
92
92
|
daemon: DaemonProcess | null;
|
|
93
93
|
storeDir: string;
|
|
94
94
|
containers: string[];
|
|
95
|
+
/** Told how long each of the cleanup's own steps took (cleanupDaemon's
|
|
96
|
+
* onStep). Optional: a custom cleanup may ignore it. */
|
|
97
|
+
onStep?: (step: string, ms: number) => void;
|
|
95
98
|
}): Promise<void>;
|
|
96
99
|
/** Docker leaves root-owned bind-mount targets under the store (smoke.ts). */
|
|
97
100
|
nukeStoreAsRoot(storeDir: string): void;
|
|
@@ -434,6 +437,10 @@ export declare class DemoSession {
|
|
|
434
437
|
* workflow's artifact upload. No env var (local dev) means no-op; every
|
|
435
438
|
* failure inside is swallowed by the helper. */
|
|
436
439
|
private preserveContainerLogs;
|
|
440
|
+
/** One `teardown: <step> <ms>ms` line per teardown step (ET-5), so the next
|
|
441
|
+
* teardown that outruns an afterAll names the step that was slow. */
|
|
442
|
+
private logStep;
|
|
443
|
+
private timedStep;
|
|
437
444
|
teardown(opts: {
|
|
438
445
|
instances: string[];
|
|
439
446
|
handles: SourceHandle[];
|
package/demo/run.js
CHANGED
|
@@ -888,6 +888,20 @@ export class DemoSession {
|
|
|
888
888
|
this.deps.log(`preserved ${n} container log(s) for ${id} -> ${join(ciLogDir, "containers", id)}`);
|
|
889
889
|
}
|
|
890
890
|
}
|
|
891
|
+
/** One `teardown: <step> <ms>ms` line per teardown step (ET-5), so the next
|
|
892
|
+
* teardown that outruns an afterAll names the step that was slow. */
|
|
893
|
+
logStep = (step, ms) => {
|
|
894
|
+
this.deps.log(`teardown: ${step} ${ms}ms`);
|
|
895
|
+
};
|
|
896
|
+
async timedStep(step, fn) {
|
|
897
|
+
const started = Date.now();
|
|
898
|
+
try {
|
|
899
|
+
return await fn();
|
|
900
|
+
}
|
|
901
|
+
finally {
|
|
902
|
+
this.logStep(step, Date.now() - started);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
891
905
|
async teardown(opts) {
|
|
892
906
|
// FIRST, while the containers are still standing. Everything below removes
|
|
893
907
|
// them, and `docker logs` is the only copy of the studio container's stdout
|
|
@@ -895,40 +909,43 @@ export class DemoSession {
|
|
|
895
909
|
// gates tell you to go and read. Unconditional rather than on-failure,
|
|
896
910
|
// because teardown is not told the verdict, and a tail per container is
|
|
897
911
|
// cheap next to a re-run.
|
|
898
|
-
this.preserveContainerLogs(opts.instances);
|
|
912
|
+
await this.timedStep("preserve container logs", () => this.preserveContainerLogs(opts.instances));
|
|
899
913
|
if (opts.handles.length)
|
|
900
|
-
await this.deps.stopSources(opts.handles).catch(() => { });
|
|
914
|
+
await this.timedStep("stop sources", () => this.deps.stopSources(opts.handles).catch(() => { }));
|
|
901
915
|
// Extras sit on the instance networks; they go before any instance does.
|
|
902
|
-
this.stopExtras();
|
|
916
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
903
917
|
if (this.policy === "reuse") {
|
|
904
918
|
for (const id of opts.instances) {
|
|
905
919
|
currentInstanceNetworks().leave(id);
|
|
906
|
-
const r = await this.cli(["instance", "delete", id, "--purge"]);
|
|
920
|
+
const r = await this.timedStep(`instance delete ${id}`, () => this.cli(["instance", "delete", id, "--purge"]));
|
|
907
921
|
if (r.exitCode !== 0)
|
|
908
922
|
this.deps.log(`instance delete ${id}: ${(r.stderr || r.stdout).trim()}`);
|
|
909
923
|
}
|
|
910
|
-
this.stopExtras();
|
|
911
|
-
this.dev?.kill();
|
|
924
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
925
|
+
await this.timedStep("stop dev backend", () => this.dev?.kill());
|
|
912
926
|
return;
|
|
913
927
|
}
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
this.
|
|
928
|
+
await this.timedStep("cleanup", async () => {
|
|
929
|
+
try {
|
|
930
|
+
await this.deps.cleanup({
|
|
931
|
+
deleteInstance: (id) => this.cli(["instance", "delete", id, "--purge"]),
|
|
932
|
+
stopDaemon: () => this.cli(["shutdown"]),
|
|
933
|
+
instances: opts.instances,
|
|
934
|
+
daemon: this.daemon,
|
|
935
|
+
storeDir: this.storeDir,
|
|
936
|
+
containers: [...opts.instances.flatMap(instanceContainerNames), ...this.extraContainers],
|
|
937
|
+
onStep: this.logStep,
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
catch (e) {
|
|
941
|
+
this.deps.log(`cleanup could not remove the store itself (${e instanceof Error ? e.message : String(e)}); nuking as root`);
|
|
942
|
+
reportForeignOwners(this.storeDir, (l) => this.deps.log(l));
|
|
943
|
+
}
|
|
944
|
+
});
|
|
945
|
+
await this.timedStep("remove moq relay", () => this.removeMoqRelay());
|
|
946
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
947
|
+
await this.timedStep("stop dev backend", () => this.dev?.kill());
|
|
948
|
+
await this.timedStep("nuke store as root", () => this.deps.nukeStoreAsRoot(this.storeDir));
|
|
932
949
|
}
|
|
933
950
|
}
|
|
934
951
|
/** `demo check --mode standalone --export-only` (05-demo s4): build the
|
|
@@ -31,7 +31,9 @@ export function withLocalLinks(workflowYaml) {
|
|
|
31
31
|
export function localStudioLauncher(opts) {
|
|
32
32
|
const port = opts.port ?? 8010;
|
|
33
33
|
const workDir = opts.workDir ?? mkdtempSync(join(tmpdir(), "conformance-studio-"));
|
|
34
|
-
|
|
34
|
+
// Folder form (#381): where Studio resolves `server.document: workflow.yml`
|
|
35
|
+
// first, as a norsk-ctl launch writes it.
|
|
36
|
+
const saves = join(workDir, "data", "studio-save-files", "workflow");
|
|
35
37
|
mkdirSync(saves, { recursive: true });
|
|
36
38
|
mkdirSync(join(workDir, "logs"), { recursive: true });
|
|
37
39
|
writeFileSync(join(saves, "workflow.yml"), opts.localLinks ? withLocalLinks(opts.workflowYaml) : opts.workflowYaml);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@norskvideo/ctl-test-harness",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.117",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -114,9 +114,9 @@
|
|
|
114
114
|
"devDependencies": {
|
|
115
115
|
"@types/bun": "latest",
|
|
116
116
|
"@types/node": "^22.10.0",
|
|
117
|
-
"@norskvideo/norsk-studio": "1.30.0-2026-10-
|
|
118
|
-
"@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-
|
|
119
|
-
"@norskvideo/norsk-sdk": "1.3.1-2026-10-
|
|
117
|
+
"@norskvideo/norsk-studio": "1.30.0-2026-10-26-896721cc",
|
|
118
|
+
"@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-26-896721cc",
|
|
119
|
+
"@norskvideo/norsk-sdk": "1.3.1-2026-10-27-8e095a74"
|
|
120
120
|
},
|
|
121
121
|
"peerDependenciesMeta": {
|
|
122
122
|
"@norskvideo/norsk-studio": {
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
type Env = Record<string, string | undefined>;
|
|
2
|
+
export declare const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
|
|
3
|
+
export declare const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
|
|
4
|
+
/** The daemon's built-in presets and their files (norsk-ctl
|
|
5
|
+
* backend/src/docker/constants.ts: SOURCE_PRESETS). */
|
|
6
|
+
export declare const SAMPLE_MEDIA_FILES: Readonly<Record<string, string>>;
|
|
7
|
+
/** The same locations as shell expressions, for scripts the harness writes. */
|
|
8
|
+
export declare const ASSET_CACHE_SHELL = "${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}";
|
|
9
|
+
export declare const ASSET_BASE_SHELL = "${NORSK_ASSET_BASE:-https://assets.norsk.video}";
|
|
10
|
+
export declare function assetCacheRoot(env?: Env): string;
|
|
11
|
+
export declare function sampleMediaCacheDir(env?: Env): string;
|
|
12
|
+
type FetchFn = (url: string) => Promise<Response>;
|
|
13
|
+
/** The cached path of asset `key`, downloading it first if absent. Streamed to
|
|
14
|
+
* a uniquely-named partial and renamed into place, so concurrent fetchers (and
|
|
15
|
+
* daemons reading the same dir) only ever see a whole file. */
|
|
16
|
+
export declare function fetchAsset(key: string, opts?: {
|
|
17
|
+
env?: Env;
|
|
18
|
+
fetch?: FetchFn;
|
|
19
|
+
}): Promise<string>;
|
|
20
|
+
/** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
|
|
21
|
+
* store looks before downloading it. For a ctl that predates
|
|
22
|
+
* NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
|
|
23
|
+
* the cache, so it costs nothing and stays a real file inside the store, which
|
|
24
|
+
* is bind-mounted into the source container; copied across filesystems. */
|
|
25
|
+
export declare function seedPresetMedia(storeDir: string, preset: string, opts?: {
|
|
26
|
+
env?: Env;
|
|
27
|
+
fetch?: FetchFn;
|
|
28
|
+
link?: (from: string, to: string) => void;
|
|
29
|
+
}): Promise<void>;
|
|
30
|
+
export {};
|
package/sample-media.js
ADDED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// The shared asset cache: immutable assets (sample media, models) fetched once
|
|
2
|
+
// per cache -- `${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}/<key>`, one per
|
|
3
|
+
// dev box or CI runner instance -- and used from there by every daemon, run and
|
|
4
|
+
// repo that shares it: the same convention the norsk and
|
|
5
|
+
// norsk-studio Makefiles follow. Every test daemon gets a fresh store, so without
|
|
6
|
+
// it each one re-downloaded the same ~190 MB of sample media from S3.
|
|
7
|
+
//
|
|
8
|
+
// Lives outside every store and every TEST_TMP_BASE, so no teardown (rm -rf of a
|
|
9
|
+
// store, the stale-dir prune, the root nuke) can reach it; a store only ever
|
|
10
|
+
// holds hardlinks or copies of what is here.
|
|
11
|
+
import { closeSync, copyFileSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, statSync, utimesSync, } from "node:fs";
|
|
12
|
+
import { open } from "node:fs/promises";
|
|
13
|
+
import { homedir } from "node:os";
|
|
14
|
+
import { dirname, join } from "node:path";
|
|
15
|
+
export const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
|
|
16
|
+
export const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
|
|
17
|
+
/** The daemon's built-in presets and their files (norsk-ctl
|
|
18
|
+
* backend/src/docker/constants.ts: SOURCE_PRESETS). */
|
|
19
|
+
export const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
|
|
20
|
+
/** The same locations as shell expressions, for scripts the harness writes. */
|
|
21
|
+
export const ASSET_CACHE_SHELL = `\${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}`;
|
|
22
|
+
export const ASSET_BASE_SHELL = `\${NORSK_ASSET_BASE:-${DEFAULT_ASSET_BASE}}`;
|
|
23
|
+
export function assetCacheRoot(env = process.env) {
|
|
24
|
+
return env.NORSK_ASSET_CACHE || join(env.HOME || homedir(), ".cache", "norsk", "assets");
|
|
25
|
+
}
|
|
26
|
+
export function sampleMediaCacheDir(env = process.env) {
|
|
27
|
+
return join(assetCacheRoot(env), SAMPLE_MEDIA_KEY_PREFIX);
|
|
28
|
+
}
|
|
29
|
+
function hasContent(path) {
|
|
30
|
+
try {
|
|
31
|
+
const s = statSync(path);
|
|
32
|
+
return s.isFile() && s.size > 0;
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return false;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** A partial older than this was left by a killed run, not a live download. */
|
|
39
|
+
const STALE_PARTIAL_MS = 2 * 60 * 60 * 1000;
|
|
40
|
+
function sweepStalePartials(dir) {
|
|
41
|
+
for (const f of readdirSync(dir)) {
|
|
42
|
+
if (!f.includes(".part."))
|
|
43
|
+
continue;
|
|
44
|
+
const path = join(dir, f);
|
|
45
|
+
try {
|
|
46
|
+
if (Date.now() - statSync(path).mtimeMs > STALE_PARTIAL_MS)
|
|
47
|
+
rmSync(path, { force: true });
|
|
48
|
+
}
|
|
49
|
+
catch { }
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
/** Touch `<cache>/.used/<key>`, which the norsk repos' scripts/prune-asset-cache.sh
|
|
53
|
+
* reads to keep what is still in use. Never the cached file itself: checkouts
|
|
54
|
+
* hardlink it, and a newer mtime would look like a rebuild to make. */
|
|
55
|
+
function recordUse(root, key) {
|
|
56
|
+
const sidecar = join(root, ".used", key);
|
|
57
|
+
try {
|
|
58
|
+
mkdirSync(dirname(sidecar), { recursive: true });
|
|
59
|
+
closeSync(openSync(sidecar, "a"));
|
|
60
|
+
const now = new Date();
|
|
61
|
+
utimesSync(sidecar, now, now);
|
|
62
|
+
}
|
|
63
|
+
catch { }
|
|
64
|
+
}
|
|
65
|
+
/** The cached path of asset `key`, downloading it first if absent. Streamed to
|
|
66
|
+
* a uniquely-named partial and renamed into place, so concurrent fetchers (and
|
|
67
|
+
* daemons reading the same dir) only ever see a whole file. */
|
|
68
|
+
export async function fetchAsset(key, opts = {}) {
|
|
69
|
+
const env = opts.env ?? process.env;
|
|
70
|
+
const root = assetCacheRoot(env);
|
|
71
|
+
const dest = join(root, key);
|
|
72
|
+
if (hasContent(dest)) {
|
|
73
|
+
recordUse(root, key);
|
|
74
|
+
return dest;
|
|
75
|
+
}
|
|
76
|
+
const base = (env.NORSK_ASSET_BASE || DEFAULT_ASSET_BASE).replace(/\/+$/, "");
|
|
77
|
+
const res = await (opts.fetch ?? fetch)(`${base}/${key}`);
|
|
78
|
+
if (!res.ok || !res.body)
|
|
79
|
+
throw new Error(`fetch-asset ${key}: ${res.status} ${res.statusText}`);
|
|
80
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
81
|
+
sweepStalePartials(dirname(dest));
|
|
82
|
+
const part = `${dest}.part.${process.pid}.${Math.random().toString(36).slice(2)}`;
|
|
83
|
+
try {
|
|
84
|
+
const out = await open(part, "w");
|
|
85
|
+
try {
|
|
86
|
+
const reader = res.body.getReader();
|
|
87
|
+
for (let r = await reader.read(); !r.done; r = await reader.read())
|
|
88
|
+
await out.write(r.value);
|
|
89
|
+
}
|
|
90
|
+
finally {
|
|
91
|
+
await out.close();
|
|
92
|
+
}
|
|
93
|
+
renameSync(part, dest);
|
|
94
|
+
}
|
|
95
|
+
catch (e) {
|
|
96
|
+
rmSync(part, { force: true });
|
|
97
|
+
throw e;
|
|
98
|
+
}
|
|
99
|
+
recordUse(root, key);
|
|
100
|
+
console.error(`fetch-asset: ${key} cached at ${dest}`);
|
|
101
|
+
return dest;
|
|
102
|
+
}
|
|
103
|
+
/** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
|
|
104
|
+
* store looks before downloading it. For a ctl that predates
|
|
105
|
+
* NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
|
|
106
|
+
* the cache, so it costs nothing and stays a real file inside the store, which
|
|
107
|
+
* is bind-mounted into the source container; copied across filesystems. */
|
|
108
|
+
export async function seedPresetMedia(storeDir, preset, opts = {}) {
|
|
109
|
+
const file = SAMPLE_MEDIA_FILES[preset];
|
|
110
|
+
if (!file)
|
|
111
|
+
return;
|
|
112
|
+
const dest = join(storeDir, "sample-media", file);
|
|
113
|
+
if (hasContent(dest))
|
|
114
|
+
return;
|
|
115
|
+
const cached = await fetchAsset(`${SAMPLE_MEDIA_KEY_PREFIX}/${file}`, opts);
|
|
116
|
+
mkdirSync(dirname(dest), { recursive: true });
|
|
117
|
+
const part = `${dest}.part.${process.pid}`;
|
|
118
|
+
rmSync(part, { force: true });
|
|
119
|
+
try {
|
|
120
|
+
(opts.link ?? linkSync)(cached, part);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
copyFileSync(cached, part);
|
|
124
|
+
}
|
|
125
|
+
renameSync(part, dest);
|
|
126
|
+
}
|
package/source-pump.d.ts
CHANGED
|
@@ -58,6 +58,7 @@ export interface StartCameraSourceOptions {
|
|
|
58
58
|
/** Test seams. */
|
|
59
59
|
fetch?: FetchLike;
|
|
60
60
|
fileExists?: (path: string) => boolean;
|
|
61
|
+
seedPreset?: (storeDir: string, preset: string) => Promise<void>;
|
|
61
62
|
}
|
|
62
63
|
export declare function startCameraSource(opts: StartCameraSourceOptions): Promise<SourceHandle>;
|
|
63
64
|
export interface SrtPumpTarget {
|
|
@@ -80,5 +81,6 @@ export declare function startSrtSources(opts: {
|
|
|
80
81
|
timeoutMs?: number;
|
|
81
82
|
fetch?: FetchLike;
|
|
82
83
|
fileExists?: (path: string) => boolean;
|
|
84
|
+
seedPreset?: (storeDir: string, preset: string) => Promise<void>;
|
|
83
85
|
}): Promise<SourceHandle[]>;
|
|
84
86
|
export declare function stopAll(handles: readonly SourceHandle[]): Promise<void>;
|
package/source-pump.js
CHANGED
|
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from "node:fs";
|
|
|
20
20
|
import { basename, extname, join } from "node:path";
|
|
21
21
|
import { storeDirForDaemonPort } from "./daemon-stores.js";
|
|
22
22
|
import { pollUntil } from "./poll.js";
|
|
23
|
+
import { seedPresetMedia } from "./sample-media.js";
|
|
23
24
|
/** Where every daemon the harness starts writes its ingress secret: the store
|
|
24
25
|
* root (the guard's stateDir is `<storeDir>` under NORSK_CTL_STORE_DIR). One
|
|
25
26
|
* reader for the whole harness — the three callers that spelled the filename
|
|
@@ -130,9 +131,17 @@ export async function startCameraSource(opts) {
|
|
|
130
131
|
return { secret, headers: secret ? { "x-norsk-proxy": secret } : {} };
|
|
131
132
|
};
|
|
132
133
|
const base = `http://localhost:${opts.daemonPort}`;
|
|
134
|
+
const fields = assetFields(asset, opts.fileExists ?? existsSync);
|
|
135
|
+
// A daemon on a ctl that predates NORSK_CTL_SAMPLE_MEDIA_DIR looks in its own
|
|
136
|
+
// store, so put the preset there from the shared cache before it goes looking.
|
|
137
|
+
// Best-effort: if this fails, the daemon downloads the file itself.
|
|
138
|
+
const storeDir = storeDirForDaemonPort(opts.daemonPort);
|
|
139
|
+
if (storeDir !== undefined && typeof fields.preset === "string") {
|
|
140
|
+
await (opts.seedPreset ?? seedPresetMedia)(storeDir, fields.preset).catch((e) => console.error(`sample media: could not seed ${fields.preset} into ${storeDir}: ${e}`));
|
|
141
|
+
}
|
|
133
142
|
const body = {
|
|
134
143
|
instanceId: opts.instanceId,
|
|
135
|
-
...
|
|
144
|
+
...fields,
|
|
136
145
|
port: opts.port,
|
|
137
146
|
name,
|
|
138
147
|
};
|
|
@@ -215,6 +224,7 @@ export async function startSrtSources(opts) {
|
|
|
215
224
|
...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
|
|
216
225
|
...(opts.fetch !== undefined ? { fetch: opts.fetch } : {}),
|
|
217
226
|
...(opts.fileExists !== undefined ? { fileExists: opts.fileExists } : {}),
|
|
227
|
+
...(opts.seedPreset !== undefined ? { seedPreset: opts.seedPreset } : {}),
|
|
218
228
|
});
|
|
219
229
|
handles.push(handle);
|
|
220
230
|
}
|