@norskvideo/ctl-test-harness 0.1.115 → 0.1.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/daemon.js CHANGED
@@ -9,7 +9,7 @@
9
9
  // consumes these same bodies instead of keeping a third fork.
10
10
  import { spawn, spawnSync } from "node:child_process";
11
11
  import { createHash } from "node:crypto";
12
- import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
+ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
13
13
  import { createServer } from "node:net";
14
14
  import { join } from "node:path";
15
15
  import { cliCommand } from "./cli-command.js";
@@ -17,6 +17,7 @@ import { currentInstanceNetworks } from "./container-net.js";
17
17
  import { rememberDaemonStore } from "./daemon-stores.js";
18
18
  import { pollUntil } from "./poll.js";
19
19
  import { reapInstance } from "./reap.js";
20
+ import { sampleMediaCacheDir } from "./sample-media.js";
20
21
  import { makeStoreDir, makeTempDir, TEST_TMP_BASE } from "./temp-dir.js";
21
22
  export const HEALTH_TIMEOUT_MS = 120_000;
22
23
  // The daemon's readiness probe (/api/ready) cannot answer until `serve` has
@@ -242,6 +243,7 @@ export function daemonSpawnEnv(o) {
242
243
  // otherwise be inherited straight through this env spread and quietly open
243
244
  // it (RFC 0003 D10b).
244
245
  const { NORSK_CTL_UNGUARDED_TEST_DAEMON: _unguarded, ...inherited } = o.inherited;
246
+ const cacheEnv = { ...inherited, ...o.overrides };
245
247
  const id = `${o.port}-${createHash("sha256").update(o.storeDir).digest("hex").slice(0, 8)}`;
246
248
  return {
247
249
  NORSK_CTL_PROXY_CONTAINER: `norsk-proxy-${id}`,
@@ -252,6 +254,12 @@ export function daemonSpawnEnv(o) {
252
254
  // sets it explicitly, with NORSK_CTL_MOQ_RELAY_CONTAINER and its ports
253
255
  // (ProductHarness's `moqRelay` does, and removes it at teardown).
254
256
  NORSK_CTL_MOQ_RELAY_ENABLE: "0",
257
+ // One sample-media dir for every daemon sharing the cache, outside every store, so
258
+ // a fresh store does not re-download it and no teardown can remove it. Only
259
+ // when the environment names the cache: the daemon bind-mounts this dir, and
260
+ // on a docker-outside-of-docker runner the default under $HOME is invisible
261
+ // to the host daemon. Without it, source-pump seeds each store instead.
262
+ ...(cacheEnv.NORSK_ASSET_CACHE ? { NORSK_CTL_SAMPLE_MEDIA_DIR: sampleMediaCacheDir(cacheEnv) } : {}),
255
263
  ...inherited,
256
264
  NORSK_CTL_STORE_DIR: o.storeDir,
257
265
  NORSK_CTL_PORT: String(o.port),
@@ -272,16 +280,21 @@ export function startDaemon(storeDir, options) {
272
280
  }
273
281
  if (options?.seedConfig !== false)
274
282
  seedDaemonConfig(storeDir);
283
+ const env = daemonSpawnEnv({
284
+ storeDir,
285
+ port,
286
+ inherited: process.env,
287
+ ...(options?.env ? { overrides: options.env } : {}),
288
+ });
289
+ // Ours before the daemon bind-mounts it: dockerd would create it root-owned,
290
+ // and a cache that outlives the run would then refuse every later download.
291
+ if (env.NORSK_CTL_SAMPLE_MEDIA_DIR)
292
+ mkdirSync(env.NORSK_CTL_SAMPLE_MEDIA_DIR, { recursive: true });
275
293
  const [cmd, ...cmdArgs] = command;
276
294
  const proc = spawn(cmd, [...cmdArgs, "serve"], {
277
295
  stdio: "inherit",
278
296
  detached: options?.detached === true,
279
- env: daemonSpawnEnv({
280
- storeDir,
281
- port,
282
- inherited: process.env,
283
- ...(options?.env ? { overrides: options.env } : {}),
284
- }),
297
+ env,
285
298
  });
286
299
  const spawnFailed = new Promise((_, reject) => {
287
300
  proc.once("error", (e) => reject(new Error(cliUnavailableMessage(command, e.message))));
@@ -23,17 +23,19 @@ export interface StandaloneSource {
23
23
  asset: SourceAsset;
24
24
  }
25
25
  /** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
26
- * `$f.part` and are renamed on success, so an interrupted fetch never poses
27
- * as a cached file; a cached file whose size disagrees with the remote
28
- * Content-Length (a truncation from before this was atomic) is re-fetched.
29
- * Offline (HEAD fails), a non-empty cache is used as it is. No `set -e`. */
26
+ * `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
27
+ * as a cached file and concurrent fetchers sharing the asset cache never write
28
+ * the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
29
+ * whose size disagrees with the remote Content-Length (a truncation from before
30
+ * this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
31
+ * used as it is. No `set -e`. */
30
32
  export declare function sampleFetchLines(file: string, at: {
31
33
  base: string;
32
34
  dir: string;
33
35
  }): string[];
34
36
  /** The pump script the sources pane runs on Enter: every source in
35
37
  * parallel with the host's ffmpeg, into the from-source engine. A preset is
36
- * fetched into the daemon's sample-media cache if it is not there yet; a
38
+ * fetched into the shared asset cache if it is not there yet; a
37
39
  * mediaFile that is absent on this box pumps its fallback. */
38
40
  export declare function pumpScript(sources: StandaloneSource[], fileExists: (path: string) => boolean): string;
39
41
  export declare function runDevLoop(spec: DemoSpec, opts: DevLoopOptions, deps?: DemoDeps): Promise<DevLoopResult>;
package/demo/dev-loop.js CHANGED
@@ -8,28 +8,29 @@
8
8
  // against the same daemon and backend; `down` tears them down and leaves the
9
9
  // developer's workdir where it is.
10
10
  import { join } from "node:path";
11
+ import { ASSET_BASE_SHELL, ASSET_CACHE_SHELL, SAMPLE_MEDIA_FILES, SAMPLE_MEDIA_KEY_PREFIX } from "../sample-media.js";
11
12
  import { DEFAULT_STANDALONE_STUDIO_PORT, DEV_LOOP_SOURCES, DEV_LOOP_STAMP, standaloneDirs, } from "./panes.js";
12
13
  import { DemoSession, defaultDemoDeps, demoSlug, exportStandaloneWorkdir, resolveIngestPort, } from "./run.js";
13
- /** The daemon's built-in presets and where it caches them (norsk-ctl
14
- * backend/src/docker/constants.ts: SOURCE_PRESETS, S3_MEDIA_BASE); the
15
- * standalone tier pumps from the same files with the host's ffmpeg. */
16
- const SAMPLE_MEDIA_BASE = "https://s3.eu-west-1.amazonaws.com/norsk.video/media-examples/data";
17
- const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
18
- const SAMPLE_MEDIA_DIR = "$HOME/.norsk-ctl/sample-media";
14
+ /** The standalone tier pumps the daemon's preset files with the host's ffmpeg,
15
+ * straight from the shared asset cache (sample-media.ts). */
16
+ const SAMPLE_MEDIA_DIR = `${ASSET_CACHE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
17
+ const SAMPLE_MEDIA_URL = `${ASSET_BASE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
19
18
  function generated(pattern) {
20
19
  const video = pattern === "bars" ? "smptebars=size=1280x720:rate=25" : "testsrc=size=1280x720:rate=25";
21
20
  return `ffmpeg -hide_banner -loglevel warning -re -f lavfi -i "${video}" -f lavfi -i "sine=frequency=440:sample_rate=48000" -c:v libx264 -preset veryfast -tune zerolatency -g 50 -c:a aac`;
22
21
  }
23
22
  /** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
24
- * `$f.part` and are renamed on success, so an interrupted fetch never poses
25
- * as a cached file; a cached file whose size disagrees with the remote
26
- * Content-Length (a truncation from before this was atomic) is re-fetched.
27
- * Offline (HEAD fails), a non-empty cache is used as it is. No `set -e`. */
23
+ * `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
24
+ * as a cached file and concurrent fetchers sharing the asset cache never write
25
+ * the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
26
+ * whose size disagrees with the remote Content-Length (a truncation from before
27
+ * this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
28
+ * used as it is. No `set -e`. */
28
29
  export function sampleFetchLines(file, at) {
29
30
  return [
30
31
  `f="${at.dir}/${file}"; u="${at.base}/${file}"; get=1`,
31
32
  `if [ -s "$f" ]; then get=0; n=$(curl -fsIL "$u" 2>/dev/null | tr -d '\\r' | awk 'tolower($1)=="content-length:"{n=$2} END{print n}'); [ -n "$n" ] && [ "$n" != "$(wc -c < "$f" | tr -d ' ')" ] && get=1; fi`,
32
- `[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -fL -o "$f.part" "$u" && mv -f "$f.part" "$f"; } || rm -f "$f.part"`,
33
+ `[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -fsSL --retry 3 -o "$f.part.$$" "$u" && mv -f "$f.part.$$" "$f"; } || rm -f "$f.part.$$"`,
33
34
  ];
34
35
  }
35
36
  function pumpLines(asset, target) {
@@ -44,13 +45,13 @@ function pumpLines(asset, target) {
44
45
  if (!file)
45
46
  throw new Error(`unknown preset '${asset.preset}'; the daemon's presets are ${Object.keys(SAMPLE_MEDIA_FILES).join(", ")}`);
46
47
  return [
47
- ...sampleFetchLines(file, { base: SAMPLE_MEDIA_BASE, dir: SAMPLE_MEDIA_DIR }),
48
+ ...sampleFetchLines(file, { base: SAMPLE_MEDIA_URL, dir: SAMPLE_MEDIA_DIR }),
48
49
  `ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$f" -c copy -f mpegts "${target}" &`,
49
50
  ];
50
51
  }
51
52
  /** The pump script the sources pane runs on Enter: every source in
52
53
  * parallel with the host's ffmpeg, into the from-source engine. A preset is
53
- * fetched into the daemon's sample-media cache if it is not there yet; a
54
+ * fetched into the shared asset cache if it is not there yet; a
54
55
  * mediaFile that is absent on this box pumps its fallback. */
55
56
  export function pumpScript(sources, fileExists) {
56
57
  const lines = [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@norskvideo/ctl-test-harness",
3
- "version": "0.1.115",
3
+ "version": "0.1.116",
4
4
  "type": "module",
5
5
  "repository": {
6
6
  "type": "git",
@@ -114,9 +114,9 @@
114
114
  "devDependencies": {
115
115
  "@types/bun": "latest",
116
116
  "@types/node": "^22.10.0",
117
- "@norskvideo/norsk-studio": "1.30.0-2026-10-23-8e050eb4",
118
- "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-23-8e050eb4",
119
- "@norskvideo/norsk-sdk": "1.3.1-2026-10-22-aeac91a4"
117
+ "@norskvideo/norsk-studio": "1.30.0-2026-10-26-896721cc",
118
+ "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-26-896721cc",
119
+ "@norskvideo/norsk-sdk": "1.3.1-2026-10-27-8e095a74"
120
120
  },
121
121
  "peerDependenciesMeta": {
122
122
  "@norskvideo/norsk-studio": {
@@ -0,0 +1,30 @@
1
+ type Env = Record<string, string | undefined>;
2
+ export declare const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
3
+ export declare const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
4
+ /** The daemon's built-in presets and their files (norsk-ctl
5
+ * backend/src/docker/constants.ts: SOURCE_PRESETS). */
6
+ export declare const SAMPLE_MEDIA_FILES: Readonly<Record<string, string>>;
7
+ /** The same locations as shell expressions, for scripts the harness writes. */
8
+ export declare const ASSET_CACHE_SHELL = "${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}";
9
+ export declare const ASSET_BASE_SHELL = "${NORSK_ASSET_BASE:-https://assets.norsk.video}";
10
+ export declare function assetCacheRoot(env?: Env): string;
11
+ export declare function sampleMediaCacheDir(env?: Env): string;
12
+ type FetchFn = (url: string) => Promise<Response>;
13
+ /** The cached path of asset `key`, downloading it first if absent. Streamed to
14
+ * a uniquely-named partial and renamed into place, so concurrent fetchers (and
15
+ * daemons reading the same dir) only ever see a whole file. */
16
+ export declare function fetchAsset(key: string, opts?: {
17
+ env?: Env;
18
+ fetch?: FetchFn;
19
+ }): Promise<string>;
20
+ /** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
21
+ * store looks before downloading it. For a ctl that predates
22
+ * NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
23
+ * the cache, so it costs nothing and stays a real file inside the store, which
24
+ * is bind-mounted into the source container; copied across filesystems. */
25
+ export declare function seedPresetMedia(storeDir: string, preset: string, opts?: {
26
+ env?: Env;
27
+ fetch?: FetchFn;
28
+ link?: (from: string, to: string) => void;
29
+ }): Promise<void>;
30
+ export {};
@@ -0,0 +1,126 @@
1
+ // The shared asset cache: immutable assets (sample media, models) fetched once
2
+ // per cache -- `${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}/<key>`, one per
3
+ // dev box or CI runner instance -- and used from there by every daemon, run and
4
+ // repo that shares it: the same convention the norsk and
5
+ // norsk-studio Makefiles follow. Every test daemon gets a fresh store, so without
6
+ // it each one re-downloaded the same ~190 MB of sample media from S3.
7
+ //
8
+ // Lives outside every store and every TEST_TMP_BASE, so no teardown (rm -rf of a
9
+ // store, the stale-dir prune, the root nuke) can reach it; a store only ever
10
+ // holds hardlinks or copies of what is here.
11
+ import { closeSync, copyFileSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, statSync, utimesSync, } from "node:fs";
12
+ import { open } from "node:fs/promises";
13
+ import { homedir } from "node:os";
14
+ import { dirname, join } from "node:path";
15
+ export const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
16
+ export const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
17
+ /** The daemon's built-in presets and their files (norsk-ctl
18
+ * backend/src/docker/constants.ts: SOURCE_PRESETS). */
19
+ export const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
20
+ /** The same locations as shell expressions, for scripts the harness writes. */
21
+ export const ASSET_CACHE_SHELL = `\${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}`;
22
+ export const ASSET_BASE_SHELL = `\${NORSK_ASSET_BASE:-${DEFAULT_ASSET_BASE}}`;
23
+ export function assetCacheRoot(env = process.env) {
24
+ return env.NORSK_ASSET_CACHE || join(env.HOME || homedir(), ".cache", "norsk", "assets");
25
+ }
26
+ export function sampleMediaCacheDir(env = process.env) {
27
+ return join(assetCacheRoot(env), SAMPLE_MEDIA_KEY_PREFIX);
28
+ }
29
+ function hasContent(path) {
30
+ try {
31
+ const s = statSync(path);
32
+ return s.isFile() && s.size > 0;
33
+ }
34
+ catch {
35
+ return false;
36
+ }
37
+ }
38
+ /** A partial older than this was left by a killed run, not a live download. */
39
+ const STALE_PARTIAL_MS = 2 * 60 * 60 * 1000;
40
+ function sweepStalePartials(dir) {
41
+ for (const f of readdirSync(dir)) {
42
+ if (!f.includes(".part."))
43
+ continue;
44
+ const path = join(dir, f);
45
+ try {
46
+ if (Date.now() - statSync(path).mtimeMs > STALE_PARTIAL_MS)
47
+ rmSync(path, { force: true });
48
+ }
49
+ catch { }
50
+ }
51
+ }
52
+ /** Touch `<cache>/.used/<key>`, which the norsk repos' scripts/prune-asset-cache.sh
53
+ * reads to keep what is still in use. Never the cached file itself: checkouts
54
+ * hardlink it, and a newer mtime would look like a rebuild to make. */
55
+ function recordUse(root, key) {
56
+ const sidecar = join(root, ".used", key);
57
+ try {
58
+ mkdirSync(dirname(sidecar), { recursive: true });
59
+ closeSync(openSync(sidecar, "a"));
60
+ const now = new Date();
61
+ utimesSync(sidecar, now, now);
62
+ }
63
+ catch { }
64
+ }
65
+ /** The cached path of asset `key`, downloading it first if absent. Streamed to
66
+ * a uniquely-named partial and renamed into place, so concurrent fetchers (and
67
+ * daemons reading the same dir) only ever see a whole file. */
68
+ export async function fetchAsset(key, opts = {}) {
69
+ const env = opts.env ?? process.env;
70
+ const root = assetCacheRoot(env);
71
+ const dest = join(root, key);
72
+ if (hasContent(dest)) {
73
+ recordUse(root, key);
74
+ return dest;
75
+ }
76
+ const base = (env.NORSK_ASSET_BASE || DEFAULT_ASSET_BASE).replace(/\/+$/, "");
77
+ const res = await (opts.fetch ?? fetch)(`${base}/${key}`);
78
+ if (!res.ok || !res.body)
79
+ throw new Error(`fetch-asset ${key}: ${res.status} ${res.statusText}`);
80
+ mkdirSync(dirname(dest), { recursive: true });
81
+ sweepStalePartials(dirname(dest));
82
+ const part = `${dest}.part.${process.pid}.${Math.random().toString(36).slice(2)}`;
83
+ try {
84
+ const out = await open(part, "w");
85
+ try {
86
+ const reader = res.body.getReader();
87
+ for (let r = await reader.read(); !r.done; r = await reader.read())
88
+ await out.write(r.value);
89
+ }
90
+ finally {
91
+ await out.close();
92
+ }
93
+ renameSync(part, dest);
94
+ }
95
+ catch (e) {
96
+ rmSync(part, { force: true });
97
+ throw e;
98
+ }
99
+ recordUse(root, key);
100
+ console.error(`fetch-asset: ${key} cached at ${dest}`);
101
+ return dest;
102
+ }
103
+ /** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
104
+ * store looks before downloading it. For a ctl that predates
105
+ * NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
106
+ * the cache, so it costs nothing and stays a real file inside the store, which
107
+ * is bind-mounted into the source container; copied across filesystems. */
108
+ export async function seedPresetMedia(storeDir, preset, opts = {}) {
109
+ const file = SAMPLE_MEDIA_FILES[preset];
110
+ if (!file)
111
+ return;
112
+ const dest = join(storeDir, "sample-media", file);
113
+ if (hasContent(dest))
114
+ return;
115
+ const cached = await fetchAsset(`${SAMPLE_MEDIA_KEY_PREFIX}/${file}`, opts);
116
+ mkdirSync(dirname(dest), { recursive: true });
117
+ const part = `${dest}.part.${process.pid}`;
118
+ rmSync(part, { force: true });
119
+ try {
120
+ (opts.link ?? linkSync)(cached, part);
121
+ }
122
+ catch {
123
+ copyFileSync(cached, part);
124
+ }
125
+ renameSync(part, dest);
126
+ }
package/source-pump.d.ts CHANGED
@@ -58,6 +58,7 @@ export interface StartCameraSourceOptions {
58
58
  /** Test seams. */
59
59
  fetch?: FetchLike;
60
60
  fileExists?: (path: string) => boolean;
61
+ seedPreset?: (storeDir: string, preset: string) => Promise<void>;
61
62
  }
62
63
  export declare function startCameraSource(opts: StartCameraSourceOptions): Promise<SourceHandle>;
63
64
  export interface SrtPumpTarget {
@@ -80,5 +81,6 @@ export declare function startSrtSources(opts: {
80
81
  timeoutMs?: number;
81
82
  fetch?: FetchLike;
82
83
  fileExists?: (path: string) => boolean;
84
+ seedPreset?: (storeDir: string, preset: string) => Promise<void>;
83
85
  }): Promise<SourceHandle[]>;
84
86
  export declare function stopAll(handles: readonly SourceHandle[]): Promise<void>;
package/source-pump.js CHANGED
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from "node:fs";
20
20
  import { basename, extname, join } from "node:path";
21
21
  import { storeDirForDaemonPort } from "./daemon-stores.js";
22
22
  import { pollUntil } from "./poll.js";
23
+ import { seedPresetMedia } from "./sample-media.js";
23
24
  /** Where every daemon the harness starts writes its ingress secret: the store
24
25
  * root (the guard's stateDir is `<storeDir>` under NORSK_CTL_STORE_DIR). One
25
26
  * reader for the whole harness — the three callers that spelled the filename
@@ -130,9 +131,17 @@ export async function startCameraSource(opts) {
130
131
  return { secret, headers: secret ? { "x-norsk-proxy": secret } : {} };
131
132
  };
132
133
  const base = `http://localhost:${opts.daemonPort}`;
134
+ const fields = assetFields(asset, opts.fileExists ?? existsSync);
135
+ // A daemon on a ctl that predates NORSK_CTL_SAMPLE_MEDIA_DIR looks in its own
136
+ // store, so put the preset there from the shared cache before it goes looking.
137
+ // Best-effort: if this fails, the daemon downloads the file itself.
138
+ const storeDir = storeDirForDaemonPort(opts.daemonPort);
139
+ if (storeDir !== undefined && typeof fields.preset === "string") {
140
+ await (opts.seedPreset ?? seedPresetMedia)(storeDir, fields.preset).catch((e) => console.error(`sample media: could not seed ${fields.preset} into ${storeDir}: ${e}`));
141
+ }
133
142
  const body = {
134
143
  instanceId: opts.instanceId,
135
- ...assetFields(asset, opts.fileExists ?? existsSync),
144
+ ...fields,
136
145
  port: opts.port,
137
146
  name,
138
147
  };
@@ -215,6 +224,7 @@ export async function startSrtSources(opts) {
215
224
  ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
216
225
  ...(opts.fetch !== undefined ? { fetch: opts.fetch } : {}),
217
226
  ...(opts.fileExists !== undefined ? { fileExists: opts.fileExists } : {}),
227
+ ...(opts.seedPreset !== undefined ? { seedPreset: opts.seedPreset } : {}),
218
228
  });
219
229
  handles.push(handle);
220
230
  }