@norskvideo/ctl-test-harness 0.1.114 → 0.1.116

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/daemon.d.ts CHANGED
@@ -21,6 +21,9 @@ export declare function runCli(storeDir: string, ...args: string[]): Promise<{
21
21
  exitCode: number;
22
22
  }>;
23
23
  export declare function isPortFree(port: number): Promise<boolean>;
24
+ /** `-sTCP:LISTEN` is load-bearing: without it lsof also reports every process
25
+ * holding a client socket to the port, including this test runner's own
26
+ * keep-alive connection to a previous daemon there. */
24
27
  export declare function killProcessOnPort(port: number): void;
25
28
  /** The daemon handle callers hold: enough to kill it and await its exit,
26
29
  * without tying the interface to one runtime's spawn return type. */
@@ -83,7 +86,7 @@ export declare function ctlSupportsEphemeralPorts(): Promise<boolean>;
83
86
  /** Root the instance working directory under the store dir (itself under
84
87
  * TEST_TMP_BASE) rather than the daemon default ~/norsk-runtime. The launcher
85
88
  * bind-mounts the workdir as /data and then mounts the stored template's
86
- * workflow.yml (which lives under the store dir) *inside* it — Docker Desktop
89
+ * assets/ and dashboards/ (which live under the store dir) *inside* it — Docker Desktop
87
90
  * rejects a nested bind mount whose inner source resolves to a different host
88
91
  * share-root than the outer, so both mounts must share the TEST_TMP_BASE root.
89
92
  * With NORSK_CTL_STORE_DIR set, config.yaml is read from <storeDir>/config.yaml.
package/daemon.js CHANGED
@@ -9,7 +9,7 @@
9
9
  // consumes these same bodies instead of keeping a third fork.
10
10
  import { spawn, spawnSync } from "node:child_process";
11
11
  import { createHash } from "node:crypto";
12
- import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
12
+ import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
13
13
  import { createServer } from "node:net";
14
14
  import { join } from "node:path";
15
15
  import { cliCommand } from "./cli-command.js";
@@ -17,6 +17,7 @@ import { currentInstanceNetworks } from "./container-net.js";
17
17
  import { rememberDaemonStore } from "./daemon-stores.js";
18
18
  import { pollUntil } from "./poll.js";
19
19
  import { reapInstance } from "./reap.js";
20
+ import { sampleMediaCacheDir } from "./sample-media.js";
20
21
  import { makeStoreDir, makeTempDir, TEST_TMP_BASE } from "./temp-dir.js";
21
22
  export const HEALTH_TIMEOUT_MS = 120_000;
22
23
  // The daemon's readiness probe (/api/ready) cannot answer until `serve` has
@@ -112,9 +113,12 @@ export function isPortFree(port) {
112
113
  srv.listen(port, () => srv.close(() => resolve(true)));
113
114
  });
114
115
  }
116
+ /** `-sTCP:LISTEN` is load-bearing: without it lsof also reports every process
117
+ * holding a client socket to the port, including this test runner's own
118
+ * keep-alive connection to a previous daemon there. */
115
119
  export function killProcessOnPort(port) {
116
120
  try {
117
- const result = spawnSync("lsof", ["-ti", `:${port}`], { encoding: "utf-8" });
121
+ const result = spawnSync("lsof", ["-ti", `:${port}`, "-sTCP:LISTEN"], { encoding: "utf-8" });
118
122
  const pids = splitLines(result.stdout ?? "");
119
123
  for (const pid of pids) {
120
124
  try {
@@ -200,7 +204,7 @@ export function ctlSupportsEphemeralPorts() {
200
204
  /** Root the instance working directory under the store dir (itself under
201
205
  * TEST_TMP_BASE) rather than the daemon default ~/norsk-runtime. The launcher
202
206
  * bind-mounts the workdir as /data and then mounts the stored template's
203
- * workflow.yml (which lives under the store dir) *inside* it — Docker Desktop
207
+ * assets/ and dashboards/ (which live under the store dir) *inside* it — Docker Desktop
204
208
  * rejects a nested bind mount whose inner source resolves to a different host
205
209
  * share-root than the outer, so both mounts must share the TEST_TMP_BASE root.
206
210
  * With NORSK_CTL_STORE_DIR set, config.yaml is read from <storeDir>/config.yaml.
@@ -239,6 +243,7 @@ export function daemonSpawnEnv(o) {
239
243
  // otherwise be inherited straight through this env spread and quietly open
240
244
  // it (RFC 0003 D10b).
241
245
  const { NORSK_CTL_UNGUARDED_TEST_DAEMON: _unguarded, ...inherited } = o.inherited;
246
+ const cacheEnv = { ...inherited, ...o.overrides };
242
247
  const id = `${o.port}-${createHash("sha256").update(o.storeDir).digest("hex").slice(0, 8)}`;
243
248
  return {
244
249
  NORSK_CTL_PROXY_CONTAINER: `norsk-proxy-${id}`,
@@ -249,6 +254,12 @@ export function daemonSpawnEnv(o) {
249
254
  // sets it explicitly, with NORSK_CTL_MOQ_RELAY_CONTAINER and its ports
250
255
  // (ProductHarness's `moqRelay` does, and removes it at teardown).
251
256
  NORSK_CTL_MOQ_RELAY_ENABLE: "0",
257
+ // One sample-media dir for every daemon sharing the cache, outside every store, so
258
+ // a fresh store does not re-download it and no teardown can remove it. Only
259
+ // when the environment names the cache: the daemon bind-mounts this dir, and
260
+ // on a docker-outside-of-docker runner the default under $HOME is invisible
261
+ // to the host daemon. Without it, source-pump seeds each store instead.
262
+ ...(cacheEnv.NORSK_ASSET_CACHE ? { NORSK_CTL_SAMPLE_MEDIA_DIR: sampleMediaCacheDir(cacheEnv) } : {}),
252
263
  ...inherited,
253
264
  NORSK_CTL_STORE_DIR: o.storeDir,
254
265
  NORSK_CTL_PORT: String(o.port),
@@ -269,16 +280,21 @@ export function startDaemon(storeDir, options) {
269
280
  }
270
281
  if (options?.seedConfig !== false)
271
282
  seedDaemonConfig(storeDir);
283
+ const env = daemonSpawnEnv({
284
+ storeDir,
285
+ port,
286
+ inherited: process.env,
287
+ ...(options?.env ? { overrides: options.env } : {}),
288
+ });
289
+ // Ours before the daemon bind-mounts it: dockerd would create it root-owned,
290
+ // and a cache that outlives the run would then refuse every later download.
291
+ if (env.NORSK_CTL_SAMPLE_MEDIA_DIR)
292
+ mkdirSync(env.NORSK_CTL_SAMPLE_MEDIA_DIR, { recursive: true });
272
293
  const [cmd, ...cmdArgs] = command;
273
294
  const proc = spawn(cmd, [...cmdArgs, "serve"], {
274
295
  stdio: "inherit",
275
296
  detached: options?.detached === true,
276
- env: daemonSpawnEnv({
277
- storeDir,
278
- port,
279
- inherited: process.env,
280
- ...(options?.env ? { overrides: options.env } : {}),
281
- }),
297
+ env,
282
298
  });
283
299
  const spawnFailed = new Promise((_, reject) => {
284
300
  proc.once("error", (e) => reject(new Error(cliUnavailableMessage(command, e.message))));
@@ -23,17 +23,19 @@ export interface StandaloneSource {
23
23
  asset: SourceAsset;
24
24
  }
25
25
  /** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
26
- * `$f.part` and are renamed on success, so an interrupted fetch never poses
27
- * as a cached file; a cached file whose size disagrees with the remote
28
- * Content-Length (a truncation from before this was atomic) is re-fetched.
29
- * Offline (HEAD fails), a non-empty cache is used as it is. No `set -e`. */
26
+ * `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
27
+ * as a cached file and concurrent fetchers sharing the asset cache never write
28
+ * the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
29
+ * whose size disagrees with the remote Content-Length (a truncation from before
30
+ * this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
31
+ * used as it is. No `set -e`. */
30
32
  export declare function sampleFetchLines(file: string, at: {
31
33
  base: string;
32
34
  dir: string;
33
35
  }): string[];
34
36
  /** The pump script the sources pane runs on Enter: every source in
35
37
  * parallel with the host's ffmpeg, into the from-source engine. A preset is
36
- * fetched into the daemon's sample-media cache if it is not there yet; a
38
+ * fetched into the shared asset cache if it is not there yet; a
37
39
  * mediaFile that is absent on this box pumps its fallback. */
38
40
  export declare function pumpScript(sources: StandaloneSource[], fileExists: (path: string) => boolean): string;
39
41
  export declare function runDevLoop(spec: DemoSpec, opts: DevLoopOptions, deps?: DemoDeps): Promise<DevLoopResult>;
package/demo/dev-loop.js CHANGED
@@ -8,28 +8,29 @@
8
8
  // against the same daemon and backend; `down` tears them down and leaves the
9
9
  // developer's workdir where it is.
10
10
  import { join } from "node:path";
11
+ import { ASSET_BASE_SHELL, ASSET_CACHE_SHELL, SAMPLE_MEDIA_FILES, SAMPLE_MEDIA_KEY_PREFIX } from "../sample-media.js";
11
12
  import { DEFAULT_STANDALONE_STUDIO_PORT, DEV_LOOP_SOURCES, DEV_LOOP_STAMP, standaloneDirs, } from "./panes.js";
12
13
  import { DemoSession, defaultDemoDeps, demoSlug, exportStandaloneWorkdir, resolveIngestPort, } from "./run.js";
13
- /** The daemon's built-in presets and where it caches them (norsk-ctl
14
- * backend/src/docker/constants.ts: SOURCE_PRESETS, S3_MEDIA_BASE); the
15
- * standalone tier pumps from the same files with the host's ffmpeg. */
16
- const SAMPLE_MEDIA_BASE = "https://s3.eu-west-1.amazonaws.com/norsk.video/media-examples/data";
17
- const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
18
- const SAMPLE_MEDIA_DIR = "$HOME/.norsk-ctl/sample-media";
14
+ /** The standalone tier pumps the daemon's preset files with the host's ffmpeg,
15
+ * straight from the shared asset cache (sample-media.ts). */
16
+ const SAMPLE_MEDIA_DIR = `${ASSET_CACHE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
17
+ const SAMPLE_MEDIA_URL = `${ASSET_BASE_SHELL}/${SAMPLE_MEDIA_KEY_PREFIX}`;
19
18
  function generated(pattern) {
20
19
  const video = pattern === "bars" ? "smptebars=size=1280x720:rate=25" : "testsrc=size=1280x720:rate=25";
21
20
  return `ffmpeg -hide_banner -loglevel warning -re -f lavfi -i "${video}" -f lavfi -i "sine=frequency=440:sample_rate=48000" -c:v libx264 -preset veryfast -tune zerolatency -g 50 -c:a aac`;
22
21
  }
23
22
  /** Fetch a sample into the cache, leaving `$f` set to it. Downloads land in
24
- * `$f.part` and are renamed on success, so an interrupted fetch never poses
25
- * as a cached file; a cached file whose size disagrees with the remote
26
- * Content-Length (a truncation from before this was atomic) is re-fetched.
27
- * Offline (HEAD fails), a non-empty cache is used as it is. No `set -e`. */
23
+ * `$f.part.$$` and are renamed on success, so an interrupted fetch never poses
24
+ * as a cached file and concurrent fetchers sharing the asset cache never write
25
+ * the same partial (sample-media.ts sweeps stale `.part.` files); a cached file
26
+ * whose size disagrees with the remote Content-Length (a truncation from before
27
+ * this was atomic) is re-fetched. Offline (HEAD fails), a non-empty cache is
28
+ * used as it is. No `set -e`. */
28
29
  export function sampleFetchLines(file, at) {
29
30
  return [
30
31
  `f="${at.dir}/${file}"; u="${at.base}/${file}"; get=1`,
31
32
  `if [ -s "$f" ]; then get=0; n=$(curl -fsIL "$u" 2>/dev/null | tr -d '\\r' | awk 'tolower($1)=="content-length:"{n=$2} END{print n}'); [ -n "$n" ] && [ "$n" != "$(wc -c < "$f" | tr -d ' ')" ] && get=1; fi`,
32
- `[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -fL -o "$f.part" "$u" && mv -f "$f.part" "$f"; } || rm -f "$f.part"`,
33
+ `[ "$get" = 0 ] || { mkdir -p "\${f%/*}" && curl -fsSL --retry 3 -o "$f.part.$$" "$u" && mv -f "$f.part.$$" "$f"; } || rm -f "$f.part.$$"`,
33
34
  ];
34
35
  }
35
36
  function pumpLines(asset, target) {
@@ -44,13 +45,13 @@ function pumpLines(asset, target) {
44
45
  if (!file)
45
46
  throw new Error(`unknown preset '${asset.preset}'; the daemon's presets are ${Object.keys(SAMPLE_MEDIA_FILES).join(", ")}`);
46
47
  return [
47
- ...sampleFetchLines(file, { base: SAMPLE_MEDIA_BASE, dir: SAMPLE_MEDIA_DIR }),
48
+ ...sampleFetchLines(file, { base: SAMPLE_MEDIA_URL, dir: SAMPLE_MEDIA_DIR }),
48
49
  `ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$f" -c copy -f mpegts "${target}" &`,
49
50
  ];
50
51
  }
51
52
  /** The pump script the sources pane runs on Enter: every source in
52
53
  * parallel with the host's ffmpeg, into the from-source engine. A preset is
53
- * fetched into the daemon's sample-media cache if it is not there yet; a
54
+ * fetched into the shared asset cache if it is not there yet; a
54
55
  * mediaFile that is absent on this box pumps its fallback. */
55
56
  export function pumpScript(sources, fileExists) {
56
57
  const lines = [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@norskvideo/ctl-test-harness",
3
- "version": "0.1.114",
3
+ "version": "0.1.116",
4
4
  "type": "module",
5
5
  "repository": {
6
6
  "type": "git",
@@ -40,6 +40,10 @@
40
40
  "types": "./reap.d.ts",
41
41
  "default": "./reap.js"
42
42
  },
43
+ "./quiet-box": {
44
+ "types": "./quiet-box.d.ts",
45
+ "default": "./quiet-box.js"
46
+ },
43
47
  "./source-pump": {
44
48
  "types": "./source-pump.d.ts",
45
49
  "default": "./source-pump.js"
@@ -110,9 +114,9 @@
110
114
  "devDependencies": {
111
115
  "@types/bun": "latest",
112
116
  "@types/node": "^22.10.0",
113
- "@norskvideo/norsk-studio": "1.30.0-2026-10-23-8e050eb4",
114
- "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-23-8e050eb4",
115
- "@norskvideo/norsk-sdk": "1.3.1-2026-10-22-aeac91a4"
117
+ "@norskvideo/norsk-studio": "1.30.0-2026-10-26-896721cc",
118
+ "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-26-896721cc",
119
+ "@norskvideo/norsk-sdk": "1.3.1-2026-10-27-8e095a74"
116
120
  },
117
121
  "peerDependenciesMeta": {
118
122
  "@norskvideo/norsk-studio": {
package/quiet-box.d.ts ADDED
@@ -0,0 +1,35 @@
1
+ import { runCli } from "./daemon.js";
2
+ /** Set to `1` to run the engine tier even when foreign instances are live — for a
3
+ * single-tenant box/CI where those instances are your own and you accept the
4
+ * shared-network churn. */
5
+ export declare const BUSY_BOX_OVERRIDE_ENV = "NORSK_ALLOW_BUSY_BOX";
6
+ /** The box's primary daemon — the one an unscoped `norsk-ctl` command reaches. */
7
+ export declare const PRIMARY_DAEMON_PORT: number;
8
+ export type QuietBoxVerdict = {
9
+ clear: true;
10
+ } | {
11
+ clear: false;
12
+ message: string;
13
+ };
14
+ /** Decide whether the engine tier may run given the foreign instances live on the
15
+ * box. Pure: the caller supplies the discovered list and the override flag. */
16
+ export declare function quietBoxVerdict(foreignInstances: readonly string[], opts: {
17
+ override: boolean;
18
+ }): QuietBoxVerdict;
19
+ export interface DiscoverDeps {
20
+ /** Does a daemon answer /api/ready on this port? */
21
+ answers(port: number): Promise<boolean>;
22
+ list(storeDir: string, port: number): ReturnType<typeof runCli>;
23
+ }
24
+ /** Running instances on the box's primary daemon that are not ours. Best-effort:
25
+ * if the primary daemon is unreachable or is our own daemon, the box is quiet
26
+ * from our perspective and this returns []. `probeStoreDir` is only used to root
27
+ * the CLI invocation; the query itself reads the daemon's HTTP API by port.
28
+ * The port is asked directly first: the CLI auto-starts a daemon on a silent
29
+ * port and polls it for 30 seconds, which is not a question but a side effect. */
30
+ export declare function discoverForeignInstances(opts: {
31
+ probeStoreDir: string;
32
+ primaryPort: number;
33
+ ownPort: number;
34
+ deps?: DiscoverDeps;
35
+ }): Promise<string[]>;
package/quiet-box.js ADDED
@@ -0,0 +1,76 @@
1
+ // Engine-tier preflight guard (ET-3). The daemon's `norsk-net` bridge and
2
+ // `norsk-proxy` container are box-global singletons keyed by a FIXED name —
3
+ // `norsk-ctl config set` exposes only working-directory / public-host / proxy-port
4
+ // / upgrade-check, so there is no way to namespace them per daemon. A second daemon
5
+ // standing instances up on the same box therefore shares them, and tearing our
6
+ // instances down churns that shared network: observed 2026-08-04 to stop a
7
+ // co-running `auth-verify` env's instances during doc-guide runs.
8
+ //
9
+ // Until norsk-ctl can namespace the network per daemon (the real isolation fix,
10
+ // tracked as ET-3), the engine harness REFUSES to run when the box already carries
11
+ // foreign launched instances rather than silently clobber them. The decision itself
12
+ // is pure and unit-tested; `discoverForeignInstances` is the impure probe that feeds
13
+ // it the live list.
14
+ import { runCli } from "./daemon.js";
15
+ /** Set to `1` to run the engine tier even when foreign instances are live — for a
16
+ * single-tenant box/CI where those instances are your own and you accept the
17
+ * shared-network churn. */
18
+ export const BUSY_BOX_OVERRIDE_ENV = "NORSK_ALLOW_BUSY_BOX";
19
+ /** The box's primary daemon — the one an unscoped `norsk-ctl` command reaches. */
20
+ export const PRIMARY_DAEMON_PORT = Number(process.env.NORSK_CTL_PORT ?? 8333);
21
+ /** Decide whether the engine tier may run given the foreign instances live on the
22
+ * box. Pure: the caller supplies the discovered list and the override flag. */
23
+ export function quietBoxVerdict(foreignInstances, opts) {
24
+ if (foreignInstances.length === 0 || opts.override)
25
+ return { clear: true };
26
+ const list = foreignInstances.join(", ");
27
+ return {
28
+ clear: false,
29
+ message: `Refusing to run the engine tier: ${foreignInstances.length} foreign norsk-ctl ` +
30
+ `instance(s) are live on this box (${list}). Standing our own instances up shares the ` +
31
+ `box-global norsk-net / norsk-proxy singletons, and tearing them down would clobber ` +
32
+ `that environment (ET-3). Stop those instances first, or set ${BUSY_BOX_OVERRIDE_ENV}=1 ` +
33
+ `if they are your own and you accept the shared-network churn.`,
34
+ };
35
+ }
36
+ const defaultDiscoverDeps = {
37
+ answers: async (port) => {
38
+ try {
39
+ const r = await fetch(`http://localhost:${port}/api/ready`, { signal: AbortSignal.timeout(1000) });
40
+ return r.ok;
41
+ }
42
+ catch {
43
+ return false;
44
+ }
45
+ },
46
+ list: (storeDir, port) => runCli(storeDir, "--port", String(port), "instance", "list", "-o", "json"),
47
+ };
48
+ /** Running instances on the box's primary daemon that are not ours. Best-effort:
49
+ * if the primary daemon is unreachable or is our own daemon, the box is quiet
50
+ * from our perspective and this returns []. `probeStoreDir` is only used to root
51
+ * the CLI invocation; the query itself reads the daemon's HTTP API by port.
52
+ * The port is asked directly first: the CLI auto-starts a daemon on a silent
53
+ * port and polls it for 30 seconds, which is not a question but a side effect. */
54
+ export async function discoverForeignInstances(opts) {
55
+ if (opts.primaryPort === opts.ownPort)
56
+ return [];
57
+ const deps = opts.deps ?? defaultDiscoverDeps;
58
+ if (!(await deps.answers(opts.primaryPort)))
59
+ return [];
60
+ let r;
61
+ try {
62
+ r = await deps.list(opts.probeStoreDir, opts.primaryPort);
63
+ }
64
+ catch {
65
+ return [];
66
+ }
67
+ if (r.exitCode !== 0)
68
+ return [];
69
+ try {
70
+ const parsed = JSON.parse(r.stdout);
71
+ return (parsed.instances ?? []).filter((i) => i.status === "running" || i.status === "healthy").map((i) => i.id);
72
+ }
73
+ catch {
74
+ return [];
75
+ }
76
+ }
@@ -0,0 +1,30 @@
1
+ type Env = Record<string, string | undefined>;
2
+ export declare const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
3
+ export declare const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
4
+ /** The daemon's built-in presets and their files (norsk-ctl
5
+ * backend/src/docker/constants.ts: SOURCE_PRESETS). */
6
+ export declare const SAMPLE_MEDIA_FILES: Readonly<Record<string, string>>;
7
+ /** The same locations as shell expressions, for scripts the harness writes. */
8
+ export declare const ASSET_CACHE_SHELL = "${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}";
9
+ export declare const ASSET_BASE_SHELL = "${NORSK_ASSET_BASE:-https://assets.norsk.video}";
10
+ export declare function assetCacheRoot(env?: Env): string;
11
+ export declare function sampleMediaCacheDir(env?: Env): string;
12
+ type FetchFn = (url: string) => Promise<Response>;
13
+ /** The cached path of asset `key`, downloading it first if absent. Streamed to
14
+ * a uniquely-named partial and renamed into place, so concurrent fetchers (and
15
+ * daemons reading the same dir) only ever see a whole file. */
16
+ export declare function fetchAsset(key: string, opts?: {
17
+ env?: Env;
18
+ fetch?: FetchFn;
19
+ }): Promise<string>;
20
+ /** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
21
+ * store looks before downloading it. For a ctl that predates
22
+ * NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
23
+ * the cache, so it costs nothing and stays a real file inside the store, which
24
+ * is bind-mounted into the source container; copied across filesystems. */
25
+ export declare function seedPresetMedia(storeDir: string, preset: string, opts?: {
26
+ env?: Env;
27
+ fetch?: FetchFn;
28
+ link?: (from: string, to: string) => void;
29
+ }): Promise<void>;
30
+ export {};
@@ -0,0 +1,126 @@
1
+ // The shared asset cache: immutable assets (sample media, models) fetched once
2
+ // per cache -- `${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}/<key>`, one per
3
+ // dev box or CI runner instance -- and used from there by every daemon, run and
4
+ // repo that shares it: the same convention the norsk and
5
+ // norsk-studio Makefiles follow. Every test daemon gets a fresh store, so without
6
+ // it each one re-downloaded the same ~190 MB of sample media from S3.
7
+ //
8
+ // Lives outside every store and every TEST_TMP_BASE, so no teardown (rm -rf of a
9
+ // store, the stale-dir prune, the root nuke) can reach it; a store only ever
10
+ // holds hardlinks or copies of what is here.
11
+ import { closeSync, copyFileSync, linkSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, statSync, utimesSync, } from "node:fs";
12
+ import { open } from "node:fs/promises";
13
+ import { homedir } from "node:os";
14
+ import { dirname, join } from "node:path";
15
+ export const DEFAULT_ASSET_BASE = "https://assets.norsk.video";
16
+ export const SAMPLE_MEDIA_KEY_PREFIX = "media-examples/data";
17
+ /** The daemon's built-in presets and their files (norsk-ctl
18
+ * backend/src/docker/constants.ts: SOURCE_PRESETS). */
19
+ export const SAMPLE_MEDIA_FILES = { camera1: "action.mp4", camera2: "wildlife.ts" };
20
+ /** The same locations as shell expressions, for scripts the harness writes. */
21
+ export const ASSET_CACHE_SHELL = `\${NORSK_ASSET_CACHE:-$HOME/.cache/norsk/assets}`;
22
+ export const ASSET_BASE_SHELL = `\${NORSK_ASSET_BASE:-${DEFAULT_ASSET_BASE}}`;
23
+ export function assetCacheRoot(env = process.env) {
24
+ return env.NORSK_ASSET_CACHE || join(env.HOME || homedir(), ".cache", "norsk", "assets");
25
+ }
26
+ export function sampleMediaCacheDir(env = process.env) {
27
+ return join(assetCacheRoot(env), SAMPLE_MEDIA_KEY_PREFIX);
28
+ }
29
+ function hasContent(path) {
30
+ try {
31
+ const s = statSync(path);
32
+ return s.isFile() && s.size > 0;
33
+ }
34
+ catch {
35
+ return false;
36
+ }
37
+ }
38
+ /** A partial older than this was left by a killed run, not a live download. */
39
+ const STALE_PARTIAL_MS = 2 * 60 * 60 * 1000;
40
+ function sweepStalePartials(dir) {
41
+ for (const f of readdirSync(dir)) {
42
+ if (!f.includes(".part."))
43
+ continue;
44
+ const path = join(dir, f);
45
+ try {
46
+ if (Date.now() - statSync(path).mtimeMs > STALE_PARTIAL_MS)
47
+ rmSync(path, { force: true });
48
+ }
49
+ catch { }
50
+ }
51
+ }
52
+ /** Touch `<cache>/.used/<key>`, which the norsk repos' scripts/prune-asset-cache.sh
53
+ * reads to keep what is still in use. Never the cached file itself: checkouts
54
+ * hardlink it, and a newer mtime would look like a rebuild to make. */
55
+ function recordUse(root, key) {
56
+ const sidecar = join(root, ".used", key);
57
+ try {
58
+ mkdirSync(dirname(sidecar), { recursive: true });
59
+ closeSync(openSync(sidecar, "a"));
60
+ const now = new Date();
61
+ utimesSync(sidecar, now, now);
62
+ }
63
+ catch { }
64
+ }
65
+ /** The cached path of asset `key`, downloading it first if absent. Streamed to
66
+ * a uniquely-named partial and renamed into place, so concurrent fetchers (and
67
+ * daemons reading the same dir) only ever see a whole file. */
68
+ export async function fetchAsset(key, opts = {}) {
69
+ const env = opts.env ?? process.env;
70
+ const root = assetCacheRoot(env);
71
+ const dest = join(root, key);
72
+ if (hasContent(dest)) {
73
+ recordUse(root, key);
74
+ return dest;
75
+ }
76
+ const base = (env.NORSK_ASSET_BASE || DEFAULT_ASSET_BASE).replace(/\/+$/, "");
77
+ const res = await (opts.fetch ?? fetch)(`${base}/${key}`);
78
+ if (!res.ok || !res.body)
79
+ throw new Error(`fetch-asset ${key}: ${res.status} ${res.statusText}`);
80
+ mkdirSync(dirname(dest), { recursive: true });
81
+ sweepStalePartials(dirname(dest));
82
+ const part = `${dest}.part.${process.pid}.${Math.random().toString(36).slice(2)}`;
83
+ try {
84
+ const out = await open(part, "w");
85
+ try {
86
+ const reader = res.body.getReader();
87
+ for (let r = await reader.read(); !r.done; r = await reader.read())
88
+ await out.write(r.value);
89
+ }
90
+ finally {
91
+ await out.close();
92
+ }
93
+ renameSync(part, dest);
94
+ }
95
+ catch (e) {
96
+ rmSync(part, { force: true });
97
+ throw e;
98
+ }
99
+ recordUse(root, key);
100
+ console.error(`fetch-asset: ${key} cached at ${dest}`);
101
+ return dest;
102
+ }
103
+ /** Put a preset's file in `<storeDir>/sample-media`, where a daemon on that
104
+ * store looks before downloading it. For a ctl that predates
105
+ * NORSK_CTL_SAMPLE_MEDIA_DIR (a product pins a released one). Hardlinked from
106
+ * the cache, so it costs nothing and stays a real file inside the store, which
107
+ * is bind-mounted into the source container; copied across filesystems. */
108
+ export async function seedPresetMedia(storeDir, preset, opts = {}) {
109
+ const file = SAMPLE_MEDIA_FILES[preset];
110
+ if (!file)
111
+ return;
112
+ const dest = join(storeDir, "sample-media", file);
113
+ if (hasContent(dest))
114
+ return;
115
+ const cached = await fetchAsset(`${SAMPLE_MEDIA_KEY_PREFIX}/${file}`, opts);
116
+ mkdirSync(dirname(dest), { recursive: true });
117
+ const part = `${dest}.part.${process.pid}`;
118
+ rmSync(part, { force: true });
119
+ try {
120
+ (opts.link ?? linkSync)(cached, part);
121
+ }
122
+ catch {
123
+ copyFileSync(cached, part);
124
+ }
125
+ renameSync(part, dest);
126
+ }
package/source-pump.d.ts CHANGED
@@ -58,6 +58,7 @@ export interface StartCameraSourceOptions {
58
58
  /** Test seams. */
59
59
  fetch?: FetchLike;
60
60
  fileExists?: (path: string) => boolean;
61
+ seedPreset?: (storeDir: string, preset: string) => Promise<void>;
61
62
  }
62
63
  export declare function startCameraSource(opts: StartCameraSourceOptions): Promise<SourceHandle>;
63
64
  export interface SrtPumpTarget {
@@ -80,5 +81,6 @@ export declare function startSrtSources(opts: {
80
81
  timeoutMs?: number;
81
82
  fetch?: FetchLike;
82
83
  fileExists?: (path: string) => boolean;
84
+ seedPreset?: (storeDir: string, preset: string) => Promise<void>;
83
85
  }): Promise<SourceHandle[]>;
84
86
  export declare function stopAll(handles: readonly SourceHandle[]): Promise<void>;
package/source-pump.js CHANGED
@@ -20,6 +20,7 @@ import { existsSync, readFileSync } from "node:fs";
20
20
  import { basename, extname, join } from "node:path";
21
21
  import { storeDirForDaemonPort } from "./daemon-stores.js";
22
22
  import { pollUntil } from "./poll.js";
23
+ import { seedPresetMedia } from "./sample-media.js";
23
24
  /** Where every daemon the harness starts writes its ingress secret: the store
24
25
  * root (the guard's stateDir is `<storeDir>` under NORSK_CTL_STORE_DIR). One
25
26
  * reader for the whole harness — the three callers that spelled the filename
@@ -130,9 +131,17 @@ export async function startCameraSource(opts) {
130
131
  return { secret, headers: secret ? { "x-norsk-proxy": secret } : {} };
131
132
  };
132
133
  const base = `http://localhost:${opts.daemonPort}`;
134
+ const fields = assetFields(asset, opts.fileExists ?? existsSync);
135
+ // A daemon on a ctl that predates NORSK_CTL_SAMPLE_MEDIA_DIR looks in its own
136
+ // store, so put the preset there from the shared cache before it goes looking.
137
+ // Best-effort: if this fails, the daemon downloads the file itself.
138
+ const storeDir = storeDirForDaemonPort(opts.daemonPort);
139
+ if (storeDir !== undefined && typeof fields.preset === "string") {
140
+ await (opts.seedPreset ?? seedPresetMedia)(storeDir, fields.preset).catch((e) => console.error(`sample media: could not seed ${fields.preset} into ${storeDir}: ${e}`));
141
+ }
133
142
  const body = {
134
143
  instanceId: opts.instanceId,
135
- ...assetFields(asset, opts.fileExists ?? existsSync),
144
+ ...fields,
136
145
  port: opts.port,
137
146
  name,
138
147
  };
@@ -215,6 +224,7 @@ export async function startSrtSources(opts) {
215
224
  ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
216
225
  ...(opts.fetch !== undefined ? { fetch: opts.fetch } : {}),
217
226
  ...(opts.fileExists !== undefined ? { fileExists: opts.fileExists } : {}),
227
+ ...(opts.seedPreset !== undefined ? { seedPreset: opts.seedPreset } : {}),
218
228
  });
219
229
  handles.push(handle);
220
230
  }