@norskvideo/ctl-test-harness 0.1.116 → 0.1.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/daemon.d.ts +9 -1
- package/daemon.js +60 -32
- package/demo/run.d.ts +7 -0
- package/demo/run.js +41 -24
- package/media-conversion/local-studio.js +3 -1
- package/package.json +1 -1
package/daemon.d.ts
CHANGED
|
@@ -117,6 +117,10 @@ export declare function startDaemon(storeDir: string, options?: StartDaemonOptio
|
|
|
117
117
|
/** How long one `instance delete` may take before teardown stops waiting for
|
|
118
118
|
* it and reaps the instance's containers itself. A healthy delete is seconds. */
|
|
119
119
|
export declare const INSTANCE_DELETE_TIMEOUT_MS = 20000;
|
|
120
|
+
/** How long `stopDaemon` (`shutdown`: instances DELETE, proxy compose-down, a
|
|
121
|
+
* 5s wait) may take before teardown moves on to the SIGKILL and the container
|
|
122
|
+
* wait, which cope with a daemon that never stopped. */
|
|
123
|
+
export declare const STOP_DAEMON_TIMEOUT_MS = 30000;
|
|
120
124
|
/**
|
|
121
125
|
* Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
|
|
122
126
|
* process, rm store. The teardown actions are injected as pre-bound thunks
|
|
@@ -131,7 +135,8 @@ export declare const INSTANCE_DELETE_TIMEOUT_MS = 20000;
|
|
|
131
135
|
* composition guide's engine running that way for 2.6 hours on 2026-10-03; it
|
|
132
136
|
* grew to 28 GB and the host OOM-killed the next morning's product CI. So each
|
|
133
137
|
* delete is bounded, and whatever an instance leaves behind is then removed by
|
|
134
|
-
* its `norsk-ctl.instance` label, which needs only Docker.
|
|
138
|
+
* its `norsk-ctl.instance` label, which needs only Docker. The daemon stop is
|
|
139
|
+
* bounded the same way (ET-5).
|
|
135
140
|
*/
|
|
136
141
|
export declare function cleanupDaemon(opts: {
|
|
137
142
|
deleteInstance: (id: string) => Promise<unknown>;
|
|
@@ -149,6 +154,9 @@ export declare function cleanupDaemon(opts: {
|
|
|
149
154
|
* foreign endpoint keeps `compose down` from removing the network. */
|
|
150
155
|
leaveNetwork?: (id: string) => void;
|
|
151
156
|
deleteTimeoutMs?: number;
|
|
157
|
+
stopDaemonTimeoutMs?: number;
|
|
158
|
+
/** Told how long each step took, after it finishes or throws. */
|
|
159
|
+
onStep?: (step: string, ms: number) => void;
|
|
152
160
|
}): Promise<void>;
|
|
153
161
|
/** Tear down a direct-API test: stop proxy, remove instances, rm temp dir. */
|
|
154
162
|
export declare function cleanupDirect(opts: {
|
package/daemon.js
CHANGED
|
@@ -345,6 +345,20 @@ async function awaitContainerShutdown(containers) {
|
|
|
345
345
|
/** How long one `instance delete` may take before teardown stops waiting for
|
|
346
346
|
* it and reaps the instance's containers itself. A healthy delete is seconds. */
|
|
347
347
|
export const INSTANCE_DELETE_TIMEOUT_MS = 20_000;
|
|
348
|
+
/** How long `stopDaemon` (`shutdown`: instances DELETE, proxy compose-down, a
|
|
349
|
+
* 5s wait) may take before teardown moves on to the SIGKILL and the container
|
|
350
|
+
* wait, which cope with a daemon that never stopped. */
|
|
351
|
+
export const STOP_DAEMON_TIMEOUT_MS = 30_000;
|
|
352
|
+
/** Run `step`, then report how long it took — thrown or not. */
|
|
353
|
+
async function timed(name, onStep, step) {
|
|
354
|
+
const started = Date.now();
|
|
355
|
+
try {
|
|
356
|
+
return await step();
|
|
357
|
+
}
|
|
358
|
+
finally {
|
|
359
|
+
onStep?.(name, Date.now() - started);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
348
362
|
/**
|
|
349
363
|
* Tear down a daemon-based test: delete instances, stop proxy, stop daemon, kill
|
|
350
364
|
* process, rm store. The teardown actions are injected as pre-bound thunks
|
|
@@ -359,58 +373,72 @@ export const INSTANCE_DELETE_TIMEOUT_MS = 20_000;
|
|
|
359
373
|
* composition guide's engine running that way for 2.6 hours on 2026-10-03; it
|
|
360
374
|
* grew to 28 GB and the host OOM-killed the next morning's product CI. So each
|
|
361
375
|
* delete is bounded, and whatever an instance leaves behind is then removed by
|
|
362
|
-
* its `norsk-ctl.instance` label, which needs only Docker.
|
|
376
|
+
* its `norsk-ctl.instance` label, which needs only Docker. The daemon stop is
|
|
377
|
+
* bounded the same way (ET-5).
|
|
363
378
|
*/
|
|
364
379
|
export async function cleanupDaemon(opts) {
|
|
365
380
|
const deleteTimeoutMs = opts.deleteTimeoutMs ?? INSTANCE_DELETE_TIMEOUT_MS;
|
|
381
|
+
const stopDaemonTimeoutMs = opts.stopDaemonTimeoutMs ?? STOP_DAEMON_TIMEOUT_MS;
|
|
366
382
|
const leave = opts.leaveNetwork ?? ((id) => currentInstanceNetworks().leave(id));
|
|
367
383
|
for (const id of opts.instances ?? []) {
|
|
368
384
|
leave(id);
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
385
|
+
await timed(`instance delete ${id}`, opts.onStep, async () => {
|
|
386
|
+
try {
|
|
387
|
+
if ((await settleWithin(opts.deleteInstance(id), deleteTimeoutMs)) === "timeout")
|
|
388
|
+
console.error(`cleanup: delete ${id} gave no answer in ${deleteTimeoutMs}ms`);
|
|
389
|
+
}
|
|
390
|
+
catch (e) {
|
|
391
|
+
console.error(`cleanup: delete ${id} failed: ${e}`);
|
|
392
|
+
}
|
|
393
|
+
});
|
|
376
394
|
}
|
|
377
395
|
if (!opts.leaveNetwork)
|
|
378
396
|
currentInstanceNetworks().leaveAll();
|
|
379
397
|
const reap = opts.reapInstance ?? reapInstance;
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
398
|
+
if ((opts.instances ?? []).length > 0)
|
|
399
|
+
await timed("reap instances", opts.onStep, () => {
|
|
400
|
+
for (const id of opts.instances ?? []) {
|
|
401
|
+
const left = reap(id);
|
|
402
|
+
if (left.length > 0)
|
|
403
|
+
console.error(`cleanup: removed ${left.length} container(s) instance ${id} left running`);
|
|
404
|
+
}
|
|
405
|
+
});
|
|
385
406
|
// Stop the proxy unless the caller says it never ran one (`proxy: false`).
|
|
386
407
|
// It used to need `proxy: true`, and three callers passing a real stopProxy
|
|
387
408
|
// omitted it — so `shutdown --daemon-only` left their proxy pair running.
|
|
388
409
|
// On the shared ar-arm-vm runner a leftover pair then answered for the next
|
|
389
410
|
// job's proxy (2026-10-02, norsk-ctl Checks 37048720935).
|
|
390
411
|
if (opts.proxy ?? true)
|
|
412
|
+
await timed("stop proxy", opts.onStep, async () => {
|
|
413
|
+
try {
|
|
414
|
+
await opts.stopProxy();
|
|
415
|
+
}
|
|
416
|
+
catch (e) {
|
|
417
|
+
console.error(`cleanup: proxy stop failed: ${e}`);
|
|
418
|
+
}
|
|
419
|
+
});
|
|
420
|
+
await timed("stop daemon", opts.onStep, async () => {
|
|
391
421
|
try {
|
|
392
|
-
await opts.
|
|
422
|
+
if ((await settleWithin(opts.stopDaemon(), stopDaemonTimeoutMs)) === "timeout")
|
|
423
|
+
console.error(`cleanup: daemon stop gave no answer in ${stopDaemonTimeoutMs}ms`);
|
|
393
424
|
}
|
|
394
425
|
catch (e) {
|
|
395
|
-
console.error(`cleanup:
|
|
426
|
+
console.error(`cleanup: daemon stop failed: ${e}`);
|
|
396
427
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
}
|
|
412
|
-
await awaitContainerShutdown(opts.containers);
|
|
413
|
-
rmSync(opts.storeDir, { recursive: true, force: true });
|
|
428
|
+
});
|
|
429
|
+
const daemon = opts.daemon;
|
|
430
|
+
if (daemon)
|
|
431
|
+
await timed("kill daemon", opts.onStep, async () => {
|
|
432
|
+
daemon.kill();
|
|
433
|
+
// Escalate to SIGKILL after 3s if process hasn't exited
|
|
434
|
+
const race = Promise.race([daemon.exited, sleep(3000).then(() => "timeout")]);
|
|
435
|
+
if ((await race) === "timeout") {
|
|
436
|
+
daemon.kill(9);
|
|
437
|
+
await Promise.race([daemon.exited, sleep(2000)]);
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
await timed("await container shutdown", opts.onStep, () => awaitContainerShutdown(opts.containers));
|
|
441
|
+
await timed("remove store", opts.onStep, () => rmSync(opts.storeDir, { recursive: true, force: true }));
|
|
414
442
|
}
|
|
415
443
|
/** Tear down a direct-API test: stop proxy, remove instances, rm temp dir. */
|
|
416
444
|
export async function cleanupDirect(opts) {
|
package/demo/run.d.ts
CHANGED
|
@@ -92,6 +92,9 @@ export interface DemoDeps {
|
|
|
92
92
|
daemon: DaemonProcess | null;
|
|
93
93
|
storeDir: string;
|
|
94
94
|
containers: string[];
|
|
95
|
+
/** Told how long each of the cleanup's own steps took (cleanupDaemon's
|
|
96
|
+
* onStep). Optional: a custom cleanup may ignore it. */
|
|
97
|
+
onStep?: (step: string, ms: number) => void;
|
|
95
98
|
}): Promise<void>;
|
|
96
99
|
/** Docker leaves root-owned bind-mount targets under the store (smoke.ts). */
|
|
97
100
|
nukeStoreAsRoot(storeDir: string): void;
|
|
@@ -434,6 +437,10 @@ export declare class DemoSession {
|
|
|
434
437
|
* workflow's artifact upload. No env var (local dev) means no-op; every
|
|
435
438
|
* failure inside is swallowed by the helper. */
|
|
436
439
|
private preserveContainerLogs;
|
|
440
|
+
/** One `teardown: <step> <ms>ms` line per teardown step (ET-5), so the next
|
|
441
|
+
* teardown that outruns an afterAll names the step that was slow. */
|
|
442
|
+
private logStep;
|
|
443
|
+
private timedStep;
|
|
437
444
|
teardown(opts: {
|
|
438
445
|
instances: string[];
|
|
439
446
|
handles: SourceHandle[];
|
package/demo/run.js
CHANGED
|
@@ -888,6 +888,20 @@ export class DemoSession {
|
|
|
888
888
|
this.deps.log(`preserved ${n} container log(s) for ${id} -> ${join(ciLogDir, "containers", id)}`);
|
|
889
889
|
}
|
|
890
890
|
}
|
|
891
|
+
/** One `teardown: <step> <ms>ms` line per teardown step (ET-5), so the next
|
|
892
|
+
* teardown that outruns an afterAll names the step that was slow. */
|
|
893
|
+
logStep = (step, ms) => {
|
|
894
|
+
this.deps.log(`teardown: ${step} ${ms}ms`);
|
|
895
|
+
};
|
|
896
|
+
async timedStep(step, fn) {
|
|
897
|
+
const started = Date.now();
|
|
898
|
+
try {
|
|
899
|
+
return await fn();
|
|
900
|
+
}
|
|
901
|
+
finally {
|
|
902
|
+
this.logStep(step, Date.now() - started);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
891
905
|
async teardown(opts) {
|
|
892
906
|
// FIRST, while the containers are still standing. Everything below removes
|
|
893
907
|
// them, and `docker logs` is the only copy of the studio container's stdout
|
|
@@ -895,40 +909,43 @@ export class DemoSession {
|
|
|
895
909
|
// gates tell you to go and read. Unconditional rather than on-failure,
|
|
896
910
|
// because teardown is not told the verdict, and a tail per container is
|
|
897
911
|
// cheap next to a re-run.
|
|
898
|
-
this.preserveContainerLogs(opts.instances);
|
|
912
|
+
await this.timedStep("preserve container logs", () => this.preserveContainerLogs(opts.instances));
|
|
899
913
|
if (opts.handles.length)
|
|
900
|
-
await this.deps.stopSources(opts.handles).catch(() => { });
|
|
914
|
+
await this.timedStep("stop sources", () => this.deps.stopSources(opts.handles).catch(() => { }));
|
|
901
915
|
// Extras sit on the instance networks; they go before any instance does.
|
|
902
|
-
this.stopExtras();
|
|
916
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
903
917
|
if (this.policy === "reuse") {
|
|
904
918
|
for (const id of opts.instances) {
|
|
905
919
|
currentInstanceNetworks().leave(id);
|
|
906
|
-
const r = await this.cli(["instance", "delete", id, "--purge"]);
|
|
920
|
+
const r = await this.timedStep(`instance delete ${id}`, () => this.cli(["instance", "delete", id, "--purge"]));
|
|
907
921
|
if (r.exitCode !== 0)
|
|
908
922
|
this.deps.log(`instance delete ${id}: ${(r.stderr || r.stdout).trim()}`);
|
|
909
923
|
}
|
|
910
|
-
this.stopExtras();
|
|
911
|
-
this.dev?.kill();
|
|
924
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
925
|
+
await this.timedStep("stop dev backend", () => this.dev?.kill());
|
|
912
926
|
return;
|
|
913
927
|
}
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
this.
|
|
928
|
+
await this.timedStep("cleanup", async () => {
|
|
929
|
+
try {
|
|
930
|
+
await this.deps.cleanup({
|
|
931
|
+
deleteInstance: (id) => this.cli(["instance", "delete", id, "--purge"]),
|
|
932
|
+
stopDaemon: () => this.cli(["shutdown"]),
|
|
933
|
+
instances: opts.instances,
|
|
934
|
+
daemon: this.daemon,
|
|
935
|
+
storeDir: this.storeDir,
|
|
936
|
+
containers: [...opts.instances.flatMap(instanceContainerNames), ...this.extraContainers],
|
|
937
|
+
onStep: this.logStep,
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
catch (e) {
|
|
941
|
+
this.deps.log(`cleanup could not remove the store itself (${e instanceof Error ? e.message : String(e)}); nuking as root`);
|
|
942
|
+
reportForeignOwners(this.storeDir, (l) => this.deps.log(l));
|
|
943
|
+
}
|
|
944
|
+
});
|
|
945
|
+
await this.timedStep("remove moq relay", () => this.removeMoqRelay());
|
|
946
|
+
await this.timedStep("stop extras", () => this.stopExtras());
|
|
947
|
+
await this.timedStep("stop dev backend", () => this.dev?.kill());
|
|
948
|
+
await this.timedStep("nuke store as root", () => this.deps.nukeStoreAsRoot(this.storeDir));
|
|
932
949
|
}
|
|
933
950
|
}
|
|
934
951
|
/** `demo check --mode standalone --export-only` (05-demo s4): build the
|
|
@@ -31,7 +31,9 @@ export function withLocalLinks(workflowYaml) {
|
|
|
31
31
|
export function localStudioLauncher(opts) {
|
|
32
32
|
const port = opts.port ?? 8010;
|
|
33
33
|
const workDir = opts.workDir ?? mkdtempSync(join(tmpdir(), "conformance-studio-"));
|
|
34
|
-
|
|
34
|
+
// Folder form (#381): where Studio resolves `server.document: workflow.yml`
|
|
35
|
+
// first, as a norsk-ctl launch writes it.
|
|
36
|
+
const saves = join(workDir, "data", "studio-save-files", "workflow");
|
|
35
37
|
mkdirSync(saves, { recursive: true });
|
|
36
38
|
mkdirSync(join(workDir, "logs"), { recursive: true });
|
|
37
39
|
writeFileSync(join(saves, "workflow.yml"), opts.localLinks ? withLocalLinks(opts.workflowYaml) : opts.workflowYaml);
|