@norskvideo/ctl-test-harness 0.1.43 → 0.1.45

package/container-logs.d.ts

@@ -32,6 +32,16 @@ export interface LogHit {
     container: string;
     line: string;
 }
+/** Copy every engine `logs` tree under an instance-runtime root to `dest`,
+ *  preserving the relative structure. Why this and not `docker logs`:
+ *  norsk-ctl bind-mounts each media container's `/var/log/norsk` to
+ *  `<runnerDir>/logs/media` on the host (override-generator), so the FULL
+ *  structured engine log (`debug.json`, every level — not the notice+ subset
+ *  `docker logs` sees, and immune to a decode-error stdout flood drowning the
+ *  tail) lives on disk there until teardown nukes the store. Call this BEFORE
+ *  cleanup to keep those logs for a CI artifact. Best-effort: never throws;
+ *  returns the number of `logs` trees copied. */
+export declare function preserveEngineLogs(runtimeRoot: string, dest: string): number;
 /** A string forbids a substring; a RegExp is tested per line. */
 export type LogPattern = string | RegExp;
 /** Every line, in any container of the instance, that matches a forbidden

package/container-logs.js

@@ -8,6 +8,8 @@
 // Pure argv/parse/format helpers; only the two exported probes shell out,
 // through an injectable spawn.
 import { spawnSync } from "node:child_process";
+import { cpSync, existsSync, readdirSync, statSync } from "node:fs";
+import { join, relative } from "node:path";
 const defaultSpawn = (cmd, args, opts) => {
     const r = spawnSync(cmd, args, opts);
     return {
@@ -16,7 +18,14 @@ const defaultSpawn = (cmd, args, opts) => {
         ...(r.error ? { error: r.error } : {}),
     };
 };
-const DEFAULT_TAIL = 80;
+// The diagnostic dump is a post-mortem: a container that has already failed,
+// whose logs die with it at teardown. 80 lines rarely reached back to the
+// cause (a ladder-node death, an nvenc failure, the FRC/surface errors that
+// precede it) — the tail is the only copy, so bias it wide. The media engine
+// log is the verbose one; a few thousand lines is cheap in an uploaded
+// artifact and routinely the difference between a diagnosable run and a
+// re-run.
+const DEFAULT_TAIL = 3000;
 /** `docker ps` argv listing "<name>\t<state>" for ONE instance's containers. */
 export function instanceContainersArgs(instanceId) {
     return ["ps", "-a", "--filter", `label=norsk-ctl.instance=${instanceId}`, "--format", "{{.Names}}\t{{.State}}"];
@@ -85,6 +94,54 @@ export function collectInstanceContainerLogs(instanceId, tail = DEFAULT_TAIL, sp
 export function dumpInstanceContainerLogs(instanceId, tail = DEFAULT_TAIL, spawn = defaultSpawn) {
     return formatContainerLogs(instanceId, collectInstanceContainerLogs(instanceId, tail, spawn));
 }
+/** Copy every engine `logs` tree under an instance-runtime root to `dest`,
+ *  preserving the relative structure. Why this and not `docker logs`:
+ *  norsk-ctl bind-mounts each media container's `/var/log/norsk` to
+ *  `<runnerDir>/logs/media` on the host (override-generator), so the FULL
+ *  structured engine log (`debug.json`, every level — not the notice+ subset
+ *  `docker logs` sees, and immune to a decode-error stdout flood drowning the
+ *  tail) lives on disk there until teardown nukes the store. Call this BEFORE
+ *  cleanup to keep those logs for a CI artifact. Best-effort: never throws;
+ *  returns the number of `logs` trees copied. */
+export function preserveEngineLogs(runtimeRoot, dest) {
+    if (!existsSync(runtimeRoot))
+        return 0;
+    let copied = 0;
+    const walk = (dir) => {
+        let names;
+        try {
+            names = readdirSync(dir);
+        }
+        catch {
+            return;
+        }
+        for (const name of names) {
+            const p = join(dir, name);
+            try {
+                if (!statSync(p).isDirectory())
+                    continue;
+            }
+            catch {
+                continue;
+            }
+            if (name === "logs") {
+                // Copy the whole tree wholesale; don't descend further.
+                try {
+                    cpSync(p, join(dest, relative(runtimeRoot, p)), { recursive: true });
+                    copied += 1;
+                }
+                catch {
+                    /* best-effort */
+                }
+            }
+            else {
+                walk(p);
+            }
+        }
+    };
+    walk(runtimeRoot);
+    return copied;
+}
 const matches = (pattern, line) => typeof pattern === "string" ? line.includes(pattern) : pattern.test(line);
 /** Every line, in any container of the instance, that matches a forbidden
  *  pattern. `clean` when there are none — including when docker cannot be

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@norskvideo/ctl-test-harness",
-  "version": "0.1.43",
+  "version": "0.1.45",
   "type": "module",
   "exports": {
     ".": {

package/smoke.js

@@ -19,7 +19,7 @@
 // rule); a product wanting typed commands bridges with toArgv in its own spec.
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { dumpInstanceContainerLogs } from "./container-logs.js";
+import { dumpInstanceContainerLogs, preserveEngineLogs } from "./container-logs.js";
 import { ensureRunnerOnNetwork, netReachMode, STUDIO_INTERNAL_PORT, studioBaseFrom, } from "./container-net.js";
 import { cleanupDaemon, requireLicenseFile, runCli, startDaemon, } from "./daemon.js";
 import { hashSlot } from "./harness-config.js";
@@ -391,6 +391,17 @@ export async function runProductSmoke(slug, spec, deps = defaultSmokeDeps) {
     catch (e) {
         journeyError = e;
     }
+    // Preserve the engine logs BEFORE anything tears the store down. norsk-ctl
+    // bind-mounts each media container's /var/log/norsk to the store, so the full
+    // structured debug.json is on disk here -- but `product remove`,
+    // `instance delete --purge`, cleanup and the root nuke below all destroy it.
+    // We pretty much always want the Norsk logs from a smoke run; when a CI log
+    // dir is set, copy every logs/ tree out first so the workflow can upload it.
+    const ciLogDir = process.env.NORSK_CI_LOG_DIR;
+    if (ciLogDir) {
+        const n = preserveEngineLogs(join(storeDir, "norsk-runtime"), join(ciLogDir, slug));
+        deps.log(`${slug}: preserved ${n} engine log tree(s) -> ${join(ciLogDir, slug)}`);
+    }
     if (handles.length)
         await deps.stopSources(handles).catch(() => { });
     // The daemon reaps its own control-plane containers on shutdown (daemon.ts,