@norskvideo/ctl-test-harness 0.1.97 → 0.1.99

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,31 +1,52 @@
1
- import type { MediaConversionV1 } from "@norskvideo/ctl-product-template-schema";
1
+ import type { MediaConversion } from "@norskvideo/ctl-product-template-schema";
2
2
  export type MediaKind = "video" | "audio" | "subtitle" | "ancillary";
3
+ /** A frame as the harness saw it: its timestamp (seconds) and when it was
4
+ * seen (wall-clock ms). The pair places it against the source's clock. */
5
+ export type Frame = {
6
+ pts: number;
7
+ at: number;
8
+ };
3
9
  /** One stream that came back: its kind, its format as the engine reports it,
4
- * and its frames' timestamps (exact, in lowest terms, as "n/d"). */
10
+ * and its frames. */
5
11
  export type ObservedStream = {
6
12
  kind: MediaKind;
7
13
  format: Record<string, string | number>;
8
- stamps: string[];
14
+ frames: Frame[];
9
15
  };
10
16
  /** Everything the source sent, and what came back in each phase of the run
11
17
  * ("initial", "after restart"). */
12
18
  export type Observation = {
13
- sent: Set<string>;
19
+ sent: Frame[];
14
20
  phases: {
15
21
  label: string;
16
22
  streams: ObservedStream[];
17
23
  }[];
18
24
  };
25
+ /** How far a returned frame may trail the moment the source sent its instant,
26
+ * and how fast that lag may change, for the timeline to count as inherited. */
27
+ export type TimelineBounds = {
28
+ maxLagMs: number;
29
+ maxDriftMsPerS: number;
30
+ };
31
+ export declare const DEFAULT_TIMELINE_BOUNDS: TimelineBounds;
19
32
  export type Check = {
20
33
  name: string;
21
34
  ok: boolean;
22
35
  detail: string;
23
36
  };
37
+ /** What a run checks the returned programme against: the tracks the source
38
+ * sent, per kind, and the format the caller expects of each kind from its
39
+ * target (keys as the engine reports them; others are not checked). */
40
+ export type ProgrammeExpectation = {
41
+ sent: Partial<Record<MediaKind, number>>;
42
+ formats?: Partial<Record<MediaKind, Record<string, string | number>>>;
43
+ };
24
44
  export declare function evaluateMediaConversion(opts: {
25
- declaration: MediaConversionV1;
26
- /** The launch values, so format references can be filled in. */
27
- parameters: Record<string, string>;
45
+ declaration: MediaConversion;
46
+ /** The programme the source sent, and the formats to expect back. */
47
+ programme: ProgrammeExpectation;
28
48
  observation: Observation;
29
49
  /** Fewer frames than this on a stream counts as the output not arriving. */
30
50
  minFrames: number;
51
+ timeline?: TimelineBounds;
31
52
  }): Check[];
@@ -1,66 +1,94 @@
1
- const PARAMETER_REFERENCE = /\$\{([^}]*)\}/g;
2
- function resolveFormat(value, parameters) {
3
- let missing = false;
4
- const resolved = value.replace(PARAMETER_REFERENCE, (_, name) => {
5
- const v = parameters[name];
6
- if (v === undefined)
7
- missing = true;
8
- return v ?? "";
9
- });
10
- return missing ? undefined : resolved;
11
- }
1
+ export const DEFAULT_TIMELINE_BOUNDS = { maxLagMs: 3000, maxDriftMsPerS: 10 };
12
2
  export function evaluateMediaConversion(opts) {
13
- const { declaration, parameters, observation, minFrames } = opts;
3
+ const { declaration, observation, minFrames } = opts;
4
+ const bounds = opts.timeline ?? DEFAULT_TIMELINE_BOUNDS;
5
+ const sourceOffset = median(observation.sent.map((f) => f.at - f.pts * 1000));
14
6
  const checks = [];
15
7
  for (const phase of observation.phases) {
16
8
  const at = (name) => `${phase.label}: ${name}`;
17
- const byKind = (kind) => phase.streams.filter((s) => s.kind === kind);
18
- const problems = [];
19
- for (const output of declaration.outputs) {
20
- const expected = output.count ?? 1;
21
- const streams = byKind(output.kind).filter((s) => s.stamps.length >= minFrames);
22
- if (streams.length !== expected) {
23
- problems.push(`${output.name}: expected ${expected} ${output.kind} stream(s) of ${minFrames}+ frames, got ${streams.length}`);
24
- }
25
- }
26
- const declaredKinds = new Set(declaration.outputs.map((o) => o.kind));
27
- for (const s of phase.streams) {
28
- if (!declaredKinds.has(s.kind))
29
- problems.push(`an undeclared ${s.kind} stream came back`);
9
+ for (const c of judgeProgramme(phase.streams, opts.programme, minFrames))
10
+ checks.push({ ...c, name: at(c.name) });
11
+ if (declaration.timeline === "inherit") {
12
+ checks.push({ name: at("timeline"), ...judgeTimeline(phase.streams, sourceOffset, bounds) });
30
13
  }
31
- checks.push({ name: at("outputs"), ok: problems.length === 0, detail: problems.join("; ") || "as declared" });
32
- for (const output of declaration.outputs) {
33
- const mismatches = [];
34
- for (const [key, declared] of Object.entries(output.format ?? {})) {
35
- const want = resolveFormat(declared, parameters);
36
- if (want === undefined) {
37
- mismatches.push(`${key}: ${declared} refers to a parameter with no value`);
38
- continue;
39
- }
40
- for (const s of byKind(output.kind)) {
41
- const got = s.format[key];
42
- if (String(got) !== want)
43
- mismatches.push(`${key}: declared ${want}, got ${got}`);
44
- }
14
+ }
15
+ return checks;
16
+ }
17
+ function median(xs) {
18
+ if (xs.length === 0)
19
+ return Number.NaN;
20
+ const sorted = [...xs].sort((a, b) => a - b);
21
+ const mid = Math.floor(sorted.length / 2);
22
+ return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
23
+ }
24
+ /** Least-squares slope of y over x. */
25
+ function slope(points) {
26
+ const n = points.length;
27
+ const mx = points.reduce((a, p) => a + p.x, 0) / n;
28
+ const my = points.reduce((a, p) => a + p.y, 0) / n;
29
+ const sxx = points.reduce((a, p) => a + (p.x - mx) ** 2, 0);
30
+ return sxx === 0 ? 0 : points.reduce((a, p) => a + (p.x - mx) * (p.y - my), 0) / sxx;
31
+ }
32
+ // An inherited timeline is the source's clock: each returned frame arrives a
33
+ // little after the source sent its instant, and that lag holds steady. The
34
+ // frames need not be the source's own (a frame rate change, or audio
35
+ // re-framed by an encoder, makes new ones); a converter keeping its own clock
36
+ // is out by the time since either clock began, and one whose clock runs at
37
+ // another rate drifts.
38
+ function judgeTimeline(streams, sourceOffset, bounds) {
39
+ const frames = streams.flatMap((s) => s.frames);
40
+ // Judges only what arrived; nothing arriving is the outputs check's failure.
41
+ if (frames.length === 0 || Number.isNaN(sourceOffset))
42
+ return { ok: true, detail: "no frames to judge" };
43
+ const lags = frames.map((f) => ({ x: f.at, y: f.at - f.pts * 1000 - sourceOffset }));
44
+ const values = lags.map((l) => l.y);
45
+ const lo = Math.min(...values);
46
+ const hi = Math.max(...values);
47
+ const problems = [];
48
+ // Slack below zero for the two sides' wall clocks being read a little apart.
49
+ if (lo < -100 || hi > bounds.maxLagMs) {
50
+ problems.push(`frames trail the source by ${Math.round(lo)} to ${Math.round(hi)} ms, outside 0-${bounds.maxLagMs} ms`);
51
+ }
52
+ const worstDrift = Math.max(...streams
53
+ .filter((s) => s.frames.length >= 2)
54
+ .map((s) => Math.abs(slope(s.frames.map((f) => ({ x: f.at, y: f.at - f.pts * 1000 })))) * 1000), 0);
55
+ if (worstDrift > bounds.maxDriftMsPerS) {
56
+ problems.push(`a stream drifts ${worstDrift.toFixed(1)} ms/s against the source, over ${bounds.maxDriftMsPerS}`);
57
+ }
58
+ return {
59
+ ok: problems.length === 0,
60
+ detail: problems.join("; ") ||
61
+ `${frames.length} frames on the source's clock, ${Math.round(median(values))} ms behind it (${Math.round(lo)}-${Math.round(hi)}), drift ${worstDrift.toFixed(1)} ms/s`,
62
+ };
63
+ }
64
+ // A conversion returns the programme: the source's tracks, each converted or passed, so
65
+ // what comes back is as many tracks of each kind as went, in the formats the
66
+ // caller's target asks for.
67
+ function judgeProgramme(streams, expect, minFrames) {
68
+ const problems = [];
69
+ const kinds = new Set([...Object.keys(expect.sent), ...streams.map((s) => s.kind)]);
70
+ for (const kind of kinds) {
71
+ const want = expect.sent[kind] ?? 0;
72
+ const got = streams.filter((s) => s.kind === kind && s.frames.length >= minFrames).length;
73
+ if (got !== want)
74
+ problems.push(`expected ${want} ${kind} track(s) of ${minFrames}+ frames, got ${got}`);
75
+ }
76
+ const checks = [
77
+ { name: "programme", ok: problems.length === 0, detail: problems.join("; ") || "every track came back" },
78
+ ];
79
+ for (const [kind, format] of Object.entries(expect.formats ?? {})) {
80
+ const mismatches = [];
81
+ for (const s of streams.filter((x) => x.kind === kind)) {
82
+ for (const [key, want] of Object.entries(format)) {
83
+ if (String(s.format[key]) !== String(want))
84
+ mismatches.push(`${key}: expected ${want}, got ${s.format[key]}`);
45
85
  }
46
- checks.push({
47
- name: at(`format of ${output.name}`),
48
- ok: mismatches.length === 0,
49
- detail: mismatches.join("; ") || "as declared",
50
- });
51
- }
52
- if (declaration.timeline === "inherit") {
53
- const stamps = phase.streams.flatMap((s) => s.stamps);
54
- const foreign = stamps.filter((t) => !observation.sent.has(t));
55
- // Judges only what arrived; nothing arriving is the outputs check's failure.
56
- checks.push({
57
- name: at("timeline"),
58
- ok: foreign.length === 0,
59
- detail: foreign.length === 0
60
- ? `${stamps.length} frames, all on the source's timestamps`
61
- : `${foreign.length} of ${stamps.length} frames carry timestamps the source never sent (first: ${foreign.slice(0, 3).join(", ")})`,
62
- });
63
86
  }
87
+ checks.push({
88
+ name: `format of ${kind}`,
89
+ ok: mismatches.length === 0,
90
+ detail: mismatches.join("; ") || "as expected",
91
+ });
64
92
  }
65
93
  return checks;
66
94
  }
@@ -1,8 +1,8 @@
1
1
  export type { CliRunner, CtlLauncherOptions } from "./ctl.js";
2
2
  export { ctlLauncher } from "./ctl.js";
3
- export type { Check, MediaKind, Observation, ObservedStream } from "./evaluate.js";
4
- export { evaluateMediaConversion } from "./evaluate.js";
3
+ export type { Check, Frame, MediaKind, Observation, ObservedStream, ProgrammeExpectation, TimelineBounds, } from "./evaluate.js";
4
+ export { DEFAULT_TIMELINE_BOUNDS, evaluateMediaConversion } from "./evaluate.js";
5
5
  export type { LocalStudioOptions } from "./local-studio.js";
6
6
  export { interpolate, localStudioLauncher, withLocalLinks } from "./local-studio.js";
7
- export type { ConformanceLinks, ConformanceOptions, ConformanceResult, ConversionLauncher } from "./run.js";
8
- export { runMediaConversionConformance } from "./run.js";
7
+ export type { ConformanceLinks, ConformanceOptions, ConformanceResult, ConversionLauncher, SourceSettings, } from "./run.js";
8
+ export { programmeKeys, runMediaConversionConformance } from "./run.js";
@@ -1,6 +1,6 @@
1
1
  // The norsk.io/media-conversion conformance harness (norsk-ctl C3): proves a
2
2
  // template honours the interface it declares. See run.ts.
3
3
  export { ctlLauncher } from "./ctl.js";
4
- export { evaluateMediaConversion } from "./evaluate.js";
4
+ export { DEFAULT_TIMELINE_BOUNDS, evaluateMediaConversion } from "./evaluate.js";
5
5
  export { interpolate, localStudioLauncher, withLocalLinks } from "./local-studio.js";
6
- export { runMediaConversionConformance } from "./run.js";
6
+ export { programmeKeys, runMediaConversionConformance } from "./run.js";
@@ -1,6 +1,6 @@
1
- import type { MediaConversionV1 } from "@norskvideo/ctl-product-template-schema";
2
- import { type Norsk, type NorskLinkTransport, type StreamKey } from "@norskvideo/norsk-sdk";
3
- import { type Check, type Observation } from "./evaluate.js";
1
+ import type { MediaConversion } from "@norskvideo/ctl-product-template-schema";
2
+ import { type ChannelLayout, type FrameRate, type Norsk, type NorskLinkTransport, type StreamKey } from "@norskvideo/norsk-sdk";
3
+ import { type Check, type Observation, type ProgrammeExpectation } from "./evaluate.js";
4
4
  export type ConversionLauncher = {
5
5
  /** Start the template with these launch values. Resolves once it is launched;
6
6
  * the harness waits for the return itself. */
@@ -21,10 +21,15 @@ export type ConformanceLinks = {
21
21
  export type ConformanceOptions = {
22
22
  /** The engine the harness (the caller) runs on. */
23
23
  norsk: Norsk;
24
- declaration: MediaConversionV1;
25
- /** The template's other parameters, as the operator would set them. Format
26
- * references in the declaration are filled in from these. */
27
- parameters: Record<string, string>;
24
+ declaration: MediaConversion;
25
+ /** The template's other parameters, as the operator would set them. */
26
+ parameters?: Record<string, string>;
27
+ /** The target the caller hands over (filled into the target binding), and
28
+ * the formats it expects back of each kind as the engine reports them. */
29
+ target: unknown;
30
+ expectFormats?: ProgrammeExpectation["formats"];
31
+ /** The test sources: 1920x1080 at 25 fps, and 48 kHz stereo, unless set. */
32
+ sources?: SourceSettings;
28
33
  launcher: ConversionLauncher;
29
34
  links: ConformanceLinks;
30
35
  /** Filled into the engineVersion binding, when the template declares one. */
@@ -37,6 +42,17 @@ export type ConformanceOptions = {
37
42
  /** Names are made unique per run; this is their prefix. */
38
43
  prefix?: string;
39
44
  };
45
+ export type SourceSettings = {
46
+ video?: {
47
+ width: number;
48
+ height: number;
49
+ frameRate: FrameRate;
50
+ };
51
+ audio?: {
52
+ sampleRate: 44100 | 48000 | 96000;
53
+ channelLayout: ChannelLayout;
54
+ };
55
+ };
40
56
  export type ConformanceResult = {
41
57
  ok: boolean;
42
58
  checks: Check[];
@@ -1,13 +1,6 @@
1
1
  import { selectAudio, selectVideo, } from "@norskvideo/norsk-sdk";
2
2
  import { evaluateMediaConversion, } from "./evaluate.js";
3
- const gcd = (a, b) => (b === 0n ? (a < 0n ? -a : a) : gcd(b, a % b));
4
- // Exact, in lowest terms: the two sides may express one instant in different units.
5
- function stamp(t) {
6
- const n = BigInt(t.n);
7
- const d = BigInt(t.d);
8
- const g = gcd(n, d) || 1n;
9
- return `${n / g}/${d / g}`;
10
- }
3
+ const seconds = (t) => Number(t.n) / Number(t.d);
11
4
  const keyOf = (k) => `${k.sourceName}/${k.programNumber}/${k.streamId}/${k.renditionName}`;
12
5
  const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
13
6
  /** The source programme's tracks: one source name, a stream id each (video
@@ -25,14 +18,18 @@ export function programmeKeys(prefix, counts) {
25
18
  }));
26
19
  }
27
20
  /** The source programme's tracks, and every node behind them (to close). */
28
- async function startSources(norsk, prefix, declaration) {
21
+ async function startSources(norsk, prefix, declaration, settings = {}) {
29
22
  if (declaration.inputs.length !== 1) {
30
23
  throw new Error(`conformance: v1 drives exactly one input, the declaration has ${declaration.inputs.length}`);
31
24
  }
32
25
  const kinds = declaration.inputs[0]?.kinds ?? {};
26
+ // A conversion returns the programme, whatever it holds, so a run sends one
27
+ // track of every kind the declaration takes, to see it come back.
33
28
  const want = (kind) => {
34
29
  const count = kinds[kind];
35
- return typeof count === "number" ? count : (count?.min ?? 0);
30
+ if (typeof count === "number")
31
+ return count;
32
+ return count ? Math.max(count.min, Math.min(1, count.max)) : 0;
36
33
  };
37
34
  for (const kind of ["subtitle", "ancillary"]) {
38
35
  if (want(kind) > 0)
@@ -46,17 +43,22 @@ async function startSources(norsk, prefix, declaration) {
46
43
  ? await norsk.input.videoTestCard({
47
44
  id,
48
45
  sourceName: id,
49
- resolution: { width: 1920, height: 1080 },
50
- frameRate: { frames: 25, seconds: 1 },
46
+ resolution: { width: settings.video?.width ?? 1920, height: settings.video?.height ?? 1080 },
47
+ frameRate: settings.video?.frameRate ?? { frames: 25, seconds: 1 },
51
48
  pattern: "smpte75",
52
49
  })
53
- : await norsk.input.audioSignal({ id, sourceName: id, channelLayout: "stereo", sampleRate: 48000 });
50
+ : await norsk.input.audioSignal({
51
+ id,
52
+ sourceName: id,
53
+ channelLayout: settings.audio?.channelLayout ?? "stereo",
54
+ sampleRate: settings.audio?.sampleRate ?? 48000,
55
+ });
54
56
  const keyed = await norsk.processor.transform.streamKeyOverride({ id: `${id}-key`, streamKey: track.key });
55
57
  keyed.subscribe([{ source: signal, sourceSelector: track.kind === "video" ? selectVideo : selectAudio }]);
56
58
  tracks.push(keyed);
57
59
  nodes.push(keyed, signal);
58
60
  }
59
- return { tracks, nodes };
61
+ return { tracks, nodes, sent: { video: want("video"), audio: want("audio") } };
60
62
  }
61
63
  export async function runMediaConversionConformance(opts) {
62
64
  const { norsk, declaration, launcher, links } = opts;
@@ -64,12 +66,12 @@ export async function runMediaConversionConformance(opts) {
64
66
  const arriveTimeoutMs = opts.arriveTimeoutMs ?? 60_000;
65
67
  const observeMs = opts.observeMs ?? 10_000;
66
68
  const selectAll = (streams) => [...selectVideo(streams), ...selectAudio(streams)];
67
- const sent = new Set();
68
- const { tracks: sources, nodes: sourceNodes } = await startSources(norsk, prefix, declaration);
69
+ const sent = [];
70
+ const { tracks: sources, nodes: sourceNodes, sent: sentKinds, } = await startSources(norsk, prefix, declaration, opts.sources);
69
71
  const near = await norsk.inspect.streamTimestampReport({
70
72
  id: `${prefix}-sent`,
71
73
  onTimestamp: (_k, t) => {
72
- sent.add(stamp(t));
74
+ sent.push({ pts: seconds(t), at: Date.now() });
73
75
  },
74
76
  });
75
77
  near.subscribe(sources.map((source) => ({ source, sourceSelector: selectAll })));
@@ -83,11 +85,12 @@ export async function runMediaConversionConformance(opts) {
83
85
  const bindings = declaration.bindings;
84
86
  const returnName = `${prefix}-return`;
85
87
  const values = {
86
- ...opts.parameters,
88
+ ...(opts.parameters ?? {}),
87
89
  [bindings.sourceUrl]: published.sourceUrl,
88
90
  [bindings.returnName]: returnName,
89
91
  ...(bindings.engineVersion && opts.engineVersion ? { [bindings.engineVersion]: opts.engineVersion } : {}),
90
92
  ...(bindings.conversionId ? { [bindings.conversionId]: prefix } : {}),
93
+ [bindings.target]: JSON.stringify(opts.target),
91
94
  };
92
95
  // What is coming back right now, per stream; cleared at each phase.
93
96
  const kinds = new Map();
@@ -121,7 +124,7 @@ export async function runMediaConversionConformance(opts) {
121
124
  onTimestamp: (k, t) => {
122
125
  const key = keyOf(k);
123
126
  const list = stamps.get(key) ?? [];
124
- list.push(stamp(t));
127
+ list.push({ pts: seconds(t), at: Date.now() });
125
128
  stamps.set(key, list);
126
129
  },
127
130
  });
@@ -138,7 +141,7 @@ export async function runMediaConversionConformance(opts) {
138
141
  const streams = [...stamps.entries()].map(([key, list]) => ({
139
142
  kind: kinds.get(key) ?? "video",
140
143
  format: formats.get(key) ?? {},
141
- stamps: list,
144
+ frames: list,
142
145
  }));
143
146
  phases.push({ label, streams });
144
147
  };
@@ -158,7 +161,7 @@ export async function runMediaConversionConformance(opts) {
158
161
  const observation = { sent, phases };
159
162
  const checks = evaluateMediaConversion({
160
163
  declaration,
161
- parameters: values,
164
+ programme: { sent: sentKinds, ...(opts.expectFormats ? { formats: opts.expectFormats } : {}) },
162
165
  observation,
163
166
  minFrames: Math.max(1, Math.floor(observeMs / 1000)),
164
167
  });
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@norskvideo/ctl-test-harness",
3
- "version": "0.1.97",
3
+ "version": "0.1.99",
4
4
  "type": "module",
5
5
  "exports": {
6
6
  ".": {
@@ -96,9 +96,9 @@
96
96
  "devDependencies": {
97
97
  "@types/bun": "latest",
98
98
  "@types/node": "^22.10.0",
99
- "@norskvideo/norsk-studio": "1.30.0-2026-10-17-4468cc12",
100
- "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-17-4468cc12",
101
- "@norskvideo/norsk-sdk": "1.3.1-2026-10-16-e43e0f90"
99
+ "@norskvideo/norsk-studio": "1.30.0-2026-10-20-6fb182ff",
100
+ "@norskvideo/norsk-studio-built-ins": "1.30.0-2026-10-20-6fb182ff",
101
+ "@norskvideo/norsk-sdk": "1.3.1-2026-10-21-c97178c2"
102
102
  },
103
103
  "peerDependenciesMeta": {
104
104
  "@norskvideo/norsk-studio": {