@crvouga/atlas 1.11.0 → 1.13.0

package/packages/visualizer/src/evidence/behavior-panel.tsx

@@ -6,22 +6,25 @@
  * record: run, environment, source, attempts). Also the coverage summary of the whole feature. A recording is always
  * labelled as illustrative evidence with its own status, provenance and revision; it never stands for approval.
  */
+import { attachmentIdentity } from '@crvouga/atlas/reports/evidence-detail';
 import { effectiveScope } from '@crvouga/atlas/reports/evidence-scope';
 import type { EvidenceResult } from '@crvouga/atlas/reports/layer-merge';
 import { environmentLabel } from '@crvouga/atlas/reports/layer-axes';
 import { Alert, Badge, Box, Group, List, Paper, Stack, Tabs, Text, Title } from '@mantine/core';
 import { useMemo, type ReactNode } from 'react';
+import type { Attachment } from '@crvouga/atlas/schema';
 
 import type { Assessment, ReleaseIdentity } from './applicability';
 import { ApplicabilitySummary } from './applicability-view';
-import { attachmentSource, useCheckDetail } from './artifacts';
+import { attachmentSource, Timeline, useCheckDetail } from './artifacts';
 import { COVERAGE, COVERAGE_ORDER, type Behavior, type CoverageCounts, type CoverageState, type Revision } from './behavior';
 import { PlainSteps } from './check-detail';
 import { useEvidence } from './evidence-context';
-import { ScopeNotice } from './evidence-scope';
+import { ScopeNotice, ScopeProvider } from './evidence-scope';
 import { plural, statusWord, useNow } from './format';
 import { exampleIndexFor, exampleRows, fillExample, type FeatureStep } from './gherkin';
 import { Origin, StatusMark } from './marks';
+import { MomentPlayback, stepsOfRecording, useMomentVideo, useRequestedMoment } from './moments';
 import { outcomeHeadline, videosIn } from './outcome';
 
 /** The short, readable form of a spec fingerprint (`<length>:<hash>:<hash>`): its length and the start of its hash. */
@@ -80,14 +83,37 @@ function expectedSteps(steps: readonly FeatureStep[]): FeatureStep[] {
   return at < 0 ? [] : steps.slice(at);
 }
 
+/** The recording itself: the one player the steps beside it seek, which marks the step playing. */
+function WatchVideo({ attachment, url }: { attachment: Attachment; url: string }) {
+  const moments = useMomentVideo(attachment);
+  return (
+    <Box
+      component="video"
+      src={url}
+      controls
+      preload="metadata"
+      aria-label={`Recording: ${attachment.description ?? attachment.name}`}
+      maw="100%"
+      w="100%"
+      mah="70vh"
+      {...moments}
+    />
+  );
+}
+
 /** The recording of one result, read only when this is on screen. */
 function Player({ result }: { result: EvidenceResult }) {
   const detail = useCheckDetail(result.layer, result.row);
   const videos = useMemo(() => videosIn(detail.data?.check), [detail.data]);
-  const sources = videos.flatMap((attachment) => {
-    const { url } = attachmentSource(attachment, result.layer.id);
-    return url ? [{ attachment, url }] : [];
-  });
+  const requested = useRequestedMoment();
+  const sources = useMemo(
+    () =>
+      videos.flatMap((attachment) => {
+        const { url } = attachmentSource(attachment, result.layer.id);
+        return url ? [{ attachment, url, list: [attachment] }] : [];
+      }),
+    [videos, result.layer.id]
+  );
   if (detail.isLoading) {
     return (
       <Text role="status" component="div" c="dimmed">
@@ -109,27 +135,20 @@ function Player({ result }: { result: EvidenceResult }) {
       </Text>
     );
   }
+  const shown = sources.find(({ attachment }) => attachmentIdentity(attachment) === requested.clip) ?? sources[0]!;
+  const { attachment, url } = shown;
+  const check = detail.data!.check;
+  const steps = stepsOfRecording(check, attachmentIdentity(attachment));
   return (
     <Stack gap="sm">
-      {sources.slice(0, 1).map(({ attachment, url }) => (
-        <Stack key={url} gap="xs">
-          {/* What the recording does and does not show, as its producer declared it; undeclared says unknown. */}
-          <ScopeNotice
-            scope={effectiveScope(attachment.scope, detail.data?.check.scope, detail.data?.inheritedScope)}
-            name={attachment.name}
-          />
-          <Box
-            component="video"
-            src={url}
-            controls
-            preload="metadata"
-            aria-label={`Recording: ${attachment.description ?? attachment.name}`}
-            maw="100%"
-            w="100%"
-            mah="70vh"
-          />
-        </Stack>
-      ))}
+      {/* What the recording does and does not show, as its producer declared it; undeclared says unknown. */}
+      <ScopeProvider scope={effectiveScope(check.scope, detail.data?.inheritedScope)}>
+        <MomentPlayback steps={steps} attachments={shown.list} layer={result.layer.id}>
+          <ScopeNotice scope={effectiveScope(attachment.scope, check.scope, detail.data?.inheritedScope)} name={attachment.name} />
+          <WatchVideo attachment={attachment} url={url} />
+          {steps.length > 0 && <Timeline steps={steps} layer={result.layer.id} />}
+        </MomentPlayback>
+      </ScopeProvider>
       {sources.length > 1 && (
         <Text component="div" c="dimmed">
           {plural(sources.length - 1, 'more recording', 'more recordings')} in Inspect.

package/packages/visualizer/src/evidence/media-timing.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, test } from 'bun:test';
+
+import { activeIndex, alignStep, clock, mediaTimingOf, pausedMs, UNALIGNED } from './media-timing';
+
+const video = (media: unknown) => ({ extensions: { 'atlas.media': media } });
+
+describe('what a recording declares about its clock', () => {
+  test('no declaration is no timing, never a timing that starts with the check', () => {
+    expect(mediaTimingOf({})).toBeUndefined();
+    expect(mediaTimingOf({ extensions: { 'com.example': {} } })).toBeUndefined();
+    expect(alignStep({ id: 'a', offsetMs: 0 }, undefined, 5000)).toEqual({ state: 'unavailable', reason: UNALIGNED.noTiming });
+  });
+
+  test('a declaration that cannot be read as a whole places nothing', () => {
+    for (const bad of [
+      'text',
+      [],
+      { checkStartMs: '1000' },
+      { checkStartMs: 0, pauses: [{ atMs: 10 }] },
+      { checkStartMs: 0, pauses: 'long' },
+      { checkStartMs: 0, marks: [{ step: 'a' }] }
+    ]) {
+      const timing = mediaTimingOf(video(bad));
+      expect(timing?.problem).toBeDefined();
+      const aligned = alignStep({ id: 'a', offsetMs: 1 }, timing, undefined);
+      expect(aligned.state).toBe('unavailable');
+    }
+  });
+
+  test('a declaration without an origin places only the steps it marks', () => {
+    const timing = mediaTimingOf(video({ marks: [{ step: 'a', mediaMs: 700 }] }));
+    expect(alignStep({ id: 'a', offsetMs: 5 }, timing, undefined)).toEqual({ state: 'aligned', mediaMs: 700, source: 'mark', pausedMs: 0 });
+    expect(alignStep({ id: 'b', offsetMs: 5 }, timing, undefined)).toEqual({ state: 'unavailable', reason: UNALIGNED.noOrigin });
+  });
+});
+
+describe('placing a step in a recording', () => {
+  const timing = mediaTimingOf(video({ checkStartMs: 1200, pauses: [{ atMs: 2000, durationMs: 500 }, { atMs: 500, durationMs: 300, reason: 'title' }] }));
+
+  test('the recording starts before the check: the step is the origin plus its offset', () => {
+    expect(alignStep({ offsetMs: 0 }, timing, 20_000)).toMatchObject({ state: 'aligned', mediaMs: 1200, source: 'offset' });
+    expect(alignStep({ offsetMs: 100 }, timing, 20_000)).toMatchObject({ mediaMs: 1300 });
+  });
+
+  test('delays put into the recording move the steps after them and not the ones before', () => {
+    expect(alignStep({ offsetMs: 499 }, timing, undefined)).toMatchObject({ mediaMs: 1699, pausedMs: 0 });
+    expect(alignStep({ offsetMs: 500 }, timing, undefined)).toMatchObject({ mediaMs: 2000, pausedMs: 300 });
+    expect(alignStep({ offsetMs: 2000 }, timing, undefined)).toMatchObject({ mediaMs: 4000, pausedMs: 800 });
+    expect(pausedMs(timing)).toBe(800);
+    expect(timing!.pauses.map((pause) => pause.reason)).toEqual(['title', undefined]);
+  });
+
+  test('a mark states the moment outright and wins over the offset', () => {
+    const marked = mediaTimingOf(video({ checkStartMs: 0, marks: [{ step: 'a', mediaMs: 9 }] }));
+    expect(alignStep({ id: 'a', offsetMs: 5000 }, marked, undefined)).toMatchObject({ mediaMs: 9, source: 'mark' });
+  });
+
+  test('a step with no offset, before the start, or after the end is unavailable and is never clamped', () => {
+    expect(alignStep({ id: 'a' }, timing, undefined)).toEqual({ state: 'unavailable', reason: UNALIGNED.noOffset });
+    const early = mediaTimingOf(video({ checkStartMs: -500 }));
+    expect(alignStep({ offsetMs: 100 }, early, undefined)).toMatchObject({ state: 'unavailable', reason: expect.stringContaining('before the recording starts') });
+    expect(alignStep({ offsetMs: 100 }, early, undefined)).not.toHaveProperty('mediaMs');
+    expect(alignStep({ offsetMs: 9000 }, timing, 5000)).toMatchObject({ state: 'unavailable', reason: expect.stringContaining('after the recording ends') });
+    // A length that is not known never excludes a moment.
+    expect(alignStep({ offsetMs: 9000 }, timing, undefined).state).toBe('aligned');
+  });
+});
+
+describe('the step playing', () => {
+  const placed = [[0, 1000], [1, 3000], [2, 3000], [3, 6000]] as const;
+  test('is the latest step started, the innermost of several that started together, and none before the first', () => {
+    expect(activeIndex(placed, 500)).toBeNull();
+    expect(activeIndex(placed, 1000)).toBe(0);
+    expect(activeIndex(placed, 2999)).toBe(0);
+    expect(activeIndex(placed, 3000)).toBe(2);
+    expect(activeIndex(placed, 99_000)).toBe(3);
+    expect(activeIndex([], 10)).toBeNull();
+  });
+
+  test('a position reads as minutes, seconds and tenths', () => {
+    expect(clock(0)).toBe('0:00.0');
+    expect(clock(4250)).toBe('0:04.3');
+    expect(clock(61_000)).toBe('1:01.0');
+  });
+});

package/packages/visualizer/src/evidence/media-timing.ts

@@ -0,0 +1,150 @@
+/**
+ * @module Recording timing
+ *
+ * Where a check's steps are in its recording. A step says when it started on the check's own clock (`offsetMs`, after
+ * the check started); a recording has its own clock, which starts when the recorder did. The two are never assumed to
+ * agree. A producer that wants a step to be placed in a recording says how the clocks relate under the `atlas.media`
+ * extension of the video attachment:
+ *
+ * ```json
+ * { "name": "run.webm", "role": "video", "durationMs": 9000,
+ *   "extensions": { "atlas.media": {
+ *     "checkStartMs": 1200,                                  // the check started 1.2 s into the recording
+ *     "pauses": [{ "atMs": 900, "durationMs": 2000, "reason": "title card" }],  // delay put into the recording, not into the check
+ *     "marks": [{ "step": "pay", "mediaMs": 4100 }]          // a step's own moment, when the producer knows it exactly
+ *   } } }
+ * ```
+ *
+ * Every field is optional and nothing here decides a status, a slot or a winner. Without a mark or a `checkStartMs` a
+ * step is not placed: it is never guessed from the step's name, from the recording starting with the check, or from
+ * another recording's timing. A marker outside the recording, or timing that cannot be read, is reported as
+ * unavailable with the reason. Pure functions; the player is `moments.tsx`.
+ */
+import type { Attachment, Step } from '@crvouga/atlas/schema';
+
+/** The extension key under which a producer may relate a recording's clock to its check's. */
+export const MEDIA_EXTENSION = 'atlas.media';
+
+/** Time added to the recording and not to the check, such as a title card or a deliberate pause for a viewer. */
+export type MediaPause = {
+  /** Where on the check's clock the delay was inserted: steps that started at or after it are later in the recording by `durationMs`. */
+  readonly atMs: number;
+  readonly durationMs: number;
+  readonly reason: string | undefined;
+};
+
+/** What a recording says about its own clock. */
+export type MediaTiming = {
+  /** Where on the recording's clock the check started (its `offsetMs` 0). Undefined: not said, so steps are not placed by offset. */
+  readonly checkStartMs: number | undefined;
+  /** Delays inserted into the recording, in the order they occur. */
+  readonly pauses: readonly MediaPause[];
+  /** Steps whose moment in the recording is stated outright, by step `id`. */
+  readonly marks: ReadonlyMap<string, number>;
+  /** Set when the declaration could not be read as a whole; no step is then placed from it. */
+  readonly problem: string | undefined;
+};
+
+const finite = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value);
+
+/**
+ * The timing a video attachment declares under {@link MEDIA_EXTENSION}. Undefined when it declares none. A declaration
+ * with a field that cannot be read is kept with a `problem`, so it is reported instead of being half-applied.
+ */
+export function mediaTimingOf(attachment: Pick<Attachment, 'extensions'>): MediaTiming | undefined {
+  const raw = attachment.extensions?.[MEDIA_EXTENSION];
+  if (raw === undefined) return undefined;
+  const empty = { checkStartMs: undefined, pauses: [], marks: new Map<string, number>() };
+  if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) return { ...empty, problem: `The ${MEDIA_EXTENSION} declaration is not an object.` };
+  const { checkStartMs, pauses, marks } = raw as Record<string, unknown>;
+  let problem: string | undefined;
+  if (checkStartMs !== undefined && !finite(checkStartMs)) problem = '`checkStartMs` is not a number.';
+  const pauseList: MediaPause[] = [];
+  if (pauses !== undefined) {
+    if (!Array.isArray(pauses)) problem ??= '`pauses` is not a list.';
+    else
+      for (const pause of pauses as unknown[]) {
+        const { atMs, durationMs, reason } = (pause !== null && typeof pause === 'object' ? pause : {}) as Record<string, unknown>;
+        if (finite(atMs) && atMs >= 0 && finite(durationMs) && durationMs >= 0)
+          pauseList.push({ atMs, durationMs, reason: typeof reason === 'string' ? reason : undefined });
+        else problem ??= 'A pause needs a non-negative `atMs` and `durationMs`.';
+      }
+  }
+  const markMap = new Map<string, number>();
+  if (marks !== undefined) {
+    if (!Array.isArray(marks)) problem ??= '`marks` is not a list.';
+    else
+      for (const mark of marks as unknown[]) {
+        const { step, mediaMs } = (mark !== null && typeof mark === 'object' ? mark : {}) as Record<string, unknown>;
+        if (typeof step === 'string' && step !== '' && finite(mediaMs)) markMap.set(step, mediaMs);
+        else problem ??= 'A mark needs a step `id` and a `mediaMs`.';
+      }
+  }
+  pauseList.sort((a, b) => a.atMs - b.atMs);
+  return { checkStartMs: finite(checkStartMs) ? checkStartMs : undefined, pauses: pauseList, marks: markMap, problem };
+}
+
+/** The total length of the delays put into a recording. */
+export function pausedMs(timing: MediaTiming | undefined): number {
+  return (timing?.pauses ?? []).reduce((sum, pause) => sum + pause.durationMs, 0);
+}
+
+/** Where a step is in a recording, or why it is not placed. */
+export type Alignment =
+  | { readonly state: 'aligned'; readonly mediaMs: number; readonly source: 'mark' | 'offset'; readonly pausedMs: number }
+  | { readonly state: 'unavailable'; readonly reason: string };
+
+/** The reasons a step cannot be placed, in words. */
+export const UNALIGNED = {
+  noTiming:
+    'This recording carries no timing metadata, so its steps cannot be placed in it. Atlas does not assume the check started when the recording did, and does not guess from step names.',
+  noOrigin: 'This recording does not say where the check starts in it, so a step is not placed by its offset.',
+  noOffset: 'This step recorded no start offset and no recording mark, so it cannot be placed.'
+} as const;
+
+/**
+ * Where `step` is in a recording that declared `timing`, with `durationMs` the recording's length when known. A mark for
+ * the step wins; otherwise the recording's clock at the check's start, plus the step's offset, plus the delays inserted
+ * before it. A moment before the recording starts or after it ends is unavailable, never clamped.
+ */
+export function alignStep(step: Pick<Step, 'id' | 'offsetMs'>, timing: MediaTiming | undefined, durationMs: number | undefined): Alignment {
+  if (!timing) return { state: 'unavailable', reason: UNALIGNED.noTiming };
+  if (timing.problem) return { state: 'unavailable', reason: `The timing metadata of this recording cannot be used: ${timing.problem}` };
+  const marked = step.id === undefined ? undefined : timing.marks.get(step.id);
+  let mediaMs: number;
+  let source: 'mark' | 'offset';
+  let paused = 0;
+  if (marked !== undefined) {
+    mediaMs = marked;
+    source = 'mark';
+  } else {
+    if (timing.checkStartMs === undefined) return { state: 'unavailable', reason: UNALIGNED.noOrigin };
+    if (step.offsetMs === undefined) return { state: 'unavailable', reason: UNALIGNED.noOffset };
+    paused = timing.pauses.filter((pause) => pause.atMs <= step.offsetMs!).reduce((sum, pause) => sum + pause.durationMs, 0);
+    mediaMs = timing.checkStartMs + step.offsetMs + paused;
+    source = 'offset';
+  }
+  if (mediaMs < 0)
+    return { state: 'unavailable', reason: `This step is placed ${clock(-mediaMs)} before the recording starts, so it is not in the recording.` };
+  if (durationMs !== undefined && durationMs > 0 && mediaMs > durationMs)
+    return { state: 'unavailable', reason: `This step is placed at ${clock(mediaMs)}, after the recording ends at ${clock(durationMs)}.` };
+  return { state: 'aligned', mediaMs, source, pausedMs: paused };
+}
+
+/**
+ * The index of the step playing at `timeMs`: the latest placed step that started at or before it (the last of them when
+ * several started together, which is the innermost). Null before the first placed step. `placed` are `[index, mediaMs]`.
+ */
+export function activeIndex(placed: readonly (readonly [number, number])[], timeMs: number): number | null {
+  let found: readonly [number, number] | null = null;
+  for (const entry of placed) if (entry[1] <= timeMs && (found === null || entry[1] >= found[1])) found = entry;
+  return found === null ? null : found[0];
+}
+
+/** A position in a recording as `m:ss.s`: `0:04.2`. */
+export function clock(ms: number): string {
+  const tenths = Math.round(Math.max(0, ms) / 100);
+  const minutes = Math.floor(tenths / 600);
+  const seconds = (tenths % 600) / 10;
+  return `${minutes}:${seconds < 10 ? '0' : ''}${seconds.toFixed(1)}`;
+}