@raydiant/playlist-window-manager 0.29.1 → 0.30.0

package/src/PlaylistWindowManager.ts

@@ -19,6 +19,11 @@ import PlaylistWindowManagerState from './PlaylistWindowManagerState';
 import WindowUnresponsiveError from './WindowUnresponsiveError';
 import WindowTimeoutError from './WindowTimeoutError';
 import WindowLoadError from './WindowLoadError';
+import SurfaceBroker, {
+  DEFAULT_SHOW_AFTER_REVEAL_MS,
+  DEFAULT_SURFACE_SWAP_DELAY_MS,
+} from './SurfaceBroker';
+import { SurfaceHost } from './surfaces';
 
 export interface PlaylistWindowManagerOptions {
   // The rule engine
@@ -83,6 +88,12 @@ export interface PlaylistWindowManagerOptions {
 
   // Called when a window needs to check if the device is online.
   getOnlineStatus?: () => boolean;
+
+  // The host's video surface, bound by the player, when the host can draw
+  // one. Absent, nothing changes: apps are offered no surface and keep
+  // <video>. A zone's manager is never given one. Its showAfterRevealMs is
+  // the device's calibration for a swap's reveal; see SurfaceHost.
+  surfaces?: SurfaceHost;
 }
 
 // Wait at least 3 seconds before restarting or moving onto the next window
@@ -118,6 +129,10 @@ export default class PlaylistWindowManager extends EventEmitter {
   private isFocused: boolean = false;
 
   private workerState: WorkerState = {};
+  private surfaceBroker: SurfaceBroker | null = null;
+  // The surface swap the next window's preload waits to finish. A newer
+  // preload or the next window's removal clears it, cancelling that preload.
+  private pendingPreload: Promise<void> | null = null;
 
   constructor(private options: PlaylistWindowManagerOptions) {
     super();
@@ -127,6 +142,9 @@ export default class PlaylistWindowManager extends EventEmitter {
     this.bounds = options.bounds;
     this.themeOverride = options.themeOverride;
     this.forceMute = options.forceMute || false;
+    if (options.surfaces) {
+      this.surfaceBroker = new SurfaceBroker(options.surfaces, options.logger);
+    }
   }
 
   private currentPresentationId(): string | null {
@@ -357,6 +375,10 @@ export default class PlaylistWindowManager extends EventEmitter {
       clearTimeout(this.currentPresentationCompleteTimeout);
     }
     await Promise.all([this.removeCurrentWindow(), this.removeNextWindow()]);
+    if (this.surfaceBroker) {
+      // Also closes a surface whose window is still leaving a swap.
+      await this.surfaceBroker.closeAll();
+    }
   }
 
   public updateBounds(bounds: WindowBounds) {
@@ -571,9 +593,29 @@ export default class PlaylistWindowManager extends EventEmitter {
       workerState: this.workerState,
       getTerminal: this.options.getTerminal,
       getOnlineStatus: this.options.getOnlineStatus,
+      ...(this.surfaceBroker
+        ? {
+            surfaceBroker: this.surfaceBroker,
+            surfaceShowDeadlineMs: this.surfaceSwapDelayMs(),
+          }
+        : {}),
     });
   }
 
+  private showAfterRevealMs() {
+    const { surfaces } = this.options;
+    return surfaces && surfaces.showAfterRevealMs !== undefined
+      ? surfaces.showAfterRevealMs
+      : DEFAULT_SHOW_AFTER_REVEAL_MS;
+  }
+
+  private surfaceSwapDelayMs() {
+    const { closeWindowDelayMs } = this.options;
+    return closeWindowDelayMs !== undefined
+      ? closeWindowDelayMs
+      : DEFAULT_SURFACE_SWAP_DELAY_MS;
+  }
+
   private async loadCurrentWindow() {
     const { logger } = this.options;
 
@@ -620,6 +662,10 @@ export default class PlaylistWindowManager extends EventEmitter {
   private async preloadNextWindow() {
     const { logger } = this.options;
 
+    // A publish, a preview change or a complete preloads now, in place of a
+    // preload still waiting for its surface swap.
+    this.pendingPreload = null;
+
     if (!this.nextPresentation) {
       logger.warn(
         `Attempted to preload next window but nextPresentation not set`
@@ -699,6 +745,7 @@ export default class PlaylistWindowManager extends EventEmitter {
   }
 
   private async removeNextWindow() {
+    this.pendingPreload = null;
     if (this.nextWindow) {
       await this.nextWindow.remove();
     }
@@ -742,33 +789,62 @@ export default class PlaylistWindowManager extends EventEmitter {
     // Clear the pending interrupt whenever we swap the current window for the next one.
     this.hasPendingInterrupt = false;
 
-    // Next window is ready, show the next one, hide the current window, and
-    // then remove the current window. Show the next window before hiding the
-    // current to avoid flashes when rendering with BrowserWindows on device.
-    if (this.nextWindow) {
+    // Only a manager given a surface host, swapping a window that holds a
+    // surface, takes the surface path; every other swap runs today's code.
+    const isSurfaceSwap =
+      !!this.surfaceBroker &&
+      ((!!this.currentWindow && this.currentWindow.holdsSurface()) ||
+        (!!this.nextWindow && this.nextWindow.holdsSurface()));
+
+    let closeWindowPromise;
+    if (isSurfaceSwap) {
+      closeWindowPromise = this.swapWithSurfaces(
+        this.currentWindow,
+        this.nextWindow
+      );
+    } else if (this.nextWindow) {
+      // Next window is ready, show the next one, hide the current window, and
+      // then remove the current window. Show the next window before hiding the
+      // current to avoid flashes when rendering with BrowserWindows on device.
       this.nextWindow.show(this.isFocused);
     }
 
     // If current window isn't defined we don't need to hide it but we still
     // want to make the next window the new current window.
-    let closeWindowPromise;
-    if (this.currentWindow) {
+    if (!isSurfaceSwap && this.currentWindow) {
       if (this.options.closeWindowDelayMs !== undefined) {
+        // Keep the outgoing window on screen until the incoming one reports
+        // its first painted frame. A newly shown window still needs a few
+        // frames to rasterize, and removing the outgoing window before that
+        // exposes the transparent window stack (a white flash on device).
+        // closeWindowDelayMs is the ceiling for windows that never report a
+        // paint (embedded apps, error states).
         // Don't await here because this will cause a slight delay when playing the next window.
         // Instead we want to wait for window close at the end of this method.
-        closeWindowPromise = new Promise((resolve) => {
-          // Capture the current window reference to close because this.currentWindow will be
-          // set to the next window before the timeout fires.
-          const windowToClose = this.currentWindow;
-          setTimeout(async () => {
-            if (windowToClose) {
-              // We can't use this.removeCurrentWindow because the currentWindow reference has already
-              // changed by the time the timeout fires.
-              await windowToClose.remove();
-            }
-            resolve(undefined);
-          }, this.options.closeWindowDelayMs);
-        });
+        // Capture both window references because this.currentWindow and
+        // this.nextWindow change before the wait finishes.
+        const windowToClose = this.currentWindow;
+        const incomingWindow = this.nextWindow;
+        const closeWindowDelayMs = this.options.closeWindowDelayMs;
+
+        // The outgoing window stays on screen only as a visual backdrop from
+        // here on: stop delivering its events (a late complete or error must
+        // not end the incoming presentation) and mute it so audio does not
+        // bleed into the incoming presentation during the overlap.
+        windowToClose.detachCallbacks();
+        windowToClose.mute();
+        closeWindowPromise = (async () => {
+          if (incomingWindow) {
+            await incomingWindow.waitForFirstPaint(closeWindowDelayMs);
+          } else {
+            await new Promise((resolve) =>
+              setTimeout(resolve, closeWindowDelayMs)
+            );
+          }
+          // We can't use this.removeCurrentWindow because the currentWindow
+          // reference has already changed by the time the wait finishes.
+          await windowToClose.remove();
+        })();
       } else {
         this.currentWindow.hide();
         await this.removeCurrentWindow();
@@ -797,7 +873,16 @@ export default class PlaylistWindowManager extends EventEmitter {
     this.playCurrentWindow();
 
     // Preload the next presentation if there is one.
-    if (shouldPreload) {
+    let preloadAfterSwap: Promise<void> | null = null;
+    if (shouldPreload && isSurfaceSwap) {
+      // Once the swap has finished, so the next surface's decoder does not
+      // start during it: on the Stick a preload started in the swap held the
+      // outgoing clip's last frame for about 70 ms. The next window is the
+      // current one now.
+      this.nextWindow = null;
+      preloadAfterSwap = closeWindowPromise as Promise<void>;
+      this.pendingPreload = preloadAfterSwap;
+    } else if (shouldPreload) {
       this.preloadNextWindow();
     } else {
       // Set next window to null to reset preload state. We don't want to
@@ -809,6 +894,58 @@ export default class PlaylistWindowManager extends EventEmitter {
     if (closeWindowPromise) {
       await closeWindowPromise;
     }
+
+    if (preloadAfterSwap && this.pendingPreload === preloadAfterSwap) {
+      this.preloadNextWindow();
+    }
+  }
+
+  // The swap when either window holds a video surface. The outgoing surface
+  // is muted and marked leaving here, in the swap's own task; the rest runs
+  // on the two windows held here, so a publish, a preview check or a preload
+  // error during its waits acts on the new next window, never these two.
+  private swapWithSurfaces(
+    outgoing: PresentationWindow | null,
+    incoming: PresentationWindow | null
+  ): Promise<void> {
+    const delayMs = this.surfaceSwapDelayMs();
+
+    if (outgoing) {
+      // A late complete or error from the outgoing window must not end the
+      // incoming presentation, and two surfaces never sound at once.
+      outgoing.detachCallbacks();
+      outgoing.muteSurfaceOrFrame();
+    }
+
+    return (async () => {
+      let isOutgoingHidden = false;
+      if (incoming) {
+        // The new is shown before the old is hidden: an incoming surface's
+        // frame is revealed over its open hole and the outgoing frame hidden
+        // in that same task, so neither the outgoing paint-over nor its theme
+        // shows through the incoming hole.
+        await incoming.show(
+          this.isFocused,
+          () => {
+            if (outgoing) {
+              outgoing.hide();
+              isOutgoingHidden = true;
+            }
+          },
+          this.showAfterRevealMs()
+        );
+        // A removed incoming window never paints.
+        if (outgoing && !incoming.wasRemoved()) {
+          await incoming.waitForFirstPaint(delayMs);
+        }
+      }
+      if (outgoing) {
+        if (!isOutgoingHidden) {
+          outgoing.hide();
+        }
+        await outgoing.remove();
+      }
+    })();
   }
 
   private evalRuleEngine(warnIfPreloadMismatch = true): EvalRuleEngineResult {
@@ -922,6 +1059,13 @@ export default class PlaylistWindowManager extends EventEmitter {
     // Clear current window error flag.
     this.currentWindowError = null;
 
+    // A complete while the next window's preload waits for its surface swap
+    // preloads it now, and the current window restarts while it loads, as
+    // for any next window not yet ready.
+    if (this.pendingPreload && this.nextPresentation) {
+      this.preloadNextWindow();
+    }
+
     // If not locked or paused, check if we can move to the next presentation.
     if (
       !this.isLocked() &&

package/src/PresentationWindow.test.ts

@@ -5,7 +5,12 @@ import {
   createRenderer,
   createPlaylistWindowManagerState,
   createRuleEngineState,
+  createFakeSurfaceHost,
+  createLogger,
 } from './testUtils';
+import * as re from '@raydiant/playlist-rule-engine';
+import SurfaceBroker from './SurfaceBroker';
+import { ApplicationVersion } from './types';
 
 const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
@@ -454,3 +459,369 @@ describe('interrupt locks', () => {
 });
 
 // TODO: Backfill tests for all legacy and embedded apps.
+
+describe('waitForFirstPaint', () => {
+  // The default (non-embedded) window resolves load() only after the app
+  // reports ready, so drive the handshake through the mock renderer.
+  const loadPresentationWindow = async (
+    testRenderer: ReturnType<typeof createRenderer>,
+    window: ReturnType<typeof createPresentationWindow>
+  ) => {
+    const loadPromise = window.load();
+    await delay(0);
+    testRenderer.runReceiveMessageCallback('loaded', undefined);
+    testRenderer.runReceiveMessageCallback('presentation_ready', undefined);
+    await loadPromise;
+  };
+
+  test('resolves when the window reports first_painted', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-first-paint',
+      renderer: testRenderer,
+    });
+    await loadPresentationWindow(testRenderer, window);
+
+    let resolved = false;
+    const wait = window.waitForFirstPaint(5000).then(() => {
+      resolved = true;
+    });
+
+    await delay(10);
+    expect(resolved).toBe(false);
+
+    testRenderer.runReceiveMessageCallback('first_painted', undefined);
+    await wait;
+    expect(resolved).toBe(true);
+  });
+
+  test('falls back to the timeout when no paint signal arrives', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-first-paint-timeout',
+      renderer: testRenderer,
+    });
+    await loadPresentationWindow(testRenderer, window);
+
+    // Resolves via the timeout even though first_painted never arrives.
+    await window.waitForFirstPaint(20);
+  });
+
+  test('resolves immediately when the window already painted since the last play', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-first-paint-immediate',
+      renderer: testRenderer,
+    });
+    await loadPresentationWindow(testRenderer, window);
+
+    testRenderer.runReceiveMessageCallback('first_painted', undefined);
+
+    let resolved = false;
+    window.waitForFirstPaint(5000).then(() => {
+      resolved = true;
+    });
+    await Promise.resolve();
+    expect(resolved).toBe(true);
+  });
+
+  test('play starts a new paint cycle', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-first-paint-replay',
+      renderer: testRenderer,
+    });
+    await loadPresentationWindow(testRenderer, window);
+
+    testRenderer.runReceiveMessageCallback('first_painted', undefined);
+    window.play();
+
+    let resolved = false;
+    const wait = window.waitForFirstPaint(5000).then(() => {
+      resolved = true;
+    });
+    await delay(10);
+    expect(resolved).toBe(false);
+
+    testRenderer.runReceiveMessageCallback('first_painted', undefined);
+    await wait;
+    expect(resolved).toBe(true);
+  });
+
+  test('resolves pending waiters when the window is removed', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-first-paint-removed',
+      renderer: testRenderer,
+    });
+    await loadPresentationWindow(testRenderer, window);
+
+    const wait = window.waitForFirstPaint(5000);
+    await window.remove();
+    await wait;
+  });
+});
+
+describe('detachCallbacks', () => {
+  test('stops delivering events to the window manager', async () => {
+    const testRenderer = createRenderer();
+    const window = createPresentationWindow({
+      windowId: 'test-detach',
+      renderer: testRenderer,
+    });
+    const loadPromise = window.load();
+    await delay(0);
+    testRenderer.runReceiveMessageCallback('loaded', undefined);
+    testRenderer.runReceiveMessageCallback('presentation_ready', undefined);
+    await loadPromise;
+
+    const onComplete = jest.fn();
+    window.onComplete(onComplete);
+    window.detachCallbacks();
+
+    testRenderer.runReceiveMessageCallback('presentation_complete', undefined);
+    expect(onComplete).not.toHaveBeenCalled();
+  });
+});
+
+describe('video surfaces', () => {
+  const CONTENT_URL = 'https://cdn.test/clip.mp4';
+
+  function setupVideoWindow(
+    opts: {
+      withHost?: boolean;
+      applicationVariables?: Record<string, unknown>;
+      applicationVersion?: Partial<ApplicationVersion>;
+    } = {}
+  ) {
+    const testRenderer = createRenderer();
+    const host = createFakeSurfaceHost();
+    const warn = jest.fn();
+    const logger = createLogger({ warn });
+    const window = createPresentationWindow({
+      windowId: 'video-window',
+      presentation: createPresentation({
+        id: 'p1',
+        applicationVariables: opts.applicationVariables ?? {
+          content: { url: CONTENT_URL, filename: 'clip.mp4' },
+        },
+      }),
+      applicationVersion: createApplicationVersion(opts.applicationVersion),
+      renderer: testRenderer,
+      logger,
+      surfaceBroker:
+        opts.withHost === false
+          ? undefined
+          : new SurfaceBroker(host.surfaces, logger),
+    });
+
+    const messages = () =>
+      testRenderer.sendMessageString.mock.calls.map(([, message]) =>
+        JSON.parse(message)
+      );
+    const refusals = () =>
+      messages()
+        .filter((m) => m.type === 'video_refused')
+        .map((m) => m.payload.reason);
+    const refusalsLogged = () =>
+      warn.mock.calls.filter(([message]) => /video_refused/.test(message))
+        .length;
+    const openVideo = (url: string) =>
+      testRenderer.runReceiveMessageCallback('open_video', {
+        url,
+        bounds: { x: 0.4, y: 10.6, width: 1919.6, height: 1080 },
+        muted: false,
+      });
+
+    return {
+      testRenderer,
+      host,
+      window,
+      messages,
+      refusals,
+      refusalsLogged,
+      openVideo,
+    };
+  }
+
+  test('open_video reaches the host, and render carries nativeVideo, only with a host', async () => {
+    const offered = setupVideoWindow();
+    offered.window.load();
+    await delay(0);
+    offered.testRenderer.runReceiveMessageCallback('loaded', undefined);
+    expect(
+      offered.messages().find((m) => m.type === 'render').payload.nativeVideo
+    ).toBe(true);
+
+    offered.openVideo(CONTENT_URL);
+    await delay(0);
+    expect(
+      offered.host.calls.filter((c) => c.method === 'openVideo')[0].args[0]
+    ).toEqual({
+      url: CONTENT_URL,
+      bounds: { x: 0, y: 11, width: 1920, height: 1080 },
+      muted: true,
+      label: 'p1',
+    });
+    expect(offered.messages().map((m) => m.type)).toContain('video_opened');
+    await offered.window.remove();
+
+    const unoffered = setupVideoWindow({ withHost: false });
+    unoffered.window.load();
+    await delay(0);
+    unoffered.testRenderer.runReceiveMessageCallback('loaded', undefined);
+    expect(
+      unoffered.messages().find((m) => m.type === 'render').payload
+    ).not.toHaveProperty('nativeVideo');
+
+    unoffered.openVideo(CONTENT_URL);
+    expect(unoffered.refusals()).toEqual(['not_offered']);
+    expect(unoffered.refusalsLogged()).toBe(1);
+    await unoffered.window.remove();
+  });
+
+  test('a rendition beside the presentation url opens a surface', async () => {
+    const { window, host, openVideo, refusals } = setupVideoWindow();
+    window.load();
+    await delay(0);
+
+    openVideo(`${CONTENT_URL}_720p`);
+    await delay(0);
+
+    expect(host.calls.filter((c) => c.method === 'openVideo')).toHaveLength(1);
+    expect(refusals()).toEqual([]);
+    await window.remove();
+  });
+
+  test.each([
+    ['a url the presentation does not hold', 'https://cdn.test/other.mp4'],
+    ['a url beside it with another suffix', `${CONTENT_URL}_1080p`],
+  ])('%s is refused not_offered', async (_, url) => {
+    const { window, host, openVideo, refusals, refusalsLogged } =
+      setupVideoWindow();
+    window.load();
+    await delay(0);
+
+    openVideo(url);
+    await delay(0);
+
+    expect(refusals()).toEqual(['not_offered']);
+    expect(refusalsLogged()).toBe(1);
+    expect(host.calls).toEqual([]);
+    await window.remove();
+  });
+
+  test('a url that is not http(s) is refused not_offered', async () => {
+    const { window, host, openVideo, refusals } = setupVideoWindow({
+      applicationVariables: { content: { url: 'blob:https://cdn.test/1' } },
+    });
+    window.load();
+    await delay(0);
+
+    openVideo('blob:https://cdn.test/1');
+    await delay(0);
+
+    expect(refusals()).toEqual(['not_offered']);
+    expect(host.calls).toEqual([]);
+    await window.remove();
+  });
+
+  test('an embedded app is refused not_offered', async () => {
+    const { window, host, testRenderer, refusals, refusalsLogged } =
+      setupVideoWindow({
+        applicationVersion: { embeddedUrlFormat: 'https://embedded.test/' },
+      });
+    await window.load();
+
+    testRenderer.runReceiveMessageCallback('open_video', {
+      url: CONTENT_URL,
+      bounds: { x: 0, y: 0, width: 1920, height: 1080 },
+    });
+    await delay(0);
+
+    expect(refusals()).toEqual(['not_offered']);
+    expect(refusalsLogged()).toBe(1);
+    expect(host.calls).toEqual([]);
+    await window.remove();
+  });
+
+  test("a zone's windows are offered no surface and refused not_offered", async () => {
+    const testRenderer = createRenderer();
+    const host = createFakeSurfaceHost();
+    const logger = createLogger();
+    const zonePresentation = createPresentation({
+      id: 'z1',
+      applicationDeploymentId: 'zone-app',
+      applicationVariables: { content: { url: CONTENT_URL } },
+    });
+    const window = createPresentationWindow({
+      windowId: 'multizone',
+      presentation: createPresentation({
+        id: 'mz',
+        r: {
+          resource: { updatedAt: 'updatedAt', createdAt: 'createdAt' },
+          presentationsPlaylists: [{ playlistId: 'zone-playlist' }],
+        },
+      }),
+      playlistWindowManagerState: createPlaylistWindowManagerState({
+        presentations: { z1: zonePresentation },
+        applicationVersions: {
+          'zone-app': createApplicationVersion({ id: 'zone-app' }),
+        },
+      }),
+      ruleEngineState: createRuleEngineState(
+        {
+          'zone-playlist': {
+            id: 'zone-playlist',
+            mode: re.PlaylistMode.Sequential,
+            items: [{ presentationId: 'z1' }],
+          },
+        },
+        { z1: { id: 'z1', applicationName: 'videos' } }
+      ),
+      renderer: testRenderer,
+      logger,
+      surfaceBroker: new SurfaceBroker(host.surfaces, logger),
+    });
+    window.load();
+    await delay(0);
+
+    testRenderer.runReceiveMessageCallback('open_child_window', {
+      windowId: 'zone1',
+      playlistId: 'zone-playlist',
+      x: 0,
+      y: 0,
+      width: 960,
+      height: 1080,
+      isMuted: false,
+      isFocused: false,
+    });
+    await delay(0);
+
+    // The renderer's callbacks now reach the zone's window.
+    const zoneWindowId = testRenderer.createWindow.mock.calls[1][0].windowId;
+    testRenderer.runReceiveMessageCallback('loaded', undefined);
+    const zoneMessages = () =>
+      testRenderer.sendMessageString.mock.calls
+        .filter(([windowId]) => windowId === zoneWindowId)
+        .map(([, message]) => JSON.parse(message));
+    expect(
+      zoneMessages().find((m) => m.type === 'render').payload
+    ).not.toHaveProperty('nativeVideo');
+
+    testRenderer.runReceiveMessageCallback('open_video', {
+      url: CONTENT_URL,
+      bounds: { x: 0, y: 0, width: 960, height: 1080 },
+      muted: false,
+    });
+    await delay(0);
+
+    expect(
+      zoneMessages()
+        .filter((m) => m.type === 'video_refused')
+        .map((m) => m.payload.reason)
+    ).toEqual(['not_offered']);
+    expect(host.calls).toEqual([]);
+    await window.remove();
+  });
+});