@lookit/record 4.0.0 → 5.0.0

package/src/recorder.ts

@@ -11,6 +11,7 @@ import play_icon from "../img/play-icon.svg";
 import record_icon from "../img/record-icon.svg";
 import {
   CreateURLError,
+  NoFileNameError,
   NoStopPromiseError,
   NoWebCamElementError,
   RecorderInitializeError,
@@ -18,8 +19,10 @@ import {
   StreamActiveOnResetError,
   StreamDataInitializeError,
   StreamInactiveInitializeError,
+  TimeoutError,
 } from "./errors";
-import { CSSWidthHeight } from "./types";
+import { CSSWidthHeight, StopOptions, StopResult } from "./types";
+import { promiseWithTimeout } from "./utils";
 
 declare const window: LookitWindow;
 
@@ -32,10 +35,11 @@ export default class Recorder {
   private localDownload: boolean =
     process.env.LOCAL_DOWNLOAD?.toLowerCase() === "true";
   private filename?: string;
-  private stopPromise?: Promise<void>;
+  private stopPromise?: Promise<string>;
   private webcam_element_id = "lookit-jspsych-webcam";
   private mimeType = "video/webm";
 
+  // persistent clone of the original stream so we can re-initialize
   private streamClone: MediaStream;
 
   /**
@@ -109,13 +113,23 @@ export default class Recorder {
     this.jsPsych.pluginAPI.initializeCameraRecorder(stream, recorder_options);
   }
 
-  /** Reset the recorder to be used again. */
+  /**
+   * Reset the recorder. This is used internally after stopping/uploading a
+   * recording, in order to create a new active stream that can be used by a new
+   * Recorder instance. This can also be used by the consuming plugin/extension
+   * when a recorder needs to be reset without the stop/upload events (e.g. in
+   * the video config plugin).
+   */
   public reset() {
+    // Reset can only be called after the current stream has been fully stopped.
     if (this.stream.active) {
       throw new StreamActiveOnResetError();
     }
+    // Ensure later recordings have a valid active stream.
     this.initializeRecorder(this.streamClone.clone());
+    // Clear blob buffer (any pending uploads are handled by LookitS3 instances and tracked globally)
     this.blobs = [];
+    // TO DO: reset S3/filename/URL?
   }
 
   /**
@@ -249,7 +263,7 @@ export default class Recorder {
 
     // create a stop promise and pass the resolve function as an argument to the stop event callback,
     // so that the stop event handler can resolve the stop promise
-    this.stopPromise = new Promise((resolve) => {
+    this.stopPromise = new Promise<string>((resolve) => {
       this.recorder.addEventListener("stop", this.handleStop(resolve));
     });
 
@@ -276,24 +290,122 @@ export default class Recorder {
    * tracks, clear the webcam feed element (if there is one), and return the
    * stop promise. This should only be called after recording has started.
    *
-   * @param maintain_container_size - Optional boolean indicating whether or not
-   *   to maintain the current size of the webcam feed container when removing
-   *   the video element. Default is false. If true, the container will be
-   *   resized to match the dimensions of the video element before it is
+   * When calling recorder.stop, plugins may:
+   *
+   * - Await the 'stopped' promise
+   * - Await the 'uploaded' promise to wait for the upload to finish before
+   *   continuing
+   * - Not await either promise to continue immediately and regardless of the
+   *   stop/upload outcomes
+   *
+   * @param options - Object with the following:
+   * @param options.maintain_container_size - Optional boolean indicating
+   *   whether or not to maintain the current size of the webcam feed container
+   *   when removing the video element. Default is false. If true, the container
+   *   will be resized to match the dimensions of the video element before it is
    *   removed. This is useful for avoiding layout jumps when the webcam
    *   container will be re-used during the trial.
-   * @returns Promise that resolves after the media recorder has stopped and
-   *   final 'dataavailable' event has occurred, when the "stop" event-related
-   *   callback function is called.
+   * @param options.stop_timeout_ms - Number of seconds to wait for the stop
+   *   process to complete.
+   * @param options.upload_timeout_ms - Number of seconds to wait for the upload
+   *   process to complete.
+   * @returns Object with two promises:
+   *
+   *   - Stopped: Promise<void> - Promise that resolves after the media recorder has
+   *       stopped and final 'dataavailable' event has occurred, when the "stop"
+   *       event-related callback function is called.
+   *   - Uploaded: Promise<void> - Promise that resolves when the S3 upload
+   *       completes.
    */
-  public stop(maintain_container_size: boolean = false) {
+  public stop({
+    maintain_container_size = false,
+    stop_timeout_ms = null,
+    upload_timeout_ms = 10000,
+  }: StopOptions = {}): StopResult {
+    this.preStopCheck();
     this.clearWebcamFeed(maintain_container_size);
     this.stopTracks();
 
-    if (!this.stopPromise) {
-      throw new NoStopPromiseError();
-    }
-    return this.stopPromise;
+    // Snapshot anything needed for upload before the Recorder instance is reset.
+    // URL is placeholder because it will not be defined until after the stop promise is resolved.
+    const snapshot = {
+      s3: !this.localDownload ? this.s3 : null,
+      filename: this.filename,
+      localDownload: this.localDownload,
+      url: "null",
+    };
+
+    // Wrap the existing stopPromise with timeout if needed, otherwise return as is.
+    const stopped: Promise<string> = stop_timeout_ms
+      ? promiseWithTimeout(
+          this.stopPromise!,
+          `${snapshot.filename}-stopped`,
+          stop_timeout_ms,
+          this.createTimeoutHandler("stop", snapshot.filename!),
+        )
+      : this.stopPromise!;
+
+    // Chain reset off the stop promise, which is either the original stop promise or a promise race with the timeout.
+    stopped.finally(() => {
+      try {
+        // It's safe to reset because recording is fully stopped and S3 info has been snapshotted.
+        this.reset();
+      } catch (err) {
+        console.error("Error while resetting recorder after stop: ", err);
+      }
+    });
+
+    // Create the upload (or local download) promise
+    const uploadPromise: Promise<void> = (async () => {
+      let url: string;
+      try {
+        url = await stopped;
+        if (url == "timeout") {
+          // Stop failed, throw so that the upload promise reflects this failure
+          throw new TimeoutError("Recorder stop timed out.");
+        }
+      } catch (err) {
+        console.warn("Upload failed because recorder stop timed out");
+        throw err;
+      }
+      snapshot.url = url;
+      if (snapshot.localDownload) {
+        try {
+          this.download(snapshot.filename!, snapshot.url);
+          await Promise.resolve();
+        } catch (err) {
+          console.error("Local download failed: ", err);
+          throw err;
+        }
+      } else {
+        try {
+          await snapshot.s3!.completeUpload();
+        } catch (err) {
+          console.error("Upload failed: ", err);
+          throw err;
+        }
+      }
+    })();
+
+    // Wrap the upload promise in a timeout if needed, otherwise return as is.
+    const uploaded: Promise<void | string> = upload_timeout_ms
+      ? promiseWithTimeout(
+          uploadPromise,
+          `${snapshot.filename}-uploaded`,
+          upload_timeout_ms,
+          this.createTimeoutHandler("upload", snapshot.filename!),
+        )
+      : uploadPromise;
+
+    // Track background uploads in case the consuming plugin is not awaiting the upload.
+    // We don't want the timeout version because this one can continue in the background.
+    window.chs.pendingUploads.push({
+      promise: uploadPromise,
+      file: snapshot.filename!,
+    });
+
+    // Return the pair of promises so that the calling plugin can await them.
+    return { stopped, uploaded };
   }
 
   /** Throw Error if there isn't a recorder provided by jsPsych. */
@@ -311,29 +423,42 @@ export default class Recorder {
     }
   }
 
+  /**
+   * Check for necessary conditions before stop/upload process, and throw errors
+   * if needed.
+   */
+  private preStopCheck() {
+    if (!this.recorder) {
+      throw new RecorderInitializeError();
+    }
+    if (!this.stream.active) {
+      throw new StreamInactiveInitializeError();
+    }
+    if (!this.stopPromise) {
+      throw new NoStopPromiseError();
+    }
+    if (!this.filename) {
+      throw new NoFileNameError();
+    }
+  }
+
   /**
    * Handle the recorder's stop event. This is a function that takes the stop
    * promise's 'resolve' as an argument and returns a function that resolves
-   * that stop promise. The function that is returned is used as the recorder's
-   * "stop" event-related callback function.
+   * that stop promise with the URL that is created from the recording.
    *
-   * @param resolve - Promise resolve function.
-   * @returns Function that is called on the recorder's "stop" event.
+   * @param resolve - Resolve function that resolves the stop promise that was
+   *   created upon the start of recording.
+   * @returns Function that is called on the recorder's "stop" event. This
+   *   function resolves the stop promise for this recording with a URL.
    */
-  private handleStop(resolve: () => void) {
-    return async () => {
+  private handleStop(resolve: (value: string | PromiseLike<string>) => void) {
+    return () => {
       if (this.blobs.length === 0) {
         throw new CreateURLError();
       }
       this.url = URL.createObjectURL(new Blob(this.blobs));
-
-      if (this.localDownload) {
-        this.download();
-      } else {
-        await this.s3.completeUpload();
-      }
-
-      resolve();
+      resolve(this.url);
     };
   }
 
@@ -343,18 +468,27 @@ export default class Recorder {
    * @param event - Event containing blob data.
    */
   private handleDataAvailable(event: BlobEvent) {
+    // Store locally for URL creation
     this.blobs.push(event.data);
     if (!this.localDownload) {
+      // Forward to LookitS3 instance, which manages uploading
       this.s3.onDataAvailable(event.data);
     }
   }
 
-  /** Download data url used in local development. */
-  private download() {
-    if (this.filename && this.url) {
+  /**
+   * Download data url used in local development. This can be used on
+   * snapshotted recordings after the recorder has been reset, so we need to
+   * pass in the filename and URL rather than relying on this.
+   *
+   * @param filename - Filename for the recording that will be downloaded.
+   * @param url - URL containing the file data to be downloaded.
+   */
+  private download(filename: string, url: string) {
+    if (filename && url) {
       const link = document.createElement("a");
-      link.href = this.url;
-      link.download = this.filename;
+      link.href = url;
+      link.download = filename;
       link.click();
     }
   }
@@ -409,4 +543,26 @@ export default class Recorder {
     const rand_digits = Math.floor(Math.random() * 1000);
     return `${prefix}_${window.chs.study.id}_${trial_id}_${window.chs.response.id}_${new Date().getTime()}_${rand_digits}.webm`;
   }
+
+  /**
+   * Create the timeout handler function for events that we're awaiting with a
+   * timeout.
+   *
+   * @param eventName - Name of the event we're awaiting, e.g. 'stop', 'upload'
+   * @param id - String to identify the promise that we're awaiting.
+   * @returns Callback function if the event promise times out.
+   */
+  private createTimeoutHandler(eventName: string, id: string) {
+    return () => {
+      console.warn(`Recorder ${eventName} timed out: ${id}`);
+      // check if reset is needed
+      if (!this.stream.active) {
+        try {
+          this.reset();
+        } catch (err) {
+          console.error("Error while resetting recorder after timeout: ", err);
+        }
+      }
+    };
+  }
 }

package/src/start.ts

@@ -1,11 +1,41 @@
 import { LookitWindow } from "@lookit/data/dist/types";
-import { JsPsych, JsPsychPlugin } from "jspsych";
+import chsTemplates from "@lookit/templates";
+import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych";
+import { version } from "../package.json";
 import { ExistingRecordingError } from "./errors";
 import Recorder from "./recorder";
 
 declare let window: LookitWindow;
 
-const info = <const>{ name: "start-record-plugin", parameters: {} };
+const info = <const>{
+  name: "start-record-plugin",
+  version,
+  parameters: {
+    /**
+     * This string can contain HTML markup. Any content provided will be
+     * displayed while the video recording connection is established. If null
+     * (the default), then the default 'establishing video connection, please
+     * wait' (or appropriate translation based on 'locale') will be displayed.
+     * Use a blank string for no message/content.
+     */
+    wait_for_connection_message: {
+      type: ParameterType.HTML_STRING,
+      default: null as null | string,
+    },
+    /**
+     * Locale code used for translating the default 'establishing video
+     * connection, please wait' message. This code must be present in the
+     * translation files. If the code is not found then English will be used. If
+     * the 'wait_for_connection_message' parameter is specified then this value
+     * is ignored.
+     */
+    locale: {
+      type: ParameterType.STRING,
+      default: "en-us",
+    },
+  },
+  data: {},
+};
 type Info = typeof info;
 
 /** Start recording. Used by researchers who want to record across trials. */
@@ -27,11 +57,27 @@ export default class StartRecordPlugin implements JsPsychPlugin<Info> {
     }
   }
 
-  /** Trial function called by jsPsych. */
-  public trial() {
-    this.recorder
+  /**
+   * Trial function called by jsPsych.
+   *
+   * @param display_element - DOM element where jsPsych content is being
+   *   rendered (set in initJsPsych and automatically made available to a
+   *   plugin's trial method via jsPsych core).
+   * @param trial - Trial object with parameters/values.
+   */
+  public async trial(
+    display_element: HTMLElement,
+    trial: TrialType<Info>,
+  ): Promise<void> {
+    if (trial.wait_for_connection_message == null) {
+      display_element.innerHTML = chsTemplates.establishingConnection(trial);
+    } else {
+      display_element.innerHTML = trial.wait_for_connection_message;
+    }
+    await this.recorder
       .start(false, `${StartRecordPlugin.info.name}-multiframe`)
       .then(() => {
+        display_element.innerHTML = "";
         this.jsPsych.finishTrial();
       });
   }

package/src/stop.ts

@@ -1,6 +1,7 @@
 import { LookitWindow } from "@lookit/data/dist/types";
 import chsTemplates from "@lookit/templates";
 import { JsPsych, JsPsychPlugin, ParameterType, TrialType } from "jspsych";
+import { version } from "../package.json";
 import { NoSessionRecordingError } from "./errors";
 import Recorder from "./recorder";
 
@@ -8,9 +9,40 @@ declare let window: LookitWindow;
 
 const info = <const>{
   name: "stop-record-plugin",
+  version,
   parameters: {
-    locale: { type: ParameterType.STRING, default: "en-us" },
+    /**
+     * This string can contain HTML markup. Any content provided will be
+     * displayed while the recording is uploading. If null (the default), then
+     * the default 'uploading video, please wait' (or appropriate translation
+     * based on 'locale') will be displayed. Use a blank string for no
+     * message/content.
+     */
+    wait_for_upload_message: {
+      type: ParameterType.HTML_STRING,
+      default: null as null | string,
+    },
+    /**
+     * Locale code used for translating the default 'uploading video, please
+     * wait' message. This code must be present in the translation files. If the
+     * code is not found then English will be used. If the
+     * 'wait_for_upload_message' parameter is specified then this value is
+     * ignored.
+     */
+    locale: {
+      type: ParameterType.STRING,
+      default: "en-us",
+    },
+    /**
+     * Maximum duration (in seconds) to wait for the session recording to finish
+     * uploading before continuing with the experiment.
+     */
+    max_upload_seconds: {
+      type: ParameterType.INT,
+      default: 10,
+    },
   },
+  data: {},
 };
 type Info = typeof info;
 
@@ -40,12 +72,31 @@ export default class StopRecordPlugin implements JsPsychPlugin<Info> {
    *   plugin's trial method via jsPsych core).
    * @param trial - Trial object with parameters/values.
    */
-  public trial(display_element: HTMLElement, trial: TrialType<Info>): void {
-    display_element.innerHTML = chsTemplates.uploadingVideo(trial);
-    this.recorder.stop().then(() => {
+  public async trial(
+    display_element: HTMLElement,
+    trial: TrialType<Info>,
+  ): Promise<void> {
+    if (trial.wait_for_upload_message == null) {
+      display_element.innerHTML = chsTemplates.uploadingVideo(trial);
+    } else {
+      display_element.innerHTML = trial.wait_for_upload_message;
+    }
+    const { stopped, uploaded } = this.recorder.stop({
+      upload_timeout_ms:
+        trial.max_upload_seconds !== null
+          ? trial.max_upload_seconds! * 1000
+          : null,
+    });
+    try {
+      await stopped;
+      await uploaded;
+    } catch (err) {
+      console.error("StopRecordPlugin: recorder stop/upload failed.", err);
+      // TO DO: display translated error msg and/or researcher contact info
+    } finally {
       window.chs.sessionRecorder = null;
       display_element.innerHTML = "";
       this.jsPsych.finishTrial();
-    });
+    }
   }
 }

package/src/trial.ts

@@ -1,16 +1,62 @@
+import chsTemplates from "@lookit/templates";
 import autoBind from "auto-bind";
-import { JsPsych, JsPsychExtension, JsPsychExtensionInfo } from "jspsych";
+import {
+  JsPsych,
+  JsPsychExtension,
+  JsPsychExtensionInfo,
+  PluginInfo,
+  TrialType,
+} from "jspsych";
+import { version } from "../package.json";
 import Recorder from "./recorder";
 import { jsPsychPluginWithInfo } from "./types";
 
-/** This extension will allow reasearchers to record trials. */
+// JsPsychExtensionInfo does not allow parameters, so we define them as interfaces and use these to type the arguments passed to extension initialize and on_start functions.
+interface Parameters {
+  /**
+   * Content that should be displayed while the recording is uploading. If null
+   * (the default), then the default 'uploading video, please wait...' (or
+   * appropriate translation based on 'locale') will be displayed. Use a blank
+   * string for no message/content. Otherwise this parameter can be set to a
+   * custom string and can contain HTML markup. If you want to embed
+   * images/video/audio in this HTML string, be sure to preload the media files
+   * with the `preload` plugin and manual preloading. Use a blank string (`""`)
+   * for no message/content.
+   *
+   * @default null
+   */
+  wait_for_upload_message?: null | string;
+  /**
+   * Locale code used for translating the default 'uploading video, please
+   * wait...' message. This code must be present in the translation files. If
+   * the code is not found then English will be used. If the
+   * 'wait_for_upload_message' parameter is specified then this value is
+   * ignored.
+   *
+   * @default "en-us"
+   */
+  locale?: string;
+  /**
+   * Maximum duration (in seconds) to wait for the trial recording to finish
+   * uploading before continuing with the experiment. Default is 10 seconds (set
+   * during initialize).
+   */
+  max_upload_seconds?: null | number;
+}
+
+/** This extension allows researchers to record webcam audio/video during trials. */
 export default class TrialRecordExtension implements JsPsychExtension {
   public static readonly info: JsPsychExtensionInfo = {
     name: "chs-trial-record-extension",
+    version,
+    data: {},
   };
 
   private recorder?: Recorder;
   private pluginName: string | undefined;
+  private uploadMsg: null | string = null;
+  private locale: string = "en-us";
+  private maxUploadSeconds: undefined | null | number = undefined;
 
   /**
    * Video recording extension.
@@ -22,30 +68,96 @@ export default class TrialRecordExtension implements JsPsychExtension {
   }
 
   /**
-   * Ran on the initialize step for extensions, called when an instance of
+   * Runs on the initialize step for extensions, called when an instance of
    * jsPsych is first initialized through initJsPsych().
+   *
+   * @param params - Parameters object
+   * @param params.wait_for_upload_message - Message to display while waiting
+   *   for upload. String or null (default)
+   * @param params.locale - Message to display while waiting for upload. String
+   *   or null (default).
    */
-  public async initialize() {}
-
-  /** Ran at the start of a trial. */
-  public on_start() {
-    this.recorder = new Recorder(this.jsPsych);
+  public async initialize(params?: Parameters) {
+    await new Promise<void>((resolve) => {
+      // set parameter defaults
+      this.uploadMsg = params?.wait_for_upload_message
+        ? params.wait_for_upload_message
+        : null;
+      this.locale = params?.locale ? params.locale : "en-us";
+      // null is a valid value - only set default if no parameter value was provided
+      this.maxUploadSeconds =
+        params?.max_upload_seconds === undefined
+          ? 10
+          : params.max_upload_seconds;
+      resolve();
+    });
   }
 
-  /** Ran when the trial has loaded. */
-  public on_load() {
+  /**
+   * Runs at the start of a trial.
+   *
+   * @param startParams - Parameters object
+   * @param startParams.wait_for_upload_message - Message to display while
+   *   waiting for upload. String or null (default). If given, this will
+   *   overwrite the value used during initialization.
+   */
+  public on_start(startParams?: Parameters) {
+    if (startParams?.wait_for_upload_message) {
+      this.uploadMsg = startParams.wait_for_upload_message;
+    }
+    if (startParams?.locale) {
+      this.locale = startParams.locale;
+    }
+    if (startParams?.max_upload_seconds !== undefined) {
+      this.maxUploadSeconds = startParams?.max_upload_seconds;
+    }
+    this.recorder = new Recorder(this.jsPsych);
     this.pluginName = this.getCurrentPluginName();
-    this.recorder?.start(false, `${this.pluginName}`);
+    this.recorder.start(false, `${this.pluginName}`);
   }
 
+  /** Runs when the trial has loaded. */
+  public on_load() {}
+
   /**
-   * Ran when trial has finished.
+   * Runs when trial has finished.
    *
-   * @returns Trial data.
+   * @returns Any data from the trial extension that should be added to the rest
+   *   of the trial data.
    */
-  public on_finish() {
-    this.recorder?.stop();
-    return {};
+  public async on_finish() {
+    const displayEl = this.jsPsych.getDisplayElement();
+    if (this.uploadMsg == null) {
+      displayEl.innerHTML = chsTemplates.uploadingVideo({
+        type: this.jsPsych.getCurrentTrial().type,
+        locale: this.locale,
+      } as TrialType<PluginInfo>);
+    } else {
+      displayEl.innerHTML = this.uploadMsg;
+    }
+    if (this.recorder) {
+      const { stopped, uploaded } = this.recorder.stop({
+        upload_timeout_ms:
+          this.maxUploadSeconds !== null ? this.maxUploadSeconds! * 1000 : null,
+      });
+      try {
+        await stopped;
+        await uploaded;
+        displayEl.innerHTML = "";
+        return {};
+      } catch (err) {
+        console.error(
+          "TrialRecordExtension: recorder stop/upload failed.",
+          err,
+        );
+        displayEl.innerHTML = "";
+        return {};
+        // TO DO: display translated error msg and/or researcher contact info
+      }
+    } else {
+      displayEl.innerHTML = "";
+      return {};
+    }
   }
 
   /**

package/src/types.ts

@@ -14,3 +14,24 @@ export type CSSWidthHeight =
   | number
   | `${number}${"px" | "cm" | "mm" | "em" | "%"}`
   | "auto";
+
+/** Options for the stop method */
+export interface StopOptions {
+  maintain_container_size?: boolean;
+  stop_timeout_ms?: number | null;
+  upload_timeout_ms?: number | null;
+}
+
+/** Result returned by the stop method */
+export interface StopResult {
+  /**
+   * Promise that resolves with the URL when the recorder has fully stopped.
+   * Returns a string with either the completion URL or "timeout".
+   */
+  stopped: Promise<string>;
+  /**
+   * Promise that resolves when the upload (or local download) completes.
+   * Returns void if succeeds or "timeout" if times out.
+   */
+  uploaded: Promise<void | string>;
+}