@lookit/record 0.0.3

package/src/recorder.ts

@@ -0,0 +1,353 @@
+import Data from "@lookit/data";
+import LookitS3 from "@lookit/data/dist/lookitS3";
+import autoBind from "auto-bind";
+import Handlebars from "handlebars";
+import { JsPsych } from "jspsych";
+import play_icon from "../img/play-icon.svg";
+import record_icon from "../img/record-icon.svg";
+import playbackFeed from "../templates/playback-feed.hbs";
+import recordFeed from "../templates/record-feed.hbs";
+import webcamFeed from "../templates/webcam-feed.hbs";
+import {
+  CreateURLError,
+  NoStopPromiseError,
+  NoWebCamElementError,
+  RecorderInitializeError,
+  S3UndefinedError,
+  StreamActiveOnResetError,
+  StreamDataInitializeError,
+  StreamInactiveInitializeError,
+} from "./errors";
+import { CSSWidthHeight } from "./types";
+
+/** Recorder handles the state of recording and data storage. */
+export default class Recorder {
+  private url?: string;
+  private _s3?: LookitS3;
+
+  private blobs: Blob[] = [];
+  private localDownload: boolean =
+    process.env.LOCAL_DOWNLOAD?.toLowerCase() === "true";
+  private filename?: string;
+  private stopPromise?: Promise<void>;
+  private webcam_element_id = "lookit-jspsych-webcam";
+
+  private streamClone: MediaStream;
+
+  /**
+   * Recorder for online experiments.
+   *
+   * @param jsPsych - Object supplied by jsPsych.
+   */
+  public constructor(private jsPsych: JsPsych) {
+    this.streamClone = this.stream.clone();
+    autoBind(this);
+  }
+
+  /**
+   * Get recorder from jsPsych plugin API.
+   *
+   * If camera recorder hasn't been initialized, then return the microphone
+   * recorder.
+   *
+   * @returns MediaRecorder from the plugin API.
+   */
+  private get recorder() {
+    return (
+      this.jsPsych.pluginAPI.getCameraRecorder() ||
+      this.jsPsych.pluginAPI.getMicrophoneRecorder()
+    );
+  }
+
+  /**
+   * Get stream from either recorder.
+   *
+   * @returns MediaStream from the plugin API.
+   */
+  private get stream() {
+    return this.recorder.stream;
+  }
+
+  /**
+   * Get s3 class variable. Throw error if doesn't exist.
+   *
+   * @returns - S3 object.
+   */
+  private get s3() {
+    if (!this._s3) {
+      throw new S3UndefinedError();
+    }
+    return this._s3;
+  }
+
+  /** Set s3 class variable. */
+  private set s3(value: LookitS3) {
+    this._s3 = value;
+  }
+
+  /**
+   * Initialize recorder using the jsPsych plugin API. There should always be a
+   * stream initialized when the Recorder class is instantiated. This method is
+   * just used to re-initialize the stream with a clone when the recorder needs
+   * to be reset during a trial.
+   *
+   * @param stream - Media stream returned from getUserMedia that should be used
+   *   to set up the jsPsych recorder.
+   * @param opts - Media recorder options to use when setting up the recorder.
+   */
+  public initializeRecorder(stream: MediaStream, opts?: MediaRecorderOptions) {
+    this.jsPsych.pluginAPI.initializeCameraRecorder(stream, opts);
+  }
+
+  /** Reset the recorder to be used again. */
+  public reset() {
+    if (this.stream.active) {
+      throw new StreamActiveOnResetError();
+    }
+    this.initializeRecorder(this.streamClone.clone());
+    this.blobs = [];
+  }
+
+  /**
+   * Insert a rendered template into an element.
+   *
+   * @param element - Element to have video inserted into.
+   * @param template - Template string
+   * @param insertStream - Should the stream be attributed to the webcam
+   *   element.
+   * @returns Webcam element
+   */
+  private insertVideoFeed(
+    element: HTMLDivElement,
+    template: string,
+    insertStream: boolean = true,
+  ) {
+    const { webcam_element_id, stream } = this;
+
+    element.innerHTML = template;
+
+    const webcam = element.querySelector<HTMLVideoElement>(
+      `#${webcam_element_id}`,
+    );
+
+    if (!webcam) {
+      throw new NoWebCamElementError();
+    }
+
+    if (insertStream) {
+      webcam.srcObject = stream;
+    }
+
+    return webcam;
+  }
+
+  /**
+   * Insert a video element containing the webcam feed onto the page.
+   *
+   * @param element - The HTML div element that should serve as the container
+   *   for the webcam display.
+   * @param width - The width of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'100%'`
+   * @param height - The height of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'auto'`
+   */
+  public insertWebcamFeed(
+    element: HTMLDivElement,
+    width: CSSWidthHeight = "100%",
+    height: CSSWidthHeight = "auto",
+  ) {
+    const { webcam_element_id } = this;
+    const view = { height, width, webcam_element_id };
+    this.insertVideoFeed(element, Handlebars.compile(webcamFeed)(view));
+  }
+
+  /**
+   * Insert video playback feed into supplied element.
+   *
+   * @param element - The HTML div element that should serve as the container
+   *   for the webcam display.
+   * @param on_ended - Callback function called when playing video ends.
+   * @param width - The width of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'100%'`
+   * @param height - The height of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'auto'`
+   */
+  public insertPlaybackFeed(
+    element: HTMLDivElement,
+    on_ended: (this: HTMLVideoElement, e: Event) => void,
+    width: CSSWidthHeight = "100%",
+    height: CSSWidthHeight = "auto",
+  ) {
+    const { webcam_element_id } = this;
+    const view = {
+      src: this.url,
+      width,
+      height,
+      webcam_element_id,
+      play_icon,
+    };
+
+    const playbackElement = this.insertVideoFeed(
+      element,
+      Handlebars.compile(playbackFeed)(view),
+      false,
+    );
+
+    playbackElement.addEventListener("ended", on_ended, { once: true });
+  }
+
+  /**
+   * Insert a feed to be used for recording into an element.
+   *
+   * @param element - Element to have record feed inserted into.
+   * @param width - The width of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'100%'`
+   * @param height - The height of the video element containing the webcam feed,
+   *   in CSS units (optional). Default is `'auto'`
+   */
+  public insertRecordFeed(
+    element: HTMLDivElement,
+    width: CSSWidthHeight = "100%",
+    height: CSSWidthHeight = "auto",
+  ) {
+    const { webcam_element_id } = this;
+    const view = { height, width, webcam_element_id, record_icon };
+    this.insertVideoFeed(element, Handlebars.compile(recordFeed)(view));
+  }
+
+  /**
+   * Start recording. Also, adds event listeners for handling data and checks
+   * for recorder initialization.
+   *
+   * @param prefix - Prefix for the video recording file name (string). This is
+   *   the string that comes before "_<TIMESTAMP>.webm".
+   */
+  public async start(prefix: "consent" | "session_video" | "trial_video") {
+    this.initializeCheck();
+
+    // Set filename
+    this.filename = `${prefix}_${new Date().getTime()}.webm`;
+
+    // Instantiate s3 object
+    if (!this.localDownload) {
+      this.s3 = new Data.LookitS3(this.filename);
+    }
+
+    this.recorder.addEventListener("dataavailable", this.handleDataAvailable);
+
+    // create a stop promise and pass the resolve function as an argument to the stop event callback,
+    // so that the stop event handler can resolve the stop promise
+    this.stopPromise = new Promise((resolve) => {
+      this.recorder.addEventListener("stop", this.handleStop(resolve));
+    });
+
+    if (!this.localDownload) {
+      await this.s3.createUpload();
+    }
+
+    this.recorder.start();
+  }
+
+  /**
+   * Stop all streams/tracks. This stops any in-progress recordings and releases
+   * the media devices. This is can be called when recording is not in progress,
+   * e.g. To end the camera/mic access when the experiment is displaying the
+   * camera feed but not recording (e.g. Video-config).
+   */
+  public stopTracks() {
+    this.recorder.stop();
+    this.stream.getTracks().map((t) => t.stop());
+  }
+
+  /**
+   * Stop recording and camera/microphone. This will stop accessing all media
+   * tracks, clear the webcam feed element (if there is one), and return the
+   * stop promise. This should only be called after recording has started.
+   *
+   * @returns Promise that resolves after the media recorder has stopped and
+   *   final 'dataavailable' event has occurred, when the "stop" event-related
+   *   callback function is called.
+   */
+  public stop() {
+    this.stopTracks();
+    this.clearWebcamFeed();
+
+    if (!this.stopPromise) {
+      throw new NoStopPromiseError();
+    }
+    return this.stopPromise;
+  }
+
+  /** Throw Error if there isn't a recorder provided by jsPsych. */
+  private initializeCheck() {
+    if (!this.recorder) {
+      throw new RecorderInitializeError();
+    }
+
+    if (!this.stream.active) {
+      throw new StreamInactiveInitializeError();
+    }
+
+    if (this.blobs.length !== 0) {
+      throw new StreamDataInitializeError();
+    }
+  }
+
+  /**
+   * Handle the recorder's stop event. This is a function that takes the stop
+   * promise's 'resolve' as an argument and returns a function that resolves
+   * that stop promise. The function that is returned is used as the recorder's
+   * "stop" event-related callback function.
+   *
+   * @param resolve - Promise resolve function.
+   * @returns Function that is called on the recorder's "stop" event.
+   */
+  private handleStop(resolve: () => void) {
+    return async () => {
+      if (this.blobs.length === 0) {
+        throw new CreateURLError();
+      }
+      this.url = URL.createObjectURL(new Blob(this.blobs));
+
+      if (this.localDownload) {
+        this.download();
+      } else {
+        await this.s3.completeUpload();
+      }
+
+      resolve();
+    };
+  }
+
+  /**
+   * Function ran at each time slice and when the recorder stopped.
+   *
+   * @param event - Event containing blob data.
+   */
+  private handleDataAvailable(event: BlobEvent) {
+    this.blobs.push(event.data);
+    if (!this.localDownload) {
+      this.s3.onDataAvailable(event.data);
+    }
+  }
+
+  /** Download data url used in local development. */
+  private download() {
+    if (this.filename && this.url) {
+      const link = document.createElement("a");
+      link.href = this.url;
+      link.download = this.filename;
+      link.click();
+    }
+  }
+
+  /** Private helper to clear the webcam feed, if there is one. */
+  private clearWebcamFeed() {
+    const webcam_feed_element = document.querySelector(
+      `#${this.webcam_element_id}`,
+    ) as HTMLVideoElement;
+    if (webcam_feed_element) {
+      webcam_feed_element.remove();
+    }
+  }
+}

package/src/start.ts

@@ -0,0 +1,36 @@
+import { LookitWindow } from "@lookit/data/dist/types";
+import { JsPsych, JsPsychPlugin } from "jspsych";
+import { ExistingRecordingError } from "./errors";
+import Recorder from "./recorder";
+
+declare let window: LookitWindow;
+
+const info = <const>{ name: "start-record-plugin", parameters: {} };
+type Info = typeof info;
+
+/** Start recording. Used by researchers who want to record across trials. */
+export default class StartRecordPlugin implements JsPsychPlugin<Info> {
+  public static readonly info = info;
+  private recorder: Recorder;
+
+  /**
+   * Plugin used to start recording.
+   *
+   * @param jsPsych - Object provided by jsPsych.
+   */
+  public constructor(private jsPsych: JsPsych) {
+    this.recorder = new Recorder(this.jsPsych);
+    if (!window.chs.sessionRecorder) {
+      window.chs.sessionRecorder = this.recorder;
+    } else {
+      throw new ExistingRecordingError();
+    }
+  }
+
+  /** Trial function called by jsPsych. */
+  public trial() {
+    this.recorder.start("session_video").then(() => {
+      this.jsPsych.finishTrial();
+    });
+  }
+}

package/src/stop.ts

@@ -0,0 +1,46 @@
+import { LookitWindow } from "@lookit/data/dist/types";
+import Handlebars from "handlebars";
+import { JsPsych, JsPsychPlugin } from "jspsych";
+import uploadingVideo from "../templates/uploading-video.hbs";
+import { NoSessionRecordingError } from "./errors";
+import Recorder from "./recorder";
+
+declare let window: LookitWindow;
+
+const info = <const>{ name: "stop-record-plugin", parameters: {} };
+type Info = typeof info;
+
+/** Stop recording. Used by researchers who want to record across trials. */
+export default class StopRecordPlugin implements JsPsychPlugin<Info> {
+  public static readonly info = info;
+  private recorder: Recorder;
+
+  /**
+   * Plugin used to stop recording.
+   *
+   * @param jsPsych - Object provided by jsPsych.
+   */
+  public constructor(private jsPsych: JsPsych) {
+    if (window.chs.sessionRecorder) {
+      this.recorder = window.chs.sessionRecorder as Recorder;
+    } else {
+      throw new NoSessionRecordingError();
+    }
+  }
+
+  /**
+   * Trial function called by jsPsych.
+   *
+   * @param display_element - DOM element where jsPsych content is being
+   *   rendered (set in initJsPsych and automatically made available to a
+   *   plugin's trial method via jsPsych core).
+   */
+  public trial(display_element: HTMLElement): void {
+    display_element.innerHTML = Handlebars.compile(uploadingVideo)({});
+    this.recorder.stop().then(() => {
+      window.chs.sessionRecorder = null;
+      display_element.innerHTML = "";
+      this.jsPsych.finishTrial();
+    });
+  }
+}

package/src/string-import.d.ts

@@ -0,0 +1,14 @@
+declare module "*.svg" {
+  const file: string;
+  export default file;
+}
+
+declare module "*.yaml" {
+  const file: string;
+  export default file;
+}
+
+declare module "*.hbs" {
+  const file: string;
+  export default file;
+}

package/src/trial.ts

@@ -0,0 +1,47 @@
+import autoBind from "auto-bind";
+import { JsPsych, JsPsychExtension, JsPsychExtensionInfo } from "jspsych";
+import Recorder from "./recorder";
+
+/** This extension will allow reasearchers to record trials. */
+export default class TrialRecordExtension implements JsPsychExtension {
+  public static readonly info: JsPsychExtensionInfo = {
+    name: "chs-trial-record-extension",
+  };
+
+  private recorder?: Recorder;
+
+  /**
+   * Video recording extension.
+   *
+   * @param jsPsych - JsPsych object passed into extensions.
+   */
+  public constructor(private jsPsych: JsPsych) {
+    autoBind(this);
+  }
+
+  /**
+   * Ran on the initialize step for extensions, called when an instance of
+   * jsPsych is first initialized through initJsPsych().
+   */
+  public async initialize() {}
+
+  /** Ran at the start of a trial. */
+  public on_start() {
+    this.recorder = new Recorder(this.jsPsych);
+  }
+
+  /** Ran when the trial has loaded. */
+  public on_load() {
+    this.recorder?.start("trial_video");
+  }
+
+  /**
+   * Ran when trial has finished.
+   *
+   * @returns Trial data.
+   */
+  public on_finish() {
+    this.recorder?.stop();
+    return {};
+  }
+}

package/src/types.ts

@@ -0,0 +1,8 @@
+/**
+ * A valid CSS height/width value, which can be a number, a string containing a
+ * number with units, or 'auto'.
+ */
+export type CSSWidthHeight =
+  | number
+  | `${number}${"px" | "cm" | "mm" | "em" | "%"}`
+  | "auto";

package/src/utils.spec.ts

@@ -0,0 +1,55 @@
+import Yaml from "js-yaml";
+import en_us from "../i18n/en-us.yaml";
+import eu from "../i18n/eu.yaml";
+import fr from "../i18n/fr.yaml";
+import hu from "../i18n/hu.yaml";
+import it from "../i18n/it.yaml";
+import ja from "../i18n/ja.yaml";
+import nl from "../i18n/nl.yaml";
+import pt_br from "../i18n/pt-br.yaml";
+import pt from "../i18n/pt.yaml";
+import { TranslationNotFoundError } from "./errors";
+import { expFormat, getTranslation } from "./utils";
+
+test("expFormat convert written text to format well in HTML", () => {
+  expect(expFormat("abcdefg")).toStrictEqual("abcdefg");
+  expect(expFormat("AAABBBCCC")).toStrictEqual("AAABBBCCC");
+  expect(expFormat("A normal sentence with multiple words.")).toStrictEqual(
+    "A normal sentence with multiple words.",
+  );
+  expect(expFormat(["Array", "of", "strings"])).toStrictEqual(
+    "Array<br><br>of<br><br>strings",
+  );
+  expect(expFormat("carriage return an newline\r\n")).toStrictEqual(
+    "carriage return an newline<br>",
+  );
+  expect(expFormat("new line\n")).toStrictEqual("new line<br>");
+  expect(expFormat("carriage return\r")).toStrictEqual("carriage return<br>");
+  expect(expFormat("\tTabbed text")).toStrictEqual(
+    "&nbsp;&nbsp;&nbsp;&nbsp;Tabbed text",
+  );
+});
+
+test("Get translation file for specified locale", () => {
+  const translations = {
+    ja,
+    pt,
+    eu,
+    fr,
+    hu,
+    it,
+    nl,
+    "en-us": en_us,
+    "pt-br": pt_br,
+  };
+
+  for (const [k, v] of Object.entries<string>(translations)) {
+    expect(getTranslation(new Intl.Locale(k))).toStrictEqual(Yaml.load(v));
+  }
+
+  expect(pt_br).not.toStrictEqual(pt);
+
+  expect(() => getTranslation(new Intl.Locale("not-a2code"))).toThrow(
+    TranslationNotFoundError,
+  );
+});

package/src/utils.ts

@@ -0,0 +1,119 @@
+import Handlebars from "handlebars";
+import i18next from "i18next";
+import ICU from "i18next-icu";
+import Yaml from "js-yaml";
+import { PluginInfo, TrialType } from "jspsych";
+import en_us from "../i18n/en-us.yaml";
+import eu from "../i18n/eu.yaml";
+import fr from "../i18n/fr.yaml";
+import hu from "../i18n/hu.yaml";
+import it from "../i18n/it.yaml";
+import ja from "../i18n/ja.yaml";
+import nl from "../i18n/nl.yaml";
+import pt_br from "../i18n/pt-br.yaml";
+import pt from "../i18n/pt.yaml";
+import { TranslationNotFoundError } from "./errors";
+
+/**
+ * Pulled from EFP. Function to convert researcher's text to HTML.
+ *
+ * @param text - Text
+ * @returns Formatted string
+ */
+export const expFormat = (text?: string | string[]) => {
+  if (!text) {
+    return "";
+  }
+
+  if (Array.isArray(text)) {
+    text = text.join("\n\n");
+  }
+
+  return text
+    .replace(/(\r\n|\n|\r)/gm, "<br>")
+    .replace(/\t/gm, "&nbsp;&nbsp;&nbsp;&nbsp;");
+};
+
+/**
+ * Get a translation file based on selected language.
+ *
+ * @param lcl - Locale object with locale
+ * @returns Translations from i18next
+ */
+export const getTranslation = (lcl: Intl.Locale) => {
+  /**
+   * Switch case to find language from a string. Will throw error is language
+   * not found.
+   *
+   * @param baseName - Base name from locale (en-us)
+   * @returns Language yaml file
+   */
+  const getYaml = (baseName: string) => {
+    switch (baseName) {
+      case "en-US":
+        return en_us;
+      case "eu":
+        return eu;
+      case "fr":
+        return fr;
+      case "hu":
+        return hu;
+      case "it":
+        return it;
+      case "ja":
+        return ja;
+      case "nl":
+        return nl;
+      case "pt-BR":
+        return pt_br;
+      case "pt":
+        return pt;
+      default:
+        throw new TranslationNotFoundError(baseName);
+    }
+  };
+
+  return Yaml.load(getYaml(lcl.baseName)) as Record<string, string>;
+};
+
+/**
+ * Initialize i18next with parameters from trial.
+ *
+ * @param trial - Trial data including user supplied parameters.
+ */
+const initI18next = (trial: TrialType<PluginInfo>) => {
+  const debug = process.env.DEBUG === "true";
+  const lcl = new Intl.Locale(trial.locale);
+  const translation = getTranslation(lcl);
+
+  i18next.use(ICU).init({
+    lng: lcl.baseName,
+    debug,
+    resources: {
+      [lcl.language]: {
+        translation,
+      },
+    },
+  });
+};
+
+/**
+ * Initialize handlebars helpers. This could be done globally, but it does go
+ * hand in hand with initializing i18n.
+ */
+const initHandlebars = () => {
+  Handlebars.registerHelper("t", (context, { hash }) =>
+    i18next.t(context, hash),
+  );
+  Handlebars.registerHelper("exp-format", (context) => expFormat(context));
+};
+
+/**
+ * Initialize both i18next and Handlebars.
+ *
+ * @param trial - Yup
+ */
+export const initI18nAndTemplates = (trial: TrialType<PluginInfo>) => {
+  initI18next(trial);
+  initHandlebars();
+};