@zenvor/hls.js 1.0.0

package/src/remux/passthrough-remuxer.ts

@@ -0,0 +1,434 @@
+import {
+  flushTextTrackMetadataCueSamples,
+  flushTextTrackUserdataCueSamples,
+} from './mp4-remuxer';
+import { ElementaryStreamTypes } from '../loader/fragment';
+import { getCodecCompatibleName } from '../utils/codecs';
+import { type ILogger, Logger } from '../utils/logger';
+import { patchEncyptionData } from '../utils/mp4-tools';
+import { getSampleData, parseInitSegment } from '../utils/mp4-tools';
+import type { HlsConfig } from '../config';
+import type { HlsEventEmitter } from '../events';
+import type { DecryptData } from '../loader/level-key';
+import type {
+  DemuxedAudioTrack,
+  DemuxedMetadataTrack,
+  DemuxedUserdataTrack,
+  PassthroughTrack,
+} from '../types/demuxer';
+import type {
+  InitSegmentData,
+  RemuxedTrack,
+  Remuxer,
+  RemuxerResult,
+} from '../types/remuxer';
+import type { TrackSet } from '../types/track';
+import type { TypeSupported } from '../utils/codecs';
+import type { InitData, InitDataTrack, TrackTimes } from '../utils/mp4-tools';
+import type { TimestampOffset } from '../utils/timescale-conversion';
+
+class PassThroughRemuxer extends Logger implements Remuxer {
+  private emitInitSegment: boolean = false;
+  private audioCodec?: string;
+  private videoCodec?: string;
+  private initData?: InitData;
+  private initPTS: TimestampOffset | null = null;
+  private initTracks?: TrackSet;
+  private lastEndTime: number | null = null;
+  private isVideoContiguous: boolean = false;
+
+  constructor(
+    observer: HlsEventEmitter,
+    config: HlsConfig,
+    typeSupported: TypeSupported,
+    logger: ILogger,
+  ) {
+    super('passthrough-remuxer', logger);
+  }
+
+  public destroy() {}
+
+  public resetTimeStamp(defaultInitPTS: TimestampOffset | null) {
+    this.lastEndTime = null;
+    const initPTS = this.initPTS;
+    if (initPTS && defaultInitPTS) {
+      if (
+        initPTS.baseTime === defaultInitPTS.baseTime &&
+        initPTS.timescale === defaultInitPTS.timescale
+      ) {
+        return;
+      }
+    }
+    this.initPTS = defaultInitPTS;
+  }
+
+  public resetNextTimestamp() {
+    this.isVideoContiguous = false;
+    this.lastEndTime = null;
+  }
+
+  public resetInitSegment(
+    initSegment: Uint8Array<ArrayBuffer> | undefined,
+    audioCodec: string | undefined,
+    videoCodec: string | undefined,
+    decryptdata: DecryptData | null,
+  ) {
+    this.audioCodec = audioCodec;
+    this.videoCodec = videoCodec;
+    this.generateInitSegment(initSegment, decryptdata);
+    this.emitInitSegment = true;
+  }
+
+  private generateInitSegment(
+    initSegment: Uint8Array<ArrayBuffer> | undefined,
+    decryptdata?: DecryptData | null,
+  ) {
+    let { audioCodec, videoCodec } = this;
+    if (!initSegment?.byteLength) {
+      this.initTracks = undefined;
+      this.initData = undefined;
+      return;
+    }
+    const { audio, video } = (this.initData = parseInitSegment(initSegment));
+
+    if (decryptdata) {
+      patchEncyptionData(initSegment, decryptdata);
+    } else {
+      const eitherTrack = audio || video;
+      if (eitherTrack?.encrypted) {
+        this.warn(
+          `Init segment with encrypted track with has no key ("${eitherTrack.codec}")!`,
+        );
+      }
+    }
+
+    // Get codec from initSegment
+    if (audio) {
+      audioCodec = getParsedTrackCodec(
+        audio,
+        ElementaryStreamTypes.AUDIO,
+        this,
+      );
+    }
+
+    if (video) {
+      videoCodec = getParsedTrackCodec(
+        video,
+        ElementaryStreamTypes.VIDEO,
+        this,
+      );
+    }
+
+    const tracks: TrackSet = {};
+    if (audio && video) {
+      tracks.audiovideo = {
+        container: 'video/mp4',
+        codec: audioCodec + ',' + videoCodec,
+        supplemental: video.supplemental,
+        encrypted: video.encrypted,
+        initSegment,
+        id: 'main',
+      };
+    } else if (audio) {
+      tracks.audio = {
+        container: 'audio/mp4',
+        codec: audioCodec,
+        encrypted: audio.encrypted,
+        initSegment,
+        id: 'audio',
+      };
+    } else if (video) {
+      tracks.video = {
+        container: 'video/mp4',
+        codec: videoCodec,
+        supplemental: video.supplemental,
+        encrypted: video.encrypted,
+        initSegment,
+        id: 'main',
+      };
+    } else {
+      this.warn('initSegment does not contain moov or trak boxes.');
+    }
+    this.initTracks = tracks;
+  }
+
+  public remux(
+    audioTrack: DemuxedAudioTrack,
+    videoTrack: PassthroughTrack,
+    id3Track: DemuxedMetadataTrack,
+    textTrack: DemuxedUserdataTrack,
+    timeOffset: number,
+    accurateTimeOffset: boolean,
+  ): RemuxerResult {
+    let { initPTS, lastEndTime } = this;
+    const result: RemuxerResult = {
+      audio: undefined,
+      video: undefined,
+      text: undefined,
+      id3: id3Track,
+      initSegment: undefined,
+    };
+
+    // If we haven't yet set a lastEndDTS, or it was reset, set it to the provided timeOffset. We want to use the
+    // lastEndDTS over timeOffset whenever possible; during progressive playback, the media source will not update
+    // the media duration (which is what timeOffset is provided as) before we need to process the next chunk.
+    if (!Number.isFinite(lastEndTime!)) {
+      lastEndTime = this.lastEndTime = timeOffset || 0;
+    }
+
+    // The binary segment data is added to the videoTrack in the mp4demuxer. We don't check to see if the data is only
+    // audio or video (or both); adding it to video was an arbitrary choice.
+    const data = videoTrack.samples;
+    if (!data.length) {
+      return result;
+    }
+
+    const initSegment: InitSegmentData = {
+      initPTS: undefined,
+      timescale: undefined,
+      trackId: undefined,
+    };
+    let initData = this.initData;
+    if (!initData?.length) {
+      this.generateInitSegment(data);
+      initData = this.initData;
+    }
+    if (!initData?.length) {
+      // We can't remux if the initSegment could not be generated
+      this.warn('Failed to generate initSegment.');
+      return result;
+    }
+    if (this.emitInitSegment) {
+      initSegment.tracks = this.initTracks;
+      this.emitInitSegment = false;
+    }
+
+    const trackSampleData = getSampleData(data, initData, this);
+    const audioSampleTimestamps = initData.audio
+      ? trackSampleData[initData.audio.id]
+      : null;
+    const videoSampleTimestamps = initData.video
+      ? trackSampleData[initData.video.id]
+      : null;
+
+    const videoStartTime = toStartEndOrDefault(videoSampleTimestamps, Infinity);
+    const audioStartTime = toStartEndOrDefault(audioSampleTimestamps, Infinity);
+    const videoEndTime = toStartEndOrDefault(videoSampleTimestamps, 0, true);
+    const audioEndTime = toStartEndOrDefault(audioSampleTimestamps, 0, true);
+
+    let decodeTime = timeOffset;
+    let duration = 0;
+
+    const syncOnAudio =
+      audioSampleTimestamps &&
+      (!videoSampleTimestamps ||
+        (!initPTS && audioStartTime < videoStartTime) ||
+        (initPTS && initPTS.trackId === initData.audio!.id));
+    const baseOffsetSamples = syncOnAudio
+      ? audioSampleTimestamps
+      : videoSampleTimestamps;
+
+    if (baseOffsetSamples) {
+      const timescale = baseOffsetSamples.timescale;
+      const baseTime = baseOffsetSamples.start - timeOffset * timescale;
+      const trackId = syncOnAudio ? initData.audio!.id : initData.video!.id;
+
+      decodeTime = baseOffsetSamples.start / timescale;
+      duration = syncOnAudio
+        ? audioEndTime - audioStartTime
+        : videoEndTime - videoStartTime;
+
+      if (
+        (accurateTimeOffset || !initPTS) &&
+        (isInvalidInitPts(initPTS, decodeTime, timeOffset, duration) ||
+          timescale !== initPTS.timescale)
+      ) {
+        if (initPTS) {
+          this.warn(
+            `Timestamps at playlist time: ${accurateTimeOffset ? '' : '~'}${timeOffset} ${baseTime / timescale} != initPTS: ${initPTS.baseTime / initPTS.timescale} (${initPTS.baseTime}/${initPTS.timescale}) trackId: ${initPTS.trackId}`,
+          );
+        }
+        this.log(
+          `Found initPTS at playlist time: ${timeOffset} offset: ${decodeTime - timeOffset} (${baseTime}/${timescale}) trackId: ${trackId}`,
+        );
+        initPTS = null;
+        initSegment.initPTS = baseTime;
+        initSegment.timescale = timescale;
+        initSegment.trackId = trackId;
+      }
+    } else {
+      this.warn(
+        `No audio or video samples found for initPTS at playlist time: ${timeOffset}`,
+      );
+    }
+    if (!initPTS) {
+      if (
+        !initSegment.timescale ||
+        initSegment.trackId === undefined ||
+        initSegment.initPTS === undefined
+      ) {
+        this.warn('Could not set initPTS');
+        initSegment.initPTS = decodeTime;
+        initSegment.timescale = 1;
+        initSegment.trackId = -1;
+      }
+      this.initPTS = initPTS = {
+        baseTime: initSegment.initPTS,
+        timescale: initSegment.timescale,
+        trackId: initSegment.trackId,
+      };
+    } else {
+      initSegment.initPTS = initPTS.baseTime;
+      initSegment.timescale = initPTS.timescale;
+      initSegment.trackId = initPTS.trackId;
+    }
+
+    const startTime = decodeTime - initPTS.baseTime / initPTS.timescale;
+    const endTime = startTime + duration;
+
+    if (duration > 0) {
+      this.lastEndTime = endTime;
+    } else {
+      this.warn('Duration parsed from mp4 should be greater than zero');
+      this.resetNextTimestamp();
+    }
+
+    const hasAudio = !!initData.audio;
+    const hasVideo = !!initData.video;
+
+    let type: any = '';
+    if (hasAudio) {
+      type += 'audio';
+    }
+
+    if (hasVideo) {
+      type += 'video';
+    }
+
+    const encrypted =
+      (initData.audio ? initData.audio.encrypted : false) ||
+      (initData.video ? initData.video.encrypted : false);
+
+    const track: RemuxedTrack = {
+      data1: data,
+      startPTS: startTime,
+      startDTS: startTime,
+      endPTS: endTime,
+      endDTS: endTime,
+      type,
+      hasAudio,
+      hasVideo,
+      nb: 1,
+      dropped: 0,
+      encrypted,
+    };
+
+    result.audio = hasAudio && !hasVideo ? track : undefined;
+    result.video = hasVideo ? track : undefined;
+    const videoSampleCount = videoSampleTimestamps?.sampleCount;
+    if (videoSampleCount) {
+      const firstKeyFrame = videoSampleTimestamps.keyFrameIndex;
+      const independent = firstKeyFrame !== -1;
+      track.nb = videoSampleCount;
+      track.dropped =
+        firstKeyFrame === 0 || this.isVideoContiguous
+          ? 0
+          : independent
+            ? firstKeyFrame
+            : videoSampleCount;
+      track.independent = independent;
+      track.firstKeyFrame = firstKeyFrame;
+      if (independent && videoSampleTimestamps.keyFrameStart) {
+        track.firstKeyFramePTS =
+          (videoSampleTimestamps.keyFrameStart - initPTS.baseTime) /
+          initPTS.timescale;
+      }
+      if (!this.isVideoContiguous) {
+        result.independent = independent;
+      }
+      this.isVideoContiguous ||= independent;
+      if (track.dropped) {
+        this.warn(
+          `fmp4 does not start with IDR: firstIDR ${firstKeyFrame}/${videoSampleCount} dropped: ${track.dropped} start: ${track.firstKeyFramePTS || 'NA'}`,
+        );
+      }
+    }
+
+    result.initSegment = initSegment;
+    result.id3 = flushTextTrackMetadataCueSamples(
+      id3Track,
+      timeOffset,
+      initPTS,
+      initPTS,
+    );
+
+    if (textTrack.samples.length) {
+      result.text = flushTextTrackUserdataCueSamples(
+        textTrack,
+        timeOffset,
+        initPTS,
+      );
+    }
+
+    return result;
+  }
+}
+
+function toStartEndOrDefault(
+  trackTimes: TrackTimes | null,
+  defaultValue: number,
+  end: boolean = false,
+): number {
+  return trackTimes?.start !== undefined
+    ? (trackTimes.start + (end ? trackTimes.duration : 0)) /
+        trackTimes.timescale
+    : defaultValue;
+}
+
+function isInvalidInitPts(
+  initPTS: TimestampOffset | null,
+  startDTS: number,
+  timeOffset: number,
+  duration: number,
+): initPTS is null {
+  if (initPTS === null) {
+    return true;
+  }
+  // InitPTS is invalid when distance from program would be more than or equal to segment duration or a minimum of one second
+  const minDuration = Math.max(duration, 1);
+  const startTime = startDTS - initPTS.baseTime / initPTS.timescale;
+  return Math.abs(startTime - timeOffset) >= minDuration;
+}
+
+function getParsedTrackCodec(
+  track: InitDataTrack,
+  type: ElementaryStreamTypes.AUDIO | ElementaryStreamTypes.VIDEO,
+  logger: ILogger,
+): string {
+  const parsedCodec = track.codec;
+  if (parsedCodec && parsedCodec.length > 4) {
+    return parsedCodec;
+  }
+  if (type === ElementaryStreamTypes.AUDIO) {
+    if (
+      parsedCodec === 'ec-3' ||
+      parsedCodec === 'ac-3' ||
+      parsedCodec === 'alac'
+    ) {
+      return parsedCodec;
+    }
+    if (parsedCodec === 'fLaC' || parsedCodec === 'Opus') {
+      // Opting not to get `preferManagedMediaSource` from player config for isSupported() check for simplicity
+      const preferManagedMediaSource = false;
+      return getCodecCompatibleName(parsedCodec, preferManagedMediaSource);
+    }
+
+    logger.warn(`Unhandled audio codec "${parsedCodec}" in mp4 MAP`);
+    return parsedCodec || 'mp4a';
+  }
+  // Provide defaults based on codec type
+  // This allows for some playback of some fmp4 playlists without CODECS defined in manifest
+  logger.warn(`Unhandled video codec "${parsedCodec}" in mp4 MAP`);
+  return parsedCodec || 'avc1';
+}
+export default PassThroughRemuxer;

package/src/task-loop.ts

@@ -0,0 +1,130 @@
+import { type ILogger, Logger } from './utils/logger';
+
+/**
+ * @ignore
+ * Sub-class specialization of EventHandler base class.
+ *
+ * TaskLoop allows to schedule a task function being called (optionnaly repeatedly) on the main loop,
+ * scheduled asynchroneously, avoiding recursive calls in the same tick.
+ *
+ * The task itself is implemented in `doTick`. It can be requested and called for single execution
+ * using the `tick` method.
+ *
+ * It will be assured that the task execution method (`tick`) only gets called once per main loop "tick",
+ * no matter how often it gets requested for execution. Execution in further ticks will be scheduled accordingly.
+ *
+ * If further execution requests have already been scheduled on the next tick, it can be checked with `hasNextTick`,
+ * and cancelled with `clearNextTick`.
+ *
+ * The task can be scheduled as an interval repeatedly with a period as parameter (see `setInterval`, `clearInterval`).
+ *
+ * Sub-classes need to implement the `doTick` method which will effectively have the task execution routine.
+ *
+ * Further explanations:
+ *
+ * The baseclass has a `tick` method that will schedule the doTick call. It may be called synchroneously
+ * only for a stack-depth of one. On re-entrant calls, sub-sequent calls are scheduled for next main loop ticks.
+ *
+ * When the task execution (`tick` method) is called in re-entrant way this is detected and
+ * we are limiting the task execution per call stack to exactly one, but scheduling/post-poning further
+ * task processing on the next main loop iteration (also known as "next tick" in the Node/JS runtime lingo).
+ */
+export default class TaskLoop extends Logger {
+  private readonly _boundTick: () => void;
+  private _tickTimer: number | null = null;
+  private _tickInterval: number | null = null;
+  private _tickCallCount = 0;
+
+  constructor(label: string, logger: ILogger) {
+    super(label, logger);
+    this._boundTick = this.tick.bind(this);
+  }
+
+  public destroy() {
+    this.onHandlerDestroying();
+    this.onHandlerDestroyed();
+  }
+
+  protected onHandlerDestroying() {
+    // clear all timers before unregistering from event bus
+    this.clearNextTick();
+    this.clearInterval();
+  }
+
+  protected onHandlerDestroyed() {}
+
+  public hasInterval(): boolean {
+    return !!this._tickInterval;
+  }
+
+  public hasNextTick(): boolean {
+    return !!this._tickTimer;
+  }
+
+  /**
+   * @param millis - Interval time (ms)
+   * @eturns True when interval has been scheduled, false when already scheduled (no effect)
+   */
+  public setInterval(millis: number): boolean {
+    if (!this._tickInterval) {
+      this._tickCallCount = 0;
+      this._tickInterval = self.setInterval(this._boundTick, millis);
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * @returns True when interval was cleared, false when none was set (no effect)
+   */
+  public clearInterval(): boolean {
+    if (this._tickInterval) {
+      self.clearInterval(this._tickInterval);
+      this._tickInterval = null;
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * @returns True when timeout was cleared, false when none was set (no effect)
+   */
+  public clearNextTick(): boolean {
+    if (this._tickTimer) {
+      self.clearTimeout(this._tickTimer);
+      this._tickTimer = null;
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * Will call the subclass doTick implementation in this main loop tick
+   * or in the next one (via setTimeout(,0)) in case it has already been called
+   * in this tick (in case this is a re-entrant call).
+   */
+  public tick(): void {
+    this._tickCallCount++;
+    if (this._tickCallCount === 1) {
+      this.doTick();
+      // re-entrant call to tick from previous doTick call stack
+      // -> schedule a call on the next main loop iteration to process this task processing request
+      if (this._tickCallCount > 1) {
+        // make sure only one timer exists at any time at max
+        this.tickImmediate();
+      }
+      this._tickCallCount = 0;
+    }
+  }
+
+  public tickImmediate(): void {
+    this.clearNextTick();
+    this._tickTimer = self.setTimeout(this._boundTick, 0);
+  }
+
+  /**
+   * For subclass to implement task logic
+   * @abstract
+   */
+  protected doTick(): void {}
+}

package/src/types/algo.ts

@@ -0,0 +1,44 @@
+export type AutoCameraItem = {
+  x: number;
+  y: number;
+  focus: number;
+  reserved: [number, number, number, number];
+};
+
+export type TrackItem = {
+  trackId: number;
+  score: number;
+  box: [number, number, number, number];
+  reserved: [number, number, number, number];
+};
+
+export type DetItem = {
+  classId: number;
+  score: number;
+  box: [number, number, number, number];
+  reserved: [number, number, number, number];
+};
+
+export type FrameItem = {
+  frameIdx: number;
+  autoCameras: AutoCameraItem;
+  tracks: TrackItem[];
+  detections: DetItem[];
+};
+
+export type AipdMessage = {
+  version: number;
+  chunkIndex: number;
+  frameSize: number;
+  frames: FrameItem[];
+};
+
+export type AlgoChunk = {
+  fragSn: number;
+  algoUrl: string;
+  chunkIndex: number;
+  frameSize: number;
+  frameRate: number;
+  startFrameIndex: number;
+  frames: FrameItem[];
+};

package/src/types/buffer.ts

@@ -0,0 +1,105 @@
+declare global {
+  interface ArrayBuffer {
+    ' buffer_kind'?: 'array';
+  }
+  interface Uint8Array {
+    ' buffer_kind'?: 'uint8';
+  }
+}
+
+export type SourceBufferName = 'video' | 'audio' | 'audiovideo';
+
+/* eslint-disable no-restricted-globals */
+// `SourceBuffer` global is restricted for is-supported check with prefixed MSE
+export type ExtendedSourceBuffer = SourceBuffer & {
+  onbufferedchange?: ((this: SourceBuffer, ev: Event) => any) | null;
+};
+/* eslint-enable no-restricted-globals */
+
+export interface BaseTrack {
+  id: 'audio' | 'main';
+  container: string;
+  codec?: string;
+  supplemental?: string;
+  encrypted?: boolean;
+  levelCodec?: string;
+  pendingCodec?: string;
+  metadata?: {
+    channelCount?: number;
+    width?: number;
+    height?: number;
+  };
+}
+
+export interface ParsedTrack extends BaseTrack {
+  initSegment?: Uint8Array;
+}
+export interface SourceBufferTrack extends BaseTrack {
+  buffer?: ExtendedSourceBuffer;
+  bufferAppendTimeoutId?: number;
+  listeners: SourceBufferListener[];
+  ending?: boolean;
+  ended?: boolean;
+}
+
+export interface PendingSourceBufferTrack extends SourceBufferTrack {
+  buffer: undefined;
+}
+
+export interface BufferCreatedTrack extends BaseTrack {
+  buffer: ExtendedSourceBuffer;
+}
+
+export type ParsedTrackSet = Partial<Record<SourceBufferName, ParsedTrack>>;
+
+export type BaseTrackSet = Partial<Record<SourceBufferName, BaseTrack>>;
+
+export type SourceBufferTrackSet = Partial<
+  Record<SourceBufferName, SourceBufferTrack>
+>;
+
+export type BufferCreatedTrackSet = Partial<
+  Record<SourceBufferName, BufferCreatedTrack>
+>;
+
+export type EmptyTuple = [null, null];
+
+export type SourceBuffersTuple = [
+  (
+    | [Extract<SourceBufferName, 'video' | 'audiovideo'>, ExtendedSourceBuffer]
+    | EmptyTuple
+  ),
+  [Extract<SourceBufferName, 'audio'>, ExtendedSourceBuffer] | EmptyTuple,
+];
+
+export type MediaOverrides = {
+  duration?: number;
+  endOfStream?: boolean;
+};
+
+export interface BufferOperationQueues {
+  video: Array<BufferOperation>;
+  audio: Array<BufferOperation>;
+  audiovideo: Array<BufferOperation>;
+}
+
+export interface BufferOperation {
+  label: string;
+  execute: Function;
+  onStart: Function;
+  onComplete: Function;
+  onError: Function;
+  start?: number;
+  end?: number;
+}
+
+export interface SourceBufferListener {
+  event: string;
+  listener: EventListener;
+}
+
+export type AttachMediaSourceData = {
+  media: HTMLMediaElement;
+  mediaSource: MediaSource | null;
+  tracks: SourceBufferTrackSet;
+};