@ai-matrx/media 0.6.67 → 0.7.0

package/dist/live-audio.js

@@ -0,0 +1,674 @@
+// src/live-audio/pcm-player.ts
+function base64ToInt16(b64) {
+  const binary = atob(b64);
+  const len = binary.length;
+  const bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) bytes[i] = binary.charCodeAt(i);
+  return new Int16Array(bytes.buffer, 0, len >> 1);
+}
+function createStreamingPcmPlayer(format) {
+  const sampleRate = format.sampleRate > 0 ? format.sampleRate : 24e3;
+  const channels = format.channels > 0 ? format.channels : 1;
+  const chunks = [];
+  const chunkEndFrames = [];
+  let totalFrames = 0;
+  let ctx = null;
+  const scheduled = [];
+  let nextChunkIndex = 0;
+  let nextPlayTime = 0;
+  let playing = false;
+  let ended = false;
+  let destroyed = false;
+  let pausedAtFrame = 0;
+  let anchorCtxTime = 0;
+  let anchorFrame = 0;
+  let updateTimer = null;
+  const updateCallbacks = /* @__PURE__ */ new Set();
+  function notify() {
+    for (const cb of updateCallbacks) {
+      try {
+        cb();
+      } catch {
+      }
+    }
+  }
+  function startUpdateLoop() {
+    if (updateTimer !== null) return;
+    updateTimer = setInterval(() => {
+      notify();
+      if (!playing && updateTimer !== null) {
+        clearInterval(updateTimer);
+        updateTimer = null;
+      }
+    }, 250);
+  }
+  function framesToMs(frames) {
+    return frames / sampleRate * 1e3;
+  }
+  function getPositionFrames() {
+    if (!playing || !ctx) return pausedAtFrame;
+    const elapsed = Math.max(0, ctx.currentTime - anchorCtxTime);
+    return Math.min(
+      anchorFrame + Math.round(elapsed * sampleRate),
+      totalFrames
+    );
+  }
+  function ensureContext() {
+    if (typeof window === "undefined") return null;
+    if (!ctx) {
+      const Ctor = window.AudioContext || window.webkitAudioContext;
+      if (!Ctor) return null;
+      ctx = new Ctor({ sampleRate });
+    }
+    if (ctx.state === "suspended") void ctx.resume();
+    return ctx;
+  }
+  function int16ToBuffer(c, pcm, fromFrame) {
+    const frames = pcm.length / channels - fromFrame;
+    if (frames <= 0) return null;
+    const buf = c.createBuffer(channels, frames, sampleRate);
+    for (let ch = 0; ch < channels; ch++) {
+      const data = buf.getChannelData(ch);
+      for (let i = 0; i < frames; i++) {
+        data[i] = (pcm[(fromFrame + i) * channels + ch] ?? 0) / 32768;
+      }
+    }
+    return buf;
+  }
+  function scheduleChunk(index, fromFrame) {
+    const pcm = chunks[index];
+    if (!ctx || !pcm) return;
+    const buf = int16ToBuffer(ctx, pcm, fromFrame);
+    if (!buf) return;
+    const src = ctx.createBufferSource();
+    src.buffer = buf;
+    src.connect(ctx.destination);
+    const now = ctx.currentTime;
+    const startAt = Math.max(now, nextPlayTime);
+    if (startAt > nextPlayTime && nextPlayTime > 0) {
+      anchorCtxTime += startAt - nextPlayTime;
+    }
+    src.start(startAt);
+    nextPlayTime = startAt + buf.duration;
+    scheduled.push(src);
+    src.onended = () => {
+      const i = scheduled.indexOf(src);
+      if (i !== -1) scheduled.splice(i, 1);
+      try {
+        src.disconnect();
+      } catch {
+      }
+      if (playing && scheduled.length === 0 && ended && nextChunkIndex >= chunks.length) {
+        pausedAtFrame = totalFrames;
+        playing = false;
+        notify();
+      }
+    };
+  }
+  function scheduleFrom(frame) {
+    if (!ctx) return;
+    let index = chunkEndFrames.findIndex((end2) => frame < end2);
+    if (index === -1) index = chunks.length;
+    const chunkStart = index === 0 ? 0 : chunkEndFrames[index - 1] ?? 0;
+    nextPlayTime = ctx.currentTime;
+    anchorCtxTime = ctx.currentTime;
+    anchorFrame = frame;
+    if (index < chunks.length) {
+      scheduleChunk(index, frame - chunkStart);
+      for (let i = index + 1; i < chunks.length; i++) scheduleChunk(i, 0);
+    }
+    nextChunkIndex = chunks.length;
+  }
+  function stopScheduled() {
+    for (const src of scheduled) {
+      try {
+        src.onended = null;
+        src.stop(0);
+        src.disconnect();
+      } catch {
+      }
+    }
+    scheduled.length = 0;
+  }
+  function enqueueBase64(b64) {
+    if (destroyed || ended) return;
+    let pcm;
+    try {
+      pcm = base64ToInt16(b64);
+    } catch {
+      return;
+    }
+    if (pcm.length === 0) return;
+    chunks.push(pcm);
+    totalFrames += pcm.length / channels;
+    chunkEndFrames.push(totalFrames);
+    if (playing && ctx) {
+      scheduleChunk(chunks.length - 1, 0);
+      nextChunkIndex = chunks.length;
+    }
+    notify();
+  }
+  function play() {
+    if (destroyed || playing) return;
+    const c = ensureContext();
+    if (!c) return;
+    if (ended && pausedAtFrame >= totalFrames) pausedAtFrame = 0;
+    playing = true;
+    scheduleFrom(pausedAtFrame);
+    startUpdateLoop();
+    notify();
+  }
+  function pause() {
+    if (!playing) return;
+    pausedAtFrame = getPositionFrames();
+    playing = false;
+    stopScheduled();
+    notify();
+  }
+  function seekMs(ms) {
+    const frame = Math.max(
+      0,
+      Math.min(Math.round(ms / 1e3 * sampleRate), totalFrames)
+    );
+    if (playing) {
+      stopScheduled();
+      scheduleFrom(frame);
+    } else {
+      pausedAtFrame = frame;
+    }
+    notify();
+  }
+  function end() {
+    ended = true;
+    notify();
+  }
+  function destroy() {
+    if (destroyed) return;
+    destroyed = true;
+    playing = false;
+    stopScheduled();
+    if (updateTimer !== null) {
+      clearInterval(updateTimer);
+      updateTimer = null;
+    }
+    updateCallbacks.clear();
+    chunks.length = 0;
+    chunkEndFrames.length = 0;
+    if (ctx) {
+      void ctx.close().catch(() => void 0);
+      ctx = null;
+    }
+  }
+  return {
+    enqueueBase64,
+    end,
+    play,
+    pause,
+    seekMs,
+    getPositionMs: () => framesToMs(getPositionFrames()),
+    getBufferedMs: () => framesToMs(totalFrames),
+    isPlaying: () => playing,
+    hasEnded: () => ended,
+    onUpdate: (cb) => {
+      updateCallbacks.add(cb);
+      return () => updateCallbacks.delete(cb);
+    },
+    destroy
+  };
+}
+
+// src/live-audio/mp3-player.ts
+function base64ToArrayBuffer(b64) {
+  const binary = atob(b64);
+  const bytes = new Uint8Array(binary.length);
+  for (let i = 0; i < binary.length; i++) {
+    bytes[i] = binary.charCodeAt(i);
+  }
+  return bytes.buffer;
+}
+function createStreamingMp3Player(options = {}) {
+  if (typeof window === "undefined" || typeof MediaSource === "undefined") {
+    console.info(
+      "[streaming-mp3] MediaSource is unavailable \u2014 live MP3 playback disabled; waiting for the canonical audio file"
+    );
+    return null;
+  }
+  const mimeType = options.mimeType || "audio/mpeg";
+  if (!MediaSource.isTypeSupported(mimeType)) {
+    console.info(
+      `[streaming-mp3] browser does not support MediaSource ${mimeType} \u2014 live playback disabled; waiting for the canonical audio file`
+    );
+    return null;
+  }
+  const mediaSource = new MediaSource();
+  const objectUrl = URL.createObjectURL(mediaSource);
+  const audio = document.createElement("audio");
+  audio.preload = "auto";
+  audio.hidden = true;
+  audio.setAttribute("aria-hidden", "true");
+  audio.src = objectUrl;
+  document.body.appendChild(audio);
+  const pending = [];
+  const callbacks = /* @__PURE__ */ new Set();
+  let sourceBuffer = null;
+  let streamEnded = false;
+  let destroyed = false;
+  let failed = false;
+  function notify() {
+    for (const callback of callbacks) callback();
+  }
+  function fail(message, cause) {
+    if (failed || destroyed) return;
+    failed = true;
+    const error = cause instanceof Error ? new Error(message, { cause }) : new Error(message);
+    console.error(`[streaming-mp3] ${message}`, cause ?? "");
+    options.onError?.(error);
+    notify();
+  }
+  function finishIfReady() {
+    if (!destroyed && !failed && streamEnded && pending.length === 0 && sourceBuffer && !sourceBuffer.updating && mediaSource.readyState === "open") {
+      try {
+        mediaSource.endOfStream();
+      } catch (error) {
+        fail("could not finalize the MediaSource stream", error);
+      }
+    }
+  }
+  function appendNext() {
+    if (destroyed || failed || !sourceBuffer || sourceBuffer.updating || mediaSource.readyState !== "open") {
+      return;
+    }
+    const next = pending.shift();
+    if (!next) {
+      finishIfReady();
+      return;
+    }
+    try {
+      sourceBuffer.appendBuffer(next);
+    } catch (error) {
+      fail("could not append an MP3 stream chunk", error);
+    }
+  }
+  mediaSource.addEventListener(
+    "sourceopen",
+    () => {
+      if (destroyed || failed || sourceBuffer) return;
+      try {
+        sourceBuffer = mediaSource.addSourceBuffer(mimeType);
+        sourceBuffer.mode = "sequence";
+        sourceBuffer.addEventListener("updateend", () => {
+          notify();
+          appendNext();
+        });
+        sourceBuffer.addEventListener("error", () => {
+          fail("MediaSource rejected the MP3 stream");
+        });
+        appendNext();
+      } catch (error) {
+        fail("could not initialize an MP3 SourceBuffer", error);
+      }
+    },
+    { once: true }
+  );
+  for (const event of ["timeupdate", "progress", "play", "pause", "ended"]) {
+    audio.addEventListener(event, notify);
+  }
+  audio.addEventListener("error", () => {
+    fail(
+      `browser could not decode the live MP3 stream${audio.error?.message ? `: ${audio.error.message}` : ""}`
+    );
+  });
+  function getBufferedMs() {
+    try {
+      const ranges = audio.buffered;
+      return ranges.length > 0 ? ranges.end(ranges.length - 1) * 1e3 : 0;
+    } catch {
+      return 0;
+    }
+  }
+  return {
+    enqueueBase64: (b64) => {
+      if (destroyed || failed || streamEnded) return;
+      try {
+        const bytes = base64ToArrayBuffer(b64);
+        if (bytes.byteLength === 0) return;
+        pending.push(bytes);
+        appendNext();
+        notify();
+      } catch (error) {
+        fail("received a malformed base64 MP3 chunk", error);
+      }
+    },
+    end: () => {
+      if (destroyed || failed) return;
+      streamEnded = true;
+      finishIfReady();
+      notify();
+    },
+    play: () => {
+      if (destroyed || failed) return;
+      void audio.play().catch((error) => {
+        fail("browser refused live MP3 playback", error);
+      });
+    },
+    pause: () => audio.pause(),
+    seekMs: (ms) => {
+      if (destroyed || failed) return;
+      const clampedMs = Math.max(0, Math.min(ms, getBufferedMs()));
+      audio.currentTime = clampedMs / 1e3;
+      notify();
+    },
+    getPositionMs: () => Number.isFinite(audio.currentTime) ? audio.currentTime * 1e3 : 0,
+    getBufferedMs,
+    isPlaying: () => !audio.paused && !audio.ended,
+    hasEnded: () => streamEnded,
+    onUpdate: (callback) => {
+      callbacks.add(callback);
+      return () => callbacks.delete(callback);
+    },
+    destroy: () => {
+      if (destroyed) return;
+      destroyed = true;
+      audio.pause();
+      pending.length = 0;
+      callbacks.clear();
+      if (sourceBuffer?.updating) {
+        try {
+          sourceBuffer.abort();
+        } catch {
+        }
+      }
+      audio.removeAttribute("src");
+      audio.load();
+      audio.remove();
+      URL.revokeObjectURL(objectUrl);
+      sourceBuffer = null;
+    }
+  };
+}
+
+// src/live-audio/controller.ts
+var LIVE_AUDIO_CHUNK = "audio_stream_chunk";
+var LIVE_AUDIO_END = "audio_stream_end";
+function isRecord(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function isLiveAudioEvent(data) {
+  return isRecord(data) && (data.type === LIVE_AUDIO_CHUNK || data.type === LIVE_AUDIO_END);
+}
+function liveAudioEncodingOf(chunk) {
+  const mime = chunk.mime_type ?? "";
+  if (chunk.encoding === "mp3" || /(?:mpeg|mp3)/i.test(mime)) return "mp3";
+  if (chunk.encoding === "pcm_s16le" || /(?:l16|pcm)/i.test(mime)) return "pcm_s16le";
+  return null;
+}
+function createLiveAudioController(options) {
+  const makePcm = options.createPcmPlayer ?? createStreamingPcmPlayer;
+  const makeMp3 = options.createMp3Player ?? createStreamingMp3Player;
+  let player = null;
+  let expectedSeq = 0;
+  let streamId = null;
+  let encoding = null;
+  let broken = false;
+  function clearPlayer() {
+    player?.destroy();
+    player = null;
+    options.onPlayer(null);
+  }
+  function drop(reason) {
+    const message = `${reason} \u2014 dropping live playback; the saved file takes over when rendering finishes`;
+    if (options.onDrop) options.onDrop(message);
+    else console.warn(`[live-audio] ${message}`);
+    broken = true;
+    clearPlayer();
+  }
+  function onChunk(e) {
+    if (broken) return;
+    const codec = liveAudioEncodingOf(e);
+    if (!codec) {
+      drop(`unsupported audio stream format ${e.encoding || e.mime_type}`);
+      return;
+    }
+    if (streamId !== null && (streamId !== e.stream_id || encoding !== codec)) {
+      drop(
+        `audio stream identity/format changed mid-render (${streamId}/${encoding} \u2192 ${e.stream_id}/${codec})`
+      );
+      return;
+    }
+    if (e.seq !== expectedSeq) {
+      drop(`audio stream gap (expected seq ${expectedSeq}, got ${e.seq})`);
+      return;
+    }
+    streamId = e.stream_id;
+    encoding = codec;
+    expectedSeq = e.seq + 1;
+    if (!player) {
+      const created = codec === "mp3" ? makeMp3({
+        mimeType: e.mime_type || "audio/mpeg",
+        onError: (error) => drop(error.message)
+      }) : makePcm({
+        sampleRate: e.sample_rate || 24e3,
+        channels: e.channels || 1
+      });
+      if (!created) {
+        drop(`no browser player is available for ${codec}`);
+        return;
+      }
+      player = created;
+      options.onPlayer(created);
+    }
+    player.enqueueBase64(e.audio_base64);
+  }
+  return {
+    handle(data) {
+      if (!isLiveAudioEvent(data)) return false;
+      if (data.type === LIVE_AUDIO_CHUNK) {
+        onChunk(data);
+        return true;
+      }
+      player?.end();
+      options.onEnd?.(data);
+      return true;
+    },
+    reset() {
+      expectedSeq = 0;
+      streamId = null;
+      encoding = null;
+      broken = false;
+      if (player) clearPlayer();
+    },
+    destroy() {
+      player?.destroy();
+      player = null;
+    },
+    get player() {
+      return player;
+    }
+  };
+}
+function takeLiveAudioHandoff(player) {
+  const resume = player.isPlaying();
+  const timeSec = player.getPositionMs() / 1e3;
+  player.pause();
+  return timeSec > 0 ? { timeSec, resume } : null;
+}
+
+// src/live-audio/registry.ts
+function createLiveAudioRegistry(options = {}) {
+  const controllers = /* @__PURE__ */ new Map();
+  const players = /* @__PURE__ */ new Map();
+  const listeners = /* @__PURE__ */ new Set();
+  const notify = () => {
+    for (const listener of listeners) {
+      try {
+        listener();
+      } catch {
+      }
+    }
+  };
+  const controllerFor = (key) => {
+    let controller = controllers.get(key);
+    if (!controller) {
+      controller = createLiveAudioController({
+        onPlayer: (player) => {
+          if (player) players.set(key, player);
+          else players.delete(key);
+          notify();
+        },
+        onEnd: (event) => options.onEnd?.(key, event),
+        onDrop: (reason) => options.onDrop ? options.onDrop(key, reason) : console.warn(`[live-audio] ${key}: ${reason}`),
+        ...options.createPcmPlayer ? { createPcmPlayer: options.createPcmPlayer } : {},
+        ...options.createMp3Player ? { createMp3Player: options.createMp3Player } : {}
+      });
+      controllers.set(key, controller);
+    }
+    return controller;
+  };
+  return {
+    handle(key, payload) {
+      if (!isLiveAudioEvent(payload)) return false;
+      return controllerFor(key).handle(payload);
+    },
+    player: (key) => players.get(key) ?? null,
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    reset(key) {
+      controllers.get(key)?.reset();
+    },
+    destroy() {
+      for (const controller of controllers.values()) controller.destroy();
+      controllers.clear();
+      const had = players.size > 0;
+      players.clear();
+      if (had) notify();
+    }
+  };
+}
+
+// src/live-audio/media-frames.ts
+function mediaFrameSourceKey(source) {
+  return `${source.nodeId ?? ""}|${source.dispatchId ?? ""}|${source.itemIndex ?? ""}`;
+}
+var MAX_OPEN_MEDIA_FRAME_SETS = 32;
+function isRecord2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function createMediaFrameAssembler(options) {
+  const open = /* @__PURE__ */ new Map();
+  const ready = [];
+  let arrival = 0;
+  const drop = (frameId, reason) => {
+    open.delete(frameId);
+    if (options.onDrop) options.onDrop(frameId, reason);
+    else console.warn(`[media-frames] dropped ${frameId}: ${reason}`);
+  };
+  const release = () => {
+    if (ready.length === 0) return;
+    ready.sort((a, b) => a.streamSeq - b.streamSeq);
+    const blocking = /* @__PURE__ */ new Map();
+    for (const set of open.values()) {
+      blocking.set(set.key, Math.min(blocking.get(set.key) ?? Infinity, set.firstSeq));
+    }
+    for (let i = 0; i < ready.length; ) {
+      const item = ready[i];
+      const key = mediaFrameSourceKey(item.source);
+      if (item.streamSeq < (blocking.get(key) ?? Infinity)) {
+        ready.splice(i, 1);
+        options.onPayload(item);
+      } else {
+        i += 1;
+      }
+    }
+  };
+  return {
+    push(frame) {
+      if (frame.kind !== "media") return false;
+      const frameId = frame.frame_id;
+      const count = frame.frame_count ?? 1;
+      const index = frame.frame_index ?? 0;
+      if (!frameId || !Number.isInteger(count) || count < 1 || !Number.isInteger(index) || index < 0 || index >= count) {
+        if (options.onDrop) options.onDrop(frameId ?? "?", "malformed frame");
+        else console.warn("[media-frames] malformed media frame dropped");
+        return true;
+      }
+      arrival += 1;
+      const seq = typeof frame.stream_seq === "number" && Number.isFinite(frame.stream_seq) ? frame.stream_seq : arrival;
+      let set = open.get(frameId);
+      if (!set) {
+        if (open.size >= MAX_OPEN_MEDIA_FRAME_SETS) {
+          const oldest = open.keys().next();
+          if (!oldest.done) drop(oldest.value, "abandoned (too many open sets)");
+        }
+        const source = {
+          nodeId: frame.node_id ?? null,
+          dispatchId: frame.dispatch_id ?? null,
+          itemIndex: frame.item_index ?? null
+        };
+        set = {
+          key: mediaFrameSourceKey(source),
+          count,
+          slices: /* @__PURE__ */ new Map(),
+          firstSeq: seq,
+          source
+        };
+        open.set(frameId, set);
+      }
+      set.slices.set(index, frame.delta);
+      if (index === 0) set.firstSeq = Math.min(set.firstSeq, seq);
+      if (set.slices.size < set.count) {
+        release();
+        return true;
+      }
+      open.delete(frameId);
+      let body = "";
+      for (let i = 0; i < set.count; i++) body += set.slices.get(i) ?? "";
+      let parsed;
+      try {
+        parsed = JSON.parse(body);
+      } catch {
+        if (options.onDrop) options.onDrop(frameId, "unparseable payload");
+        else console.warn(`[media-frames] dropped ${frameId}: unparseable payload`);
+        release();
+        return true;
+      }
+      if (!isRecord2(parsed) || typeof parsed.type !== "string") {
+        if (options.onDrop) options.onDrop(frameId, "payload has no type");
+        else console.warn(`[media-frames] dropped ${frameId}: payload has no type`);
+        release();
+        return true;
+      }
+      for (const [id, other] of [...open.entries()]) {
+        if (other.key === set.key && other.firstSeq < set.firstSeq) {
+          drop(id, "incomplete set");
+        }
+      }
+      ready.push({ payload: parsed, source: set.source, streamSeq: set.firstSeq });
+      release();
+      return true;
+    },
+    flush() {
+      for (const id of [...open.keys()]) drop(id, "incomplete at stream end");
+      release();
+    },
+    get openCount() {
+      return open.size;
+    }
+  };
+}
+export {
+  LIVE_AUDIO_CHUNK,
+  LIVE_AUDIO_END,
+  MAX_OPEN_MEDIA_FRAME_SETS,
+  createLiveAudioController,
+  createLiveAudioRegistry,
+  createMediaFrameAssembler,
+  createStreamingMp3Player,
+  createStreamingPcmPlayer,
+  isLiveAudioEvent,
+  liveAudioEncodingOf,
+  mediaFrameSourceKey,
+  takeLiveAudioHandoff
+};
+//# sourceMappingURL=live-audio.js.map