image-to-pixel 3.0.0 → 3.1.0

package/lib/core/source.js

@@ -1,89 +1,89 @@
-// Turning whatever the caller has into a "frame": { width, height, canvas?, data }.
-// Accepts URLs, Blobs/Files, <img>, <canvas>, OffscreenCanvas, ImageBitmap, <video> (current frame),
-// VideoFrame, ImageData or any { width, height, data } RGBA object, plus p5 / q5 images.
-
-export const MAX_SOURCE = 4096;
-const hasDOM = typeof document !== 'undefined';
-
-const isRGBA = s => s && typeof s === 'object' && 'data' in s && 'width' in s && 'height' in s && s.data && s.data.length === s.width * s.height * 4;
-
-/** Load a URL or Blob into something drawable. */
-export async function loadImage(src) {
-  if (typeof Blob !== 'undefined' && src instanceof Blob) {
-    if (typeof createImageBitmap === 'function') {
-      try { return await createImageBitmap(src); } catch { /* fall through (e.g. SVG) */ }
-    }
-    src = URL.createObjectURL(src);
-  }
-  if (!hasDOM) throw new Error('Loading images by URL needs a browser; pass { width, height, data } in Node.');
-  const img = new Image();
-  img.crossOrigin = 'anonymous';
-  img.decoding = 'async';
-  img.src = src;
-  await img.decode();
-  return img;
-}
-
-/** Natural pixel size of a drawable. */
-export function sizeOf(el) {
-  return {
-    width: el.naturalWidth || el.videoWidth || el.displayWidth || el.codedWidth || el.width || 0,
-    height: el.naturalHeight || el.videoHeight || el.displayHeight || el.codedHeight || el.height || 0,
-  };
-}
-
-function unwrap(src) {
-  // p5.Image / q5 image / p5.Graphics / p5.Renderer
-  if (src && !isRGBA(src) && typeof src === 'object') {
-    if (src.canvas && typeof src.canvas === 'object' && 'getContext' in src.canvas) return src.canvas;
-    if (src.elt && typeof src.elt === 'object' && 'getContext' in src.elt) return src.elt;
-  }
-  return src;
-}
-
-/**
- * @param {any} source
- * @param {{ maxSize?: number, canvas?: HTMLCanvasElement }} [opts] canvas: scratch canvas to reuse
- */
-export async function toFrame(source, opts = {}) {
-  const maxSize = opts.maxSize || MAX_SOURCE;
-  let src = unwrap(source);
-  if (typeof src === 'string' || (typeof Blob !== 'undefined' && src instanceof Blob)) src = await loadImage(src);
-  if (src && typeof src.complete === 'boolean' && !src.complete && src.decode) await src.decode();
-
-  if (isRGBA(src)) {
-    const frame = { width: src.width, height: src.height, data: src.data, canvas: null };
-    if (hasDOM) {
-      Object.defineProperty(frame, 'canvas', {
-        configurable: true,
-        get() {
-          const c = document.createElement('canvas'); c.width = src.width; c.height = src.height;
-          const img = new ImageData(new Uint8ClampedArray(src.data), src.width, src.height);
-          c.getContext('2d').putImageData(img, 0, 0);
-          Object.defineProperty(frame, 'canvas', { value: c });
-          return c;
-        },
-      });
-    }
-    return frame;
-  }
-  if (!hasDOM) throw new Error('Unsupported source in a non-browser environment. Pass { width, height, data }.');
-
-  let { width, height } = sizeOf(src);
-  if (!width || !height) throw new Error('Source has no size yet (is the image or video loaded?)');
-  const k = Math.min(1, maxSize / Math.max(width, height));
-  width = Math.max(1, Math.round(width * k)); height = Math.max(1, Math.round(height * k));
-
-  const canvas = opts.canvas || document.createElement('canvas');
-  canvas.width = width; canvas.height = height;
-  const ctx = canvas.getContext('2d', { willReadFrequently: true });
-  ctx.clearRect(0, 0, width, height);
-  ctx.imageSmoothingEnabled = k < 1;
-  ctx.imageSmoothingQuality = 'high';
-  ctx.drawImage(src, 0, 0, width, height);
-  let cached = null;
-  return {
-    width, height, canvas, dirty: true,
-    get data() { return (cached ||= ctx.getImageData(0, 0, width, height).data); },
-  };
-}
+// Turning whatever the caller has into a "frame": { width, height, canvas?, data }.
+// Accepts URLs, Blobs/Files, <img>, <canvas>, OffscreenCanvas, ImageBitmap, <video> (current frame),
+// VideoFrame, ImageData or any { width, height, data } RGBA object, plus p5 / q5 images.
+
+export const MAX_SOURCE = 4096;
+const hasDOM = typeof document !== 'undefined';
+
+const isRGBA = s => s && typeof s === 'object' && 'data' in s && 'width' in s && 'height' in s && s.data && s.data.length === s.width * s.height * 4;
+
+/** Load a URL or Blob into something drawable. */
+export async function loadImage(src) {
+  if (typeof Blob !== 'undefined' && src instanceof Blob) {
+    if (typeof createImageBitmap === 'function') {
+      try { return await createImageBitmap(src); } catch { /* fall through (e.g. SVG) */ }
+    }
+    src = URL.createObjectURL(src);
+  }
+  if (!hasDOM) throw new Error('Loading images by URL needs a browser; pass { width, height, data } in Node.');
+  const img = new Image();
+  img.crossOrigin = 'anonymous';
+  img.decoding = 'async';
+  img.src = src;
+  await img.decode();
+  return img;
+}
+
+/** Natural pixel size of a drawable. */
+export function sizeOf(el) {
+  return {
+    width: el.naturalWidth || el.videoWidth || el.displayWidth || el.codedWidth || el.width || 0,
+    height: el.naturalHeight || el.videoHeight || el.displayHeight || el.codedHeight || el.height || 0,
+  };
+}
+
+function unwrap(src) {
+  // p5.Image / q5 image / p5.Graphics / p5.Renderer
+  if (src && !isRGBA(src) && typeof src === 'object') {
+    if (src.canvas && typeof src.canvas === 'object' && 'getContext' in src.canvas) return src.canvas;
+    if (src.elt && typeof src.elt === 'object' && 'getContext' in src.elt) return src.elt;
+  }
+  return src;
+}
+
+/**
+ * @param {any} source
+ * @param {{ maxSize?: number, canvas?: HTMLCanvasElement }} [opts] canvas: scratch canvas to reuse
+ */
+export async function toFrame(source, opts = {}) {
+  const maxSize = opts.maxSize || MAX_SOURCE;
+  let src = unwrap(source);
+  if (typeof src === 'string' || (typeof Blob !== 'undefined' && src instanceof Blob)) src = await loadImage(src);
+  if (src && typeof src.complete === 'boolean' && !src.complete && src.decode) await src.decode();
+
+  if (isRGBA(src)) {
+    const frame = { width: src.width, height: src.height, data: src.data, canvas: null };
+    if (hasDOM) {
+      Object.defineProperty(frame, 'canvas', {
+        configurable: true,
+        get() {
+          const c = document.createElement('canvas'); c.width = src.width; c.height = src.height;
+          const img = new ImageData(new Uint8ClampedArray(src.data), src.width, src.height);
+          c.getContext('2d').putImageData(img, 0, 0);
+          Object.defineProperty(frame, 'canvas', { value: c });
+          return c;
+        },
+      });
+    }
+    return frame;
+  }
+  if (!hasDOM) throw new Error('Unsupported source in a non-browser environment. Pass { width, height, data }.');
+
+  let { width, height } = sizeOf(src);
+  if (!width || !height) throw new Error('Source has no size yet (is the image or video loaded?)');
+  const k = Math.min(1, maxSize / Math.max(width, height));
+  width = Math.max(1, Math.round(width * k)); height = Math.max(1, Math.round(height * k));
+
+  const canvas = opts.canvas || document.createElement('canvas');
+  canvas.width = width; canvas.height = height;
+  const ctx = canvas.getContext('2d', { willReadFrequently: true });
+  ctx.clearRect(0, 0, width, height);
+  ctx.imageSmoothingEnabled = k < 1;
+  ctx.imageSmoothingQuality = 'high';
+  ctx.drawImage(src, 0, 0, width, height);
+  let cached = null;
+  return {
+    width, height, canvas, dirty: true,
+    get data() { return (cached ||= ctx.getImageData(0, 0, width, height).data); },
+  };
+}

package/lib/core/video/audio.js

@@ -0,0 +1,159 @@
+// Audio for video export: read a source file's soundtrack as PCM, and encode PCM with WebCodecs.
+// The muxers (mux-mp4.js, mux-webm.js) take the encoded chunks as a second track next to the video.
+
+/** WebCodecs codec strings for the audio formats the muxers can store. */
+export const AUDIO_CODECS = { aac: 'mp4a.40.2', opus: 'opus' };
+
+const DEFAULT_RATE = 48000;   // Opus only encodes at 48 kHz (and lower); AAC is fine with it too
+
+async function toArrayBuffer(source) {
+  if (source instanceof ArrayBuffer) return source.slice(0);
+  if (ArrayBuffer.isView(source)) return source.buffer.slice(source.byteOffset, source.byteOffset + source.byteLength);
+  if (typeof Blob !== 'undefined' && source instanceof Blob) return source.arrayBuffer();
+  const url = typeof source === 'string' ? source : source && (source.currentSrc || source.src);
+  if (!url) throw new TypeError('decodeAudio() takes a File, Blob, ArrayBuffer, URL or <video>/<audio> element');
+  const res = await fetch(url);
+  if (!res.ok) throw new Error(`Couldn't load the audio: ${res.status} ${res.statusText}`);
+  return res.arrayBuffer();
+}
+
+/**
+ * Decode the audio track of a video (or audio) file into PCM, resampled and trimmed.
+ * Resolves with null when there is no audio track, it can't be decoded, or Web Audio isn't available (Node).
+ * More than two channels are mixed down to stereo with the standard Web Audio speaker rules.
+ * @param {File|Blob|ArrayBuffer|ArrayBufferView|string|HTMLMediaElement} source
+ * @param {{ start?: number, end?: number, sampleRate?: number, channels?: 1 | 2 }} [opts] start/end in seconds
+ * @returns {Promise<null | { sampleRate: number, numberOfChannels: number, channels: Float32Array[], duration: number }>}
+ */
+export async function decodeAudio(source, opts = {}) {
+  if (typeof OfflineAudioContext === 'undefined') return null;
+  const sampleRate = opts.sampleRate || DEFAULT_RATE;
+  const bytes = await toArrayBuffer(source);
+  let decoded;
+  try {
+    decoded = await new OfflineAudioContext(1, 1, sampleRate).decodeAudioData(bytes);
+  } catch (_) {
+    return null;                                   // no audio track, or a codec the browser can't decode
+  }
+  const start = Math.max(0, opts.start || 0);
+  const end = Math.min(decoded.duration, opts.end > start ? opts.end : decoded.duration);
+  const length = Math.floor((end - start) * sampleRate);
+  if (length < 1) return null;
+  const numberOfChannels = Math.min(opts.channels || 2, decoded.numberOfChannels);
+  // Rendering the decoded buffer through an offline graph trims it and mixes 5.1 etc. down properly.
+  const ctx = new OfflineAudioContext(numberOfChannels, length, sampleRate);
+  const node = ctx.createBufferSource();
+  node.buffer = decoded;
+  node.connect(ctx.destination);
+  node.start(0, start, length / sampleRate);
+  const out = await ctx.startRendering();
+  const channels = [];
+  for (let c = 0; c < numberOfChannels; c++) channels.push(out.getChannelData(c));
+  return { sampleRate, numberOfChannels, channels, duration: length / sampleRate };
+}
+
+/**
+ * Find an AudioEncoder configuration this browser accepts for `codec`, or null.
+ * Platform AAC encoders (Windows Media Foundation, macOS AudioToolbox) are picky about sample rate and
+ * bitrate, so a few common combinations are tried: the requested rate first, then 48 and 44.1 kHz.
+ * @returns {Promise<AudioEncoderConfig | null>} the config to pass to encodeAudio({ config })
+ */
+export async function findAudioConfig(codec, sampleRate = DEFAULT_RATE, numberOfChannels = 2, bitrate = 128000) {
+  if (typeof AudioEncoder === 'undefined' || !AUDIO_CODECS[codec]) return null;
+  const rates = codec === 'opus' ? [48000] : [...new Set([sampleRate, 48000, 44100])];
+  const bitrates = [...new Set([bitrate, 128000, 192000, 160000, 96000, 0])];
+  for (const rate of rates) {
+    for (const br of bitrates) {
+      const config = {
+        codec: AUDIO_CODECS[codec], sampleRate: rate, numberOfChannels,
+        ...(br ? { bitrate: br } : {}),
+        ...(codec === 'aac' ? { aac: { format: 'aac' } } : {}),   // raw frames + AudioSpecificConfig, which MP4 needs
+      };
+      const r = await AudioEncoder.isConfigSupported(config).catch(() => ({}));
+      if (r.supported) return config;
+    }
+  }
+  return null;
+}
+
+/** Can this browser encode `codec` ('aac' | 'opus') with these parameters (or a close variant)? */
+export async function audioCodecSupported(codec, sampleRate = DEFAULT_RATE, numberOfChannels = 2) {
+  return !!(await findAudioConfig(codec, sampleRate, numberOfChannels));
+}
+
+/** Resample PCM with Web Audio (browser only). */
+export async function resamplePcm(pcm, sampleRate) {
+  if (pcm.sampleRate === sampleRate) return pcm;
+  const { numberOfChannels, channels } = pcm;
+  const length = Math.max(1, Math.round(channels[0].length * sampleRate / pcm.sampleRate));
+  const ctx = new OfflineAudioContext(numberOfChannels, length, sampleRate);
+  const buf = ctx.createBuffer(numberOfChannels, channels[0].length, pcm.sampleRate);
+  for (let c = 0; c < numberOfChannels; c++) buf.copyToChannel(channels[c], c);
+  const node = ctx.createBufferSource();
+  node.buffer = buf;
+  node.connect(ctx.destination);
+  node.start();
+  const out = await ctx.startRendering();
+  const res = [];
+  for (let c = 0; c < numberOfChannels; c++) res.push(out.getChannelData(c));
+  return { sampleRate, numberOfChannels, channels: res, duration: length / sampleRate };
+}
+
+/**
+ * Encode PCM (as returned by decodeAudio) with WebCodecs' AudioEncoder.
+ * @param {{ sampleRate: number, numberOfChannels: number, channels: Float32Array[] }} pcm
+ * @param {{ codec?: 'aac' | 'opus', config?: AudioEncoderConfig, bitrate?: number, maxDuration?: number,
+ *           onChunk: (chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) => void }} opts
+ *   config: from findAudioConfig(); otherwise built from codec + bitrate. PCM at a different rate is resampled.
+ *   maxDuration: seconds; longer audio is cut (used to match the video's length)
+ */
+export async function encodeAudio(pcm, opts) {
+  if (typeof AudioEncoder === 'undefined') throw new Error('This browser has no WebCodecs AudioEncoder');
+  const config = opts.config || {
+    codec: AUDIO_CODECS[opts.codec], sampleRate: pcm.sampleRate, numberOfChannels: pcm.numberOfChannels,
+    bitrate: opts.bitrate || 128000,
+    ...(opts.codec === 'aac' ? { aac: { format: 'aac' } } : {}),
+  };
+  if (config.sampleRate !== pcm.sampleRate) pcm = await resamplePcm(pcm, config.sampleRate);
+  const { sampleRate, numberOfChannels, channels } = pcm;
+  let error = null;
+  const encoder = new AudioEncoder({ output: (chunk, meta) => opts.onChunk(chunk, meta), error: e => { error = e; } });
+  encoder.configure(config);
+  const limit = Number.isFinite(opts.maxDuration) ? Math.floor(opts.maxDuration * sampleRate) : Infinity;
+  const total = Math.min(channels[0].length, limit);
+  const BLOCK = Math.round(sampleRate / 10);        // 100 ms per AudioData
+  for (let o = 0; o < total; o += BLOCK) {
+    if (error) break;
+    const n = Math.min(BLOCK, total - o);
+    const data = new Float32Array(n * numberOfChannels);
+    for (let c = 0; c < numberOfChannels; c++) data.set(channels[c].subarray(o, o + n), c * n);
+    const frame = new AudioData({ format: 'f32-planar', sampleRate, numberOfFrames: n, numberOfChannels, timestamp: Math.round(o * 1e6 / sampleRate), data });
+    encoder.encode(frame);
+    frame.close();
+    if (encoder.encodeQueueSize > 16) await new Promise(r => setTimeout(r, 0));
+  }
+  if (!error) await encoder.flush().catch(e => { error = e; });
+  if (encoder.state !== 'closed') encoder.close();
+  if (error) throw error;
+}
+
+/** OpusHead (RFC 7845), little-endian: what Matroska's CodecPrivate holds for Opus. */
+export function opusHead(numberOfChannels, sampleRate, preSkip = 312) {
+  const b = new Uint8Array(19);
+  b.set([0x4f, 0x70, 0x75, 0x73, 0x48, 0x65, 0x61, 0x64]);   // 'OpusHead'
+  const dv = new DataView(b.buffer);
+  dv.setUint8(8, 1);
+  dv.setUint8(9, numberOfChannels);
+  dv.setUint16(10, preSkip, true);
+  dv.setUint32(12, sampleRate, true);
+  dv.setInt16(16, 0, true);
+  dv.setUint8(18, 0);
+  return b;
+}
+
+/** Read { channels, preSkip, sampleRate } from an OpusHead, or null if it isn't one. */
+export function parseOpusHead(b) {
+  if (!b || b.length < 19 || String.fromCharCode(...b.subarray(0, 8)) !== 'OpusHead') return null;
+  const dv = new DataView(b.buffer, b.byteOffset, b.byteLength);
+  return { channels: dv.getUint8(9), preSkip: dv.getUint16(10, true), sampleRate: dv.getUint32(12, true) };
+}

package/lib/core/video/frames.js

@@ -67,6 +67,44 @@ export async function sniffVideoCodec(file) {
   return '';
 }
 
+/**
+ * Does this MP4/MOV/WebM file have an audio track? Reads only the container headers: for MP4/MOV the
+ * top-level boxes are walked to 'moov' (which phones often write at the end) and only that box is read.
+ * @param {Blob} file
+ * @returns {Promise<boolean | null>} null when it can't tell
+ */
+export async function hasAudioTrack(file) {
+  try {
+    const head = new Uint8Array(await file.slice(0, 12).arrayBuffer());
+    if (head[0] === 0x1a && head[1] === 0x45 && head[2] === 0xdf && head[3] === 0xa3) {
+      // WebM/Matroska: Tracks sit near the start; audio CodecIDs all begin with A_
+      const text = new TextDecoder('latin1').decode(await file.slice(0, 4 << 20).arrayBuffer());
+      return /A_(OPUS|VORBIS|AAC|MPEG|AC3|EAC3|FLAC|PCM|DTS|TRUEHD|ALAC)/.test(text);
+    }
+    for (let o = 0; o + 8 <= file.size;) {
+      const dv = new DataView(await file.slice(o, o + 16).arrayBuffer());
+      let size = dv.getUint32(0);
+      const type = String.fromCharCode(dv.getUint8(4), dv.getUint8(5), dv.getUint8(6), dv.getUint8(7));
+      if (size === 1) size = Number(dv.getBigUint64(8));
+      else if (size === 0) size = file.size - o;
+      if (size < 8) return null;
+      if (type === 'moov') {
+        const moov = new Uint8Array(await file.slice(o, o + size).arrayBuffer());
+        // each track's 'hdlr' box names its kind 12 bytes after the box type: 'vide', 'soun', …
+        for (let i = 4; i + 16 <= moov.length; i++) {
+          if (moov[i] === 0x68 && moov[i + 1] === 0x64 && moov[i + 2] === 0x6c && moov[i + 3] === 0x72
+            && moov[i + 12] === 0x73 && moov[i + 13] === 0x6f && moov[i + 14] === 0x75 && moov[i + 15] === 0x6e) return true;
+        }
+        return false;
+      }
+      o += size;
+    }
+    return null;
+  } catch (_) {
+    return null;
+  }
+}
+
 /** Seek and resolve once the frame at `time` is decoded and paintable. */
 export async function seekTo(video, time) {
   time = Math.max(0, Math.min(time, (video.duration || 0) - 1e-4));