@talmolab/sleap-io.js 0.1.5 → 0.1.7

package/dist/index.d.ts

@@ -342,6 +342,43 @@ declare function isStreamingSupported(): boolean;
  */
 declare function openStreamingH5(url: string, options?: StreamingH5Options): Promise<StreamingH5File>;
 
+/**
+ * Video backend for embedded images in HDF5 files accessed via streaming.
+ *
+ * This backend uses StreamingH5File (Web Worker + range requests) instead of
+ * a synchronous h5wasm File object, making it suitable for browser environments
+ * where the SLP file is loaded via HTTP range requests.
+ *
+ * Supports two data storage formats:
+ * 1. vlen-encoded: Array of individual frame blobs (each index = one frame)
+ * 2. Contiguous buffer: Single buffer with all frames concatenated
+ */
+declare class StreamingHdf5VideoBackend implements VideoBackend {
+    filename: string;
+    dataset?: string | null;
+    shape?: [number, number, number, number];
+    fps?: number;
+    private h5file;
+    private datasetPath;
+    private frameNumbers;
+    private format;
+    private channelOrder;
+    private cachedData;
+    private frameOffsets;
+    constructor(options: {
+        filename: string;
+        h5file: StreamingH5File;
+        datasetPath: string;
+        frameNumbers?: number[];
+        format?: string;
+        channelOrder?: string;
+        shape?: [number, number, number, number];
+        fps?: number;
+    });
+    getFrame(frameIndex: number): Promise<VideoFrame | null>;
+    close(): void;
+}
+
 type SlpSource = string | ArrayBuffer | Uint8Array | File | FileSystemFileHandle;
 type StreamMode = "auto" | "range" | "download";
 type OpenH5Options = {
@@ -702,12 +739,18 @@ interface StreamingSlpOptions {
     h5wasmUrl?: string;
     /** Filename hint for the HDF5 file */
     filenameHint?: string;
+    /** Whether to open video backends for embedded videos (default: false) */
+    openVideos?: boolean;
 }
 /**
  * Read an SLP file using HTTP range requests for efficient streaming.
  *
  * This function downloads only the data needed (metadata, frames, instances, points)
- * rather than the entire file. Embedded videos are NOT loaded - only metadata.
+ * rather than the entire file.
+ *
+ * When `openVideos` is true, video backends are created for embedded videos,
+ * allowing frame data to be retrieved. The underlying HDF5 file remains open
+ * until all video backends are closed.
  *
  * @param url - URL to the SLP file (must support HTTP range requests)
  * @param options - Optional settings
@@ -715,10 +758,13 @@ interface StreamingSlpOptions {
  *
  * @example
  * ```typescript
- * const labels = await readSlpStreaming('https://example.com/labels.slp');
- * console.log(`Loaded ${labels.labeledFrames.length} frames`);
+ * // Load with video backends for embedded images
+ * const labels = await readSlpStreaming('https://example.com/labels.slp', {
+ *   openVideos: true
+ * });
+ * const frame = await labels.video.getFrame(0);
  * ```
  */
 declare function readSlpStreaming(url: string, options?: StreamingSlpOptions): Promise<Labels>;
 
-export { Camera, CameraGroup, type ColorScheme, type ColorSpec, FrameGroup, Instance, InstanceContext, InstanceGroup, LabeledFrame, Labels, type LabelsDict, LabelsSet, MARKER_FUNCTIONS, type MarkerShape, Mp4BoxVideoBackend, NAMED_COLORS, PALETTES, type PaletteName, PredictedInstance, type RGB, type RGBA, RecordingSession, RenderContext, type RenderOptions, Skeleton, StreamingH5File, SuggestionFrame, Track, Video, type VideoBackend, type VideoFrame, type VideoOptions, checkFfmpeg, decodeYamlSkeleton, determineColorScheme, drawCircle, drawCross, drawDiamond, drawSquare, drawTriangle, encodeYamlSkeleton, fromDict, fromNumpy, getMarkerFunction, getPalette, isStreamingSupported, labelsFromNumpy, loadSlp, loadVideo, makeCameraFromDict, openStreamingH5, readSlpStreaming, renderImage, renderVideo, resolveColor, rgbToCSS, rodriguesTransformation, saveImage, saveSlp, toDataURL, toDict, toJPEG, toNumpy, toPNG };
+export { Camera, CameraGroup, type ColorScheme, type ColorSpec, FrameGroup, Instance, InstanceContext, InstanceGroup, LabeledFrame, Labels, type LabelsDict, LabelsSet, MARKER_FUNCTIONS, type MarkerShape, Mp4BoxVideoBackend, NAMED_COLORS, PALETTES, type PaletteName, PredictedInstance, type RGB, type RGBA, RecordingSession, RenderContext, type RenderOptions, Skeleton, StreamingH5File, StreamingHdf5VideoBackend, SuggestionFrame, Track, Video, type VideoBackend, type VideoFrame, type VideoOptions, checkFfmpeg, decodeYamlSkeleton, determineColorScheme, drawCircle, drawCross, drawDiamond, drawSquare, drawTriangle, encodeYamlSkeleton, fromDict, fromNumpy, getMarkerFunction, getPalette, isStreamingSupported, labelsFromNumpy, loadSlp, loadVideo, makeCameraFromDict, openStreamingH5, readSlpStreaming, renderImage, renderVideo, resolveColor, rgbToCSS, rodriguesTransformation, saveImage, saveSlp, toDataURL, toDict, toJPEG, toNumpy, toPNG };

package/dist/index.js

@@ -1206,6 +1206,168 @@ var Mp4BoxVideoBackend = class {
   }
 };
 
+// src/video/streaming-hdf5-video.ts
+var isBrowser3 = typeof window !== "undefined" && typeof document !== "undefined";
+var PNG_MAGIC = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]);
+var JPEG_MAGIC = new Uint8Array([255, 216, 255]);
+var StreamingHdf5VideoBackend = class {
+  filename;
+  dataset;
+  shape;
+  fps;
+  h5file;
+  datasetPath;
+  frameNumbers;
+  format;
+  channelOrder;
+  cachedData;
+  frameOffsets;
+  // For contiguous buffer: byte offsets of each frame
+  constructor(options) {
+    this.filename = options.filename;
+    this.h5file = options.h5file;
+    this.datasetPath = options.datasetPath;
+    this.dataset = options.datasetPath;
+    this.frameNumbers = options.frameNumbers ?? [];
+    this.format = options.format ?? "png";
+    this.channelOrder = options.channelOrder ?? "RGB";
+    this.shape = options.shape;
+    this.fps = options.fps;
+    this.cachedData = null;
+    this.frameOffsets = null;
+  }
+  async getFrame(frameIndex) {
+    const index = this.frameNumbers.length ? this.frameNumbers.indexOf(frameIndex) : frameIndex;
+    if (index < 0) return null;
+    if (!this.cachedData) {
+      try {
+        const data = await this.h5file.getDatasetValue(this.datasetPath);
+        this.cachedData = normalizeVideoData(data.value, data.shape);
+        if (isContiguousEncodedBuffer(this.cachedData, this.format, this.shape)) {
+          this.frameOffsets = findEncodedFrameOffsets(
+            this.cachedData,
+            this.format,
+            this.shape?.[0] ?? 0
+          );
+        }
+      } catch {
+        return null;
+      }
+    }
+    let rawBytes;
+    if (this.frameOffsets && this.frameOffsets.length > index) {
+      const buffer = this.cachedData;
+      const start = this.frameOffsets[index];
+      const end = index + 1 < this.frameOffsets.length ? this.frameOffsets[index + 1] : buffer.length;
+      rawBytes = buffer.slice(start, end);
+    } else {
+      const entry = this.cachedData[index];
+      if (entry == null) return null;
+      rawBytes = toUint8Array(entry);
+    }
+    if (!rawBytes || rawBytes.length === 0) return null;
+    if (isEncodedFormat(this.format)) {
+      const decoded = await decodeImageBytes(rawBytes, this.format);
+      return decoded ?? rawBytes;
+    }
+    const image = decodeRawFrame(rawBytes, this.shape, this.channelOrder);
+    return image ?? rawBytes;
+  }
+  close() {
+    this.cachedData = null;
+    this.frameOffsets = null;
+  }
+};
+function normalizeVideoData(value, _shape) {
+  if (Array.isArray(value)) {
+    return value;
+  }
+  if (ArrayBuffer.isView(value)) {
+    const arr = value;
+    return new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
+  }
+  return [];
+}
+function isContiguousEncodedBuffer(data, format, shape) {
+  if (!isEncodedFormat(format)) return false;
+  if (!(data instanceof Uint8Array)) return false;
+  if (data.length < 8) return false;
+  const isPng = matchesMagic(data, PNG_MAGIC);
+  const isJpeg = matchesMagic(data, JPEG_MAGIC);
+  if (!isPng && !isJpeg) return false;
+  if (shape) {
+    const frameCount = shape[0];
+    if (frameCount > 1 && data.length > 1e4) {
+      return true;
+    }
+  }
+  return true;
+}
+function matchesMagic(buffer, magic) {
+  if (buffer.length < magic.length) return false;
+  for (let i = 0; i < magic.length; i++) {
+    if (buffer[i] !== magic[i]) return false;
+  }
+  return true;
+}
+function findEncodedFrameOffsets(buffer, format, expectedFrameCount) {
+  const offsets = [];
+  const magic = format.toLowerCase() === "png" ? PNG_MAGIC : JPEG_MAGIC;
+  for (let i = 0; i <= buffer.length - magic.length; i++) {
+    if (matchesMagic(buffer.subarray(i), magic)) {
+      offsets.push(i);
+      i += magic.length - 1;
+      if (expectedFrameCount > 0 && offsets.length >= expectedFrameCount) {
+        break;
+      }
+    }
+  }
+  return offsets;
+}
+function toUint8Array(entry) {
+  if (entry instanceof Uint8Array) return entry;
+  if (entry instanceof ArrayBuffer) return new Uint8Array(entry);
+  if (ArrayBuffer.isView(entry)) return new Uint8Array(entry.buffer, entry.byteOffset, entry.byteLength);
+  if (Array.isArray(entry)) return new Uint8Array(entry.flat());
+  if (entry && typeof entry === "object" && "buffer" in entry) {
+    return new Uint8Array(entry.buffer);
+  }
+  return null;
+}
+function isEncodedFormat(format) {
+  const normalized = format.toLowerCase();
+  return normalized === "png" || normalized === "jpg" || normalized === "jpeg";
+}
+async function decodeImageBytes(bytes, format) {
+  if (!isBrowser3 || typeof createImageBitmap === "undefined") return null;
+  const mime = format.toLowerCase() === "png" ? "image/png" : "image/jpeg";
+  const safeBytes = new Uint8Array(bytes);
+  const blob = new Blob([safeBytes.buffer], { type: mime });
+  return createImageBitmap(blob);
+}
+function decodeRawFrame(bytes, shape, channelOrder) {
+  if (!isBrowser3 || !shape) return null;
+  const [, height, width, channels] = shape;
+  if (!height || !width || !channels) return null;
+  const expectedLength = height * width * channels;
+  if (bytes.length < expectedLength) return null;
+  const rgba = new Uint8ClampedArray(width * height * 4);
+  const useBgr = channelOrder.toUpperCase() === "BGR";
+  for (let i = 0; i < width * height; i += 1) {
+    const base = i * channels;
+    const r = bytes[base + (useBgr ? 2 : 0)] ?? 0;
+    const g = bytes[base + 1] ?? 0;
+    const b = bytes[base + (useBgr ? 0 : 2)] ?? 0;
+    const a = channels === 4 ? bytes[base + 3] ?? 255 : 255;
+    const out = i * 4;
+    rgba[out] = r;
+    rgba[out + 1] = g;
+    rgba[out + 2] = b;
+    rgba[out + 3] = a;
+  }
+  return new ImageData(rgba, width, height);
+}
+
 // src/codecs/slp/h5-worker.ts
 var H5_WORKER_CODE = `
 // h5wasm streaming worker
@@ -1719,7 +1881,9 @@ function getH5FileSystem(module) {
 }
 
 // src/video/hdf5-video.ts
-var isBrowser3 = typeof window !== "undefined" && typeof document !== "undefined";
+var isBrowser4 = typeof window !== "undefined" && typeof document !== "undefined";
+var PNG_MAGIC2 = new Uint8Array([137, 80, 78, 71, 13, 10, 26, 10]);
+var JPEG_MAGIC2 = new Uint8Array([255, 216, 255]);
 var Hdf5VideoBackend = class {
   filename;
   dataset;
@@ -1731,6 +1895,7 @@ var Hdf5VideoBackend = class {
   format;
   channelOrder;
   cachedData;
+  frameOffsets;
   constructor(options) {
     this.filename = options.filename;
     this.file = options.file;
@@ -1742,6 +1907,7 @@ var Hdf5VideoBackend = class {
     this.shape = options.shape;
     this.fps = options.fps;
     this.cachedData = null;
+    this.frameOffsets = null;
   }
   async getFrame(frameIndex) {
     const dataset = this.file.get(this.datasetPath);
@@ -1749,24 +1915,87 @@ var Hdf5VideoBackend = class {
     const index = this.frameNumbers.length ? this.frameNumbers.indexOf(frameIndex) : frameIndex;
     if (index < 0) return null;
     if (!this.cachedData) {
-      this.cachedData = dataset.value;
+      const value = dataset.value;
+      this.cachedData = normalizeVideoData2(value);
+      if (isContiguousEncodedBuffer2(this.cachedData, this.format, this.shape)) {
+        this.frameOffsets = findEncodedFrameOffsets2(
+          this.cachedData,
+          this.format,
+          this.shape?.[0] ?? 0
+        );
+      }
     }
-    const entry = this.cachedData[index];
-    if (entry == null) return null;
-    const rawBytes = toUint8Array(entry);
-    if (!rawBytes) return null;
-    if (isEncodedFormat(this.format)) {
-      const decoded = await decodeImageBytes(rawBytes, this.format);
+    let rawBytes;
+    if (this.frameOffsets && this.frameOffsets.length > index) {
+      const buffer = this.cachedData;
+      const start = this.frameOffsets[index];
+      const end = index + 1 < this.frameOffsets.length ? this.frameOffsets[index + 1] : buffer.length;
+      rawBytes = buffer.slice(start, end);
+    } else {
+      const entry = this.cachedData[index];
+      if (entry == null) return null;
+      rawBytes = toUint8Array2(entry);
+    }
+    if (!rawBytes || rawBytes.length === 0) return null;
+    if (isEncodedFormat2(this.format)) {
+      const decoded = await decodeImageBytes2(rawBytes, this.format);
       return decoded ?? rawBytes;
     }
-    const image = decodeRawFrame(rawBytes, this.shape, this.channelOrder);
+    const image = decodeRawFrame2(rawBytes, this.shape, this.channelOrder);
     return image ?? rawBytes;
   }
   close() {
     this.cachedData = null;
+    this.frameOffsets = null;
   }
 };
-function toUint8Array(entry) {
+function normalizeVideoData2(value) {
+  if (Array.isArray(value)) {
+    return value;
+  }
+  if (ArrayBuffer.isView(value)) {
+    const arr = value;
+    return new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength);
+  }
+  return [];
+}
+function isContiguousEncodedBuffer2(data, format, shape) {
+  if (!isEncodedFormat2(format)) return false;
+  if (!(data instanceof Uint8Array)) return false;
+  if (data.length < 8) return false;
+  const isPng = matchesMagic2(data, PNG_MAGIC2);
+  const isJpeg = matchesMagic2(data, JPEG_MAGIC2);
+  if (!isPng && !isJpeg) return false;
+  if (shape) {
+    const frameCount = shape[0];
+    if (frameCount > 1 && data.length > 1e4) {
+      return true;
+    }
+  }
+  return true;
+}
+function matchesMagic2(buffer, magic) {
+  if (buffer.length < magic.length) return false;
+  for (let i = 0; i < magic.length; i++) {
+    if (buffer[i] !== magic[i]) return false;
+  }
+  return true;
+}
+function findEncodedFrameOffsets2(buffer, format, expectedFrameCount) {
+  const offsets = [];
+  const magic = format.toLowerCase() === "png" ? PNG_MAGIC2 : JPEG_MAGIC2;
+  for (let i = 0; i <= buffer.length - magic.length; i++) {
+    if (matchesMagic2(buffer.subarray(i), magic)) {
+      offsets.push(i);
+      i += magic.length - 1;
+      if (expectedFrameCount > 0 && offsets.length >= expectedFrameCount) {
+        break;
+      }
+    }
+  }
+  return offsets;
+}
+function toUint8Array2(entry) {
   if (entry instanceof Uint8Array) return entry;
   if (entry instanceof ArrayBuffer) return new Uint8Array(entry);
   if (ArrayBuffer.isView(entry)) return new Uint8Array(entry.buffer, entry.byteOffset, entry.byteLength);
@@ -1774,19 +2003,19 @@ function toUint8Array(entry) {
   if (entry?.buffer) return new Uint8Array(entry.buffer);
   return null;
 }
-function isEncodedFormat(format) {
+function isEncodedFormat2(format) {
   const normalized = format.toLowerCase();
   return normalized === "png" || normalized === "jpg" || normalized === "jpeg";
 }
-async function decodeImageBytes(bytes, format) {
-  if (!isBrowser3 || typeof createImageBitmap === "undefined") return null;
+async function decodeImageBytes2(bytes, format) {
+  if (!isBrowser4 || typeof createImageBitmap === "undefined") return null;
   const mime = format.toLowerCase() === "png" ? "image/png" : "image/jpeg";
   const safeBytes = new Uint8Array(bytes);
   const blob = new Blob([safeBytes.buffer], { type: mime });
   return createImageBitmap(blob);
 }
-function decodeRawFrame(bytes, shape, channelOrder) {
-  if (!isBrowser3 || !shape) return null;
+function decodeRawFrame2(bytes, shape, channelOrder) {
+  if (!isBrowser4 || !shape) return null;
   const [, height, width, channels] = shape;
   if (!height || !width || !channels) return null;
   const expectedLength = height * width * channels;
@@ -2235,13 +2464,16 @@ async function readSlpStreaming(url, options) {
     h5wasmUrl: options?.h5wasmUrl,
     filenameHint: options?.filenameHint
   });
+  const openVideos = options?.openVideos ?? false;
   try {
-    return await readFromStreamingFile(file, url, options?.filenameHint);
+    return await readFromStreamingFile(file, url, options?.filenameHint, openVideos);
   } finally {
-    await file.close();
+    if (!openVideos) {
+      await file.close();
+    }
   }
 }
-async function readFromStreamingFile(file, url, filenameHint) {
+async function readFromStreamingFile(file, url, filenameHint, openVideos = false) {
   const metadataAttrs = await file.getAttrs("metadata");
   const formatId = Number(
     metadataAttrs["format_id"]?.value ?? metadataAttrs["format_id"] ?? 1
@@ -2250,7 +2482,7 @@ async function readFromStreamingFile(file, url, filenameHint) {
   const labelsPath = filenameHint ?? url.split("/").pop()?.split("?")[0] ?? "slp-data.slp";
   const skeletons = parseSkeletons(metadataJson);
   const tracks = await readTracksStreaming(file);
-  const videos = await readVideosStreaming(file, labelsPath);
+  const videos = await readVideosStreaming(file, labelsPath, openVideos);
   const suggestions = await readSuggestionsStreaming(file, videos);
   const framesData = await readStructDatasetStreaming(file, "frames");
   const instancesData = await readStructDatasetStreaming(file, "instances");
@@ -2288,27 +2520,66 @@ async function readTracksStreaming(file) {
     return [];
   }
 }
-async function readVideosStreaming(file, labelsPath) {
+async function readVideosStreaming(file, labelsPath, openVideos = false) {
   try {
     const keys = file.keys();
     if (!keys.includes("videos_json")) return [];
     const data = await file.getDatasetValue("videos_json");
     const values = normalizeDatasetArray(data.value);
     const metadataList = parseVideosMetadata(values, labelsPath);
-    return metadataList.map((meta) => new Video({
-      filename: meta.filename,
-      backend: null,
-      // No backend in streaming mode
-      backendMetadata: {
-        dataset: meta.dataset,
-        format: meta.format,
-        shape: meta.frameCount && meta.height && meta.width && meta.channels ? [meta.frameCount, meta.height, meta.width, meta.channels] : void 0,
-        fps: meta.fps,
-        channel_order: meta.channelOrder
-      },
-      sourceVideo: meta.sourceVideo ? new Video({ filename: meta.sourceVideo.filename }) : null,
-      openBackend: false
-    }));
+    const videos = [];
+    for (const meta of metadataList) {
+      const shape = meta.frameCount && meta.height && meta.width && meta.channels ? [meta.frameCount, meta.height, meta.width, meta.channels] : void 0;
+      let backend = null;
+      if (openVideos && meta.embedded && meta.dataset) {
+        const frameNumbers = await readFrameNumbersStreaming(file, meta.dataset);
+        backend = new StreamingHdf5VideoBackend({
+          filename: meta.filename,
+          h5file: file,
+          datasetPath: meta.dataset,
+          frameNumbers,
+          format: meta.format ?? "png",
+          channelOrder: meta.channelOrder ?? "RGB",
+          shape,
+          fps: meta.fps
+        });
+      }
+      videos.push(new Video({
+        filename: meta.filename,
+        backend,
+        backendMetadata: {
+          dataset: meta.dataset,
+          format: meta.format,
+          shape,
+          fps: meta.fps,
+          channel_order: meta.channelOrder
+        },
+        sourceVideo: meta.sourceVideo ? new Video({ filename: meta.sourceVideo.filename }) : null,
+        openBackend: openVideos && meta.embedded
+      }));
+    }
+    return videos;
+  } catch {
+    return [];
+  }
+}
+async function readFrameNumbersStreaming(file, datasetPath) {
+  try {
+    const groupPath = datasetPath.endsWith("/video") ? datasetPath.slice(0, -6) : datasetPath;
+    const frameNumbersPath = `${groupPath}/frame_numbers`;
+    const groupKeys = await file.getKeys(groupPath);
+    if (!groupKeys.includes("frame_numbers")) {
+      return [];
+    }
+    const data = await file.getDatasetValue(frameNumbersPath);
+    const values = data.value;
+    if (Array.isArray(values)) {
+      return values.map((v) => Number(v));
+    }
+    if (ArrayBuffer.isView(values)) {
+      return Array.from(values).map(Number);
+    }
+    return [];
   } catch {
     return [];
   }
@@ -2827,15 +3098,16 @@ function createMatrixDataset(file, name, rows, fieldNames, dtype) {
 function isProbablyUrl2(source) {
   return typeof source === "string" && /^https?:\/\//i.test(source);
 }
-function isBrowser4() {
+function isBrowser5() {
   return typeof window !== "undefined" && typeof Worker !== "undefined";
 }
 async function loadSlp(source, options) {
   const streamMode = options?.h5?.stream ?? "auto";
-  if (isProbablyUrl2(source) && isBrowser4() && isStreamingSupported() && (streamMode === "range" || streamMode === "auto")) {
+  if (isProbablyUrl2(source) && isBrowser5() && isStreamingSupported() && (streamMode === "range" || streamMode === "auto")) {
     try {
       return await readSlpStreaming(source, {
-        filenameHint: options?.h5?.filenameHint
+        filenameHint: options?.h5?.filenameHint,
+        openVideos: options?.openVideos ?? true
       });
     } catch (e) {
       if (streamMode === "auto") {
@@ -3714,6 +3986,7 @@ export {
   RenderContext,
   Skeleton,
   StreamingH5File,
+  StreamingHdf5VideoBackend,
   SuggestionFrame,
   Symmetry,
   Track,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@talmolab/sleap-io.js",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "private": false,
   "type": "module",
   "exports": {