@skippr/live-agent-sdk 0.97.0 → 0.99.0

package/dist/esm/lib-exports.js

@@ -3146,6 +3146,7 @@ function useSessionHold({
   const [countdown, setCountdown] = useState9(timeoutSeconds);
   const screenWasSharedBeforeHoldRef = useRef11(false);
   const holdTransitionInFlightRef = useRef11(false);
+  const clearPausedCaptureRef = useRef11(null);
   const publish = useCallback10(async (type) => {
     await localParticipant.publishData(textEncoder2.encode(JSON.stringify({ type })), {
       reliable: true,
@@ -3192,6 +3193,7 @@ function useSessionHold({
     try {
       await localParticipant.setMicrophoneEnabled(true);
       await resumeScreenShareOnResume();
+      clearPausedCaptureRef.current?.();
       await publish("resume");
       setIsHeld(false);
     } catch (error) {
@@ -3223,7 +3225,7 @@ function useSessionHold({
       holdTransitionInFlightRef.current = false;
     });
   }, [isHeld, hold, release]);
-  return { isHeld, countdown, toggle };
+  return { isHeld, countdown, toggle, clearPausedCaptureRef };
 }
 
 // src/context/SessionHoldContext.tsx
@@ -3231,7 +3233,8 @@ import { jsx as jsx13 } from "react/jsx-runtime";
 var SessionHoldContext = createContext3({
   isHeld: false,
   countdown: 0,
-  toggle: () => {}
+  toggle: () => {},
+  clearPausedCaptureRef: { current: null }
 });
 function SessionHoldProvider({ children }) {
   const { pauseSession } = useLiveAgent();
@@ -4976,6 +4979,7 @@ function accessibleName(element, options = {}) {
 var REF_PREFIX = "r-";
 var VALUE_MAX_CHARS = 80;
 var HREF_MAX_CHARS = 60;
+var TABLE_ROW_MAX_CHARS = 160;
 var PAGE_CONTENT_MAX_BYTES = 14000;
 var NODE_LIMIT = 1500;
 var SENSITIVE_IFRAME_HOSTS = [
@@ -5321,8 +5325,9 @@ function ensureStableRef(element) {
 function formatNodeAsLine(entry) {
   const indent = "  ".repeat(entry.depth);
   const namePart = entry.name ? ` "${escapeAttributeValue(entry.name)}"` : "";
+  const refPart = entry.ref ? ` ${entry.ref}` : "";
   const attrsPart = entry.attrs.length > 0 ? ` ${entry.attrs.join(" ")}` : "";
-  return `${indent}${entry.role}${namePart} ${entry.ref}${attrsPart}`;
+  return `${indent}${entry.role}${namePart}${refPart}${attrsPart}`;
 }
 function shouldSkipSubtree(element) {
   if (element.id === WIDGET_ROOT_ID)
@@ -5350,13 +5355,66 @@ function getSameOriginIframeBody(element) {
     return null;
   }
 }
-function walkAndCollect(element, depth, entries) {
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+function formatTableRowText(row) {
+  const cellValues = Array.from(row.children).filter((cell) => cell.tagName === "TD" || cell.tagName === "TH").map((cell) => normalizeText(cell.textContent).slice(0, VALUE_MAX_CHARS));
+  return cellValues.join(" | ").slice(0, TABLE_ROW_MAX_CHARS).trim();
+}
+function isTableHeaderRow(row) {
+  return Array.from(row.children).some((cell) => cell.tagName === "TH");
+}
+function directTextOfElement(element) {
+  let combinedText = "";
+  for (const childNode of Array.from(element.childNodes)) {
+    if (childNode.nodeType === Node.TEXT_NODE)
+      combinedText += `${childNode.textContent} `;
+  }
+  return normalizeText(combinedText);
+}
+var REFLESS_TEXT_ENTRY_REF = "";
+function collectRefLessTextLeaf(text, depth, rect, entries) {
+  entries.push({
+    depth,
+    role: "text",
+    name: text.slice(0, VALUE_MAX_CHARS),
+    ref: REFLESS_TEXT_ENTRY_REF,
+    attrs: [],
+    area: rect.width * rect.height
+  });
+}
+function collectTableRowAsSingleEntry(row, depth, rect, entries) {
+  const rowText = formatTableRowText(row);
+  if (!rowText)
+    return false;
+  entries.push({
+    depth,
+    role: "row",
+    name: rowText,
+    ref: ensureStableRef(row),
+    attrs: isTableHeaderRow(row) ? ["header"] : [],
+    area: rect.width * rect.height
+  });
+  for (const child of Array.from(row.children)) {
+    walkAndCollect(child, depth + 1, entries, true);
+  }
+  return true;
+}
+function walkAndCollect(element, depth, entries, suppressTextLeaves = false) {
   if (shouldSkipSubtree(element))
     return;
   const rect = element.getBoundingClientRect();
+  if (element.tagName === "TR") {
+    if (!isVisible(element, rect))
+      return;
+    if (collectTableRowAsSingleEntry(element, depth, rect, entries))
+      return;
+  }
   const role = inferRole(element) ?? (isCursorClickable(element, rect) ? "button" : null);
   const isVisibleAndEmittable = role !== null && isVisible(element, rect) && shouldEmitElement(element, role);
   let childDepth = depth;
+  let childSuppressTextLeaves = suppressTextLeaves;
   if (isVisibleAndEmittable && role) {
     const ref = ensureStableRef(element);
     const name = accessibleName(element);
@@ -5369,12 +5427,20 @@ function walkAndCollect(element, depth, entries) {
       area: rect.width * rect.height
     });
     childDepth = depth + 1;
+    const nameAlreadyCoversSubtreeText = name !== "";
+    if (nameAlreadyCoversSubtreeText)
+      childSuppressTextLeaves = true;
+  } else if (!suppressTextLeaves) {
+    const ownText = directTextOfElement(element);
+    if (ownText && isVisible(element, rect)) {
+      collectRefLessTextLeaf(ownText, depth, rect, entries);
+    }
   }
   if (role === "iframe") {
     const iframeBody = getSameOriginIframeBody(element);
     if (iframeBody) {
       for (const child of Array.from(iframeBody.children)) {
-        walkAndCollect(child, childDepth, entries);
+        walkAndCollect(child, childDepth, entries, childSuppressTextLeaves);
       }
     }
     return;
@@ -5382,15 +5448,19 @@ function walkAndCollect(element, depth, entries) {
   if (role && isBlindRegion(role))
     return;
   for (const child of Array.from(element.children)) {
-    walkAndCollect(child, childDepth, entries);
+    walkAndCollect(child, childDepth, entries, childSuppressTextLeaves);
   }
 }
-function dropSmallestAreaEntriesUntilUnderLimit(entries) {
+var TRUNCATION_DROP_ORDER_BY_ROLE = { text: 0, row: 1 };
+function truncationDropOrder(entry) {
+  return TRUNCATION_DROP_ORDER_BY_ROLE[entry.role] ?? 2;
+}
+function dropInformationalEntriesFirstUntilUnderLimit(entries) {
   if (entries.length <= NODE_LIMIT)
     return entries;
   const indexedEntries = entries.map((entry, index2) => ({ entry, index: index2 }));
-  const entriesByAreaAscending = indexedEntries.slice(1).sort((a, b) => a.entry.area - b.entry.area);
-  const indicesToDrop = new Set(entriesByAreaAscending.slice(0, entries.length - NODE_LIMIT).map((indexed) => indexed.index));
+  const entriesMostDroppableFirst = indexedEntries.slice(1).sort((a, b) => truncationDropOrder(a.entry) - truncationDropOrder(b.entry) || a.entry.area - b.entry.area);
+  const indicesToDrop = new Set(entriesMostDroppableFirst.slice(0, entries.length - NODE_LIMIT).map((indexed) => indexed.index));
   const keptEntries = [];
   for (let i = 0;i < entries.length; i++) {
     if (!indicesToDrop.has(i))
@@ -5415,7 +5485,7 @@ function buildAccessibilityTree() {
     }
   }
   const exceededNodeLimit = collectedEntries.length > NODE_LIMIT;
-  const finalEntries = exceededNodeLimit ? dropSmallestAreaEntriesUntilUnderLimit(collectedEntries) : collectedEntries;
+  const finalEntries = exceededNodeLimit ? dropInformationalEntriesFirstUntilUnderLimit(collectedEntries) : collectedEntries;
   const treeLines = finalEntries.map(formatNodeAsLine);
   let pageContent = treeLines.join(`
 `);
@@ -5790,21 +5860,46 @@ function paintCursorOverlay(ctx, transform, brand = DEFAULT_BRAND_PALETTE) {
 }
 
 // src/capture/snapdom.ts
+var UNLOADABLE_IMAGE_PLACEHOLDER_FILL = "#e2e8f0";
+function placeholderForUnloadableImage({
+  width = 100,
+  height = 100
+}) {
+  const placeholderSvg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"><rect width="100%" height="100%" fill="${UNLOADABLE_IMAGE_PLACEHOLDER_FILL}"/></svg>`;
+  return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(placeholderSvg)}`;
+}
 var cachedSnapdomModule = null;
 function loadSnapdom() {
-  if (!cachedSnapdomModule)
+  if (!cachedSnapdomModule) {
     cachedSnapdomModule = import("@zumer/snapdom");
+  }
   return cachedSnapdomModule;
 }
+function preCacheSnapshotResources() {
+  loadSnapdom().then(({ preCache }) => preCache(document, { embedFonts: true })).catch(() => {
+    return;
+  });
+}
 async function snapToCanvas(element, options = {}) {
   const snapdomModule = await loadSnapdom();
-  return snapdomModule.snapdom.toCanvas(element, options);
+  const optionsWithEngineDefaults = {
+    embedFonts: true,
+    fallbackURL: placeholderForUnloadableImage,
+    ...options,
+    snap: snapdomModule.snapdom
+  };
+  return snapdomModule.snapdom.toCanvas(element, optionsWithEngineDefaults);
 }
 
 // src/components/DomCapture.tsx
 var SNAPSHOT_INTERVAL_MS = 3000;
 var A11Y_PUBLISH_INTERVAL_MS = 2000;
 var FIRST_SNAPSHOT_DELAY_MS = 400;
+var RASTERIZE_TIMEOUT_MS = 1e4;
+var FULL_QUALITY_TIMEOUTS_BEFORE_COOLDOWN = 1;
+var FAST_CAPTURES_BEFORE_FULL_QUALITY_RETRY = 10;
+var RASTERIZE_TIMED_OUT = Symbol("rasterize-timed-out");
+var SCROLL_SETTLE_MS = 300;
 var CAPTURE_PRESET = ScreenSharePresets2.h1080fps30;
 var CAPTURE_FPS = CAPTURE_PRESET.encoding.maxFramerate ?? 30;
 var CAPTURE_BITRATE = 4000000;
@@ -5812,15 +5907,20 @@ var DOM_SNAPSHOT_GZIP_THRESHOLD_BYTES = 14000;
 var MAX_CANVAS_DPR = 1.5;
 var CONSECUTIVE_FAILURES_BEFORE_REPORT = 3;
 var textEncoder4 = new TextEncoder;
+var SNAPSHOT_EXCLUDE_SELECTORS = [`#${WIDGET_ROOT_ID}`, `[${PRIVATE_ATTR}]`];
+function isCapturableIframe(element) {
+  return isSameOriginIframe(element) && !isSensitiveIframe(element);
+}
 function shouldIncludeInSnapshot(element) {
-  if (!(element instanceof Element))
-    return true;
-  if (element.id === WIDGET_ROOT_ID)
-    return false;
-  if (element.hasAttribute?.(PRIVATE_ATTR))
+  if (element instanceof HTMLIFrameElement && !isCapturableIframe(element))
     return false;
   return true;
 }
+function shouldIncludeInFirstFrame(element) {
+  if (element.tagName === "IMG")
+    return false;
+  return shouldIncludeInSnapshot(element);
+}
 function createPublishingCanvas() {
   const dprBoost = Math.min(window.devicePixelRatio || 1, MAX_CANVAS_DPR);
   const canvas = document.createElement("canvas");
@@ -5841,22 +5941,23 @@ function getCanvasCaptureStream(canvas, fps) {
   }
   return captureStreamFn.call(canvas, fps);
 }
-async function rasterizeViewportToPageLayer(width, height) {
-  const dpr = window.devicePixelRatio || 1;
+async function rasterizeViewportToPageLayer(width, height, { skipCrossOriginImageSettling, includeImages }) {
+  const requestedDpr = Math.min(window.devicePixelRatio || 1, MAX_CANVAS_DPR);
+  const viewportWidthAtCaptureStart = window.innerWidth;
   const snapshotCanvas = await snapToCanvas(document.documentElement, {
-    filter: shouldIncludeInSnapshot,
-    filterMode: "remove",
+    exclude: SNAPSHOT_EXCLUDE_SELECTORS,
+    excludeMode: "remove",
+    filter: includeImages ? shouldIncludeInSnapshot : shouldIncludeInFirstFrame,
+    filterMode: "hide",
     backgroundColor: "#ffffff",
-    fast: false,
-    dpr
+    fast: skipCrossOriginImageSettling,
+    dpr: requestedDpr,
+    clip: "viewport"
   });
-  const sourceX = window.scrollX * dpr;
-  const sourceY = window.scrollY * dpr;
-  const sourceWidth = window.innerWidth * dpr;
-  const sourceHeight = window.innerHeight * dpr;
-  const fitScale = Math.min(width / sourceWidth, height / sourceHeight);
-  const destWidth = sourceWidth * fitScale;
-  const destHeight = sourceHeight * fitScale;
+  const canvasPixelsPerCssPixel = viewportWidthAtCaptureStart > 0 ? snapshotCanvas.width / viewportWidthAtCaptureStart : requestedDpr;
+  const fitScale = Math.min(width / snapshotCanvas.width, height / snapshotCanvas.height);
+  const destWidth = snapshotCanvas.width * fitScale;
+  const destHeight = snapshotCanvas.height * fitScale;
   const destX = (width - destWidth) / 2;
   const destY = (height - destHeight) / 2;
   const layer = document.createElement("canvas");
@@ -5867,10 +5968,14 @@ async function rasterizeViewportToPageLayer(width, height) {
     throw new Error("Failed to get 2D context for page layer");
   layerCtx.fillStyle = "#ffffff";
   layerCtx.fillRect(0, 0, width, height);
-  layerCtx.drawImage(snapshotCanvas, sourceX, sourceY, sourceWidth, sourceHeight, destX, destY, destWidth, destHeight);
+  layerCtx.drawImage(snapshotCanvas, destX, destY, destWidth, destHeight);
   return {
     canvas: layer,
-    viewportToCanvas: { scale: dpr * fitScale, offsetX: destX, offsetY: destY }
+    viewportToCanvas: {
+      scale: canvasPixelsPerCssPixel * fitScale,
+      offsetX: destX,
+      offsetY: destY
+    }
   };
 }
 function compositePageAndCursorOverlay(ctx, canvas, cachedPage, brand) {
@@ -5894,6 +5999,12 @@ async function gzipIfBeneficial(snapshotBytes) {
     return { bytes: snapshotBytes, compressionFlag: 0 };
   }
 }
+function frameWithCompressionFlag(compressionFlag, payload) {
+  const frame = new Uint8Array(payload.byteLength + 1);
+  frame[0] = compressionFlag;
+  frame.set(payload, 1);
+  return frame;
+}
 async function buildDomSnapshotFrame() {
   const a11yTree = buildAccessibilityTree();
   const snapshotJson = JSON.stringify({
@@ -5906,14 +6017,12 @@ async function buildDomSnapshotFrame() {
   });
   const snapshotBytes = textEncoder4.encode(snapshotJson);
   const { bytes, compressionFlag } = await gzipIfBeneficial(snapshotBytes);
-  const frame = new Uint8Array(bytes.byteLength + 1);
-  frame[0] = compressionFlag;
-  frame.set(bytes, 1);
-  return frame;
+  return frameWithCompressionFlag(compressionFlag, bytes);
 }
 function reportCaptureError(localParticipant) {
   try {
-    localParticipant.publishData(textEncoder4.encode(JSON.stringify({ type: "capture_error" })), {
+    const payload = textEncoder4.encode(JSON.stringify({ type: "capture_error" }));
+    localParticipant.publishData(frameWithCompressionFlag(0, payload), {
       reliable: true,
       topic: DOM_SNAPSHOT_TOPIC
     }).catch(() => {
@@ -5930,7 +6039,7 @@ async function unpublishAndStopTrack(localParticipant, videoTrack) {
 function DomCapture({ pushOrTapMicMode = false }) {
   const { localParticipant } = useLocalParticipant5();
   const connectionState = useConnectionState2();
-  const { isHeld } = useSessionHoldContext();
+  const { isHeld, clearPausedCaptureRef } = useSessionHoldContext();
   const accentColor = useContext9(LiveAgentContext)?.appearance?.accentColor ?? null;
   const brand = useMemo4(() => resolveBrandPalette(accentColor), [accentColor]);
   const brandRef = useRef17(brand);
@@ -5956,12 +6065,18 @@ function DomCapture({ pushOrTapMicMode = false }) {
       return;
     videoTrack.contentHint = "text";
     didStartRef.current = true;
+    preCacheSnapshotResources();
     let cancelled = false;
     let snapshotInFlight = false;
     let a11yPublishInFlight = false;
     let a11yPublishPending = false;
     let consecutiveCaptureFailures = 0;
     let cachedPage = null;
+    let rasterizeSequence = 0;
+    let lastCommittedSequence = 0;
+    let firstFrameCommitted = false;
+    let capturesUntilFullQualityRetry = 0;
+    let consecutiveFullQualityTimeouts = 0;
     if (!pushOrTapMicMode && !localParticipant.isMicrophoneEnabled) {
       localParticipant.setMicrophoneEnabled(true).catch((error) => console.error("Failed to enable microphone:", error));
     }
@@ -5977,16 +6092,61 @@ function DomCapture({ pushOrTapMicMode = false }) {
       for (const track of canvasStream.getTracks())
         track.stop();
     });
-    const refreshPageAndComposite = async () => {
-      if (cancelled || snapshotInFlight || pausedRef.current)
+    const commitPageLayerIfNewest = (sequence, page) => {
+      if (cancelled || pausedRef.current || sequence <= lastCommittedSequence)
+        return false;
+      lastCommittedSequence = sequence;
+      cachedPage = page;
+      compositePageAndCursorOverlay(ctx, canvas, page, brandRef.current);
+      firstFrameCommitted = true;
+      return true;
+    };
+    const rasterizeAndCommitWaitingOutTimeout = async (options) => {
+      const sequence = ++rasterizeSequence;
+      const captureInProgress = rasterizeViewportToPageLayer(canvas.width, canvas.height, options);
+      const captureOrTimeout = await Promise.race([
+        captureInProgress,
+        new Promise((resolve) => setTimeout(() => resolve(RASTERIZE_TIMED_OUT), RASTERIZE_TIMEOUT_MS))
+      ]);
+      if (captureOrTimeout === RASTERIZE_TIMED_OUT) {
+        captureInProgress.then((lateCapture) => commitPageLayerIfNewest(sequence, lateCapture)).catch(() => {
+          return;
+        });
+        return { timedOut: true };
+      }
+      commitPageLayerIfNewest(sequence, captureOrTimeout);
+      return { timedOut: false };
+    };
+    let followUpRefreshQueued = false;
+    const refreshPageAndComposite = async ({ queueWhenBusy = false } = {}) => {
+      if (cancelled || snapshotInFlight || pausedRef.current) {
+        if (snapshotInFlight && !cancelled && queueWhenBusy)
+          followUpRefreshQueued = true;
         return;
+      }
       snapshotInFlight = true;
       try {
-        cachedPage = await rasterizeViewportToPageLayer(canvas.width, canvas.height);
-        if (cancelled || pausedRef.current)
-          return;
-        compositePageAndCursorOverlay(ctx, canvas, cachedPage, brandRef.current);
+        const isImagelessFirstFrame = !firstFrameCommitted;
+        const useFullQuality = firstFrameCommitted && capturesUntilFullQualityRetry === 0;
+        const { timedOut } = await rasterizeAndCommitWaitingOutTimeout({
+          skipCrossOriginImageSettling: !useFullQuality,
+          includeImages: !isImagelessFirstFrame
+        });
+        if (isImagelessFirstFrame && firstFrameCommitted)
+          followUpRefreshQueued = true;
         consecutiveCaptureFailures = 0;
+        if (useFullQuality) {
+          if (timedOut) {
+            consecutiveFullQualityTimeouts += 1;
+            if (consecutiveFullQualityTimeouts >= FULL_QUALITY_TIMEOUTS_BEFORE_COOLDOWN) {
+              capturesUntilFullQualityRetry = FAST_CAPTURES_BEFORE_FULL_QUALITY_RETRY;
+            }
+          } else {
+            consecutiveFullQualityTimeouts = 0;
+          }
+        } else if (capturesUntilFullQualityRetry > 0) {
+          capturesUntilFullQualityRetry -= 1;
+        }
       } catch {
         if (cancelled)
           return;
@@ -5996,6 +6156,10 @@ function DomCapture({ pushOrTapMicMode = false }) {
         }
       } finally {
         snapshotInFlight = false;
+        if (followUpRefreshQueued && !cancelled) {
+          followUpRefreshQueued = false;
+          refreshPageAndComposite();
+        }
       }
     };
     let overlayAnimationFrame = null;
@@ -6061,36 +6225,55 @@ function DomCapture({ pushOrTapMicMode = false }) {
     });
     const cleanupSnapshotRequest = onSnapshotRequest(() => {
       tickA11yPublish();
-      refreshPageAndComposite();
+      refreshPageAndComposite({ queueWhenBusy: true });
     });
+    let scrollSettleTimer = null;
+    const refreshOnceScrollSettles = () => {
+      if (scrollSettleTimer)
+        clearTimeout(scrollSettleTimer);
+      scrollSettleTimer = setTimeout(() => {
+        tickA11yPublish();
+        refreshPageAndComposite({ queueWhenBusy: true });
+      }, SCROLL_SETTLE_MS);
+    };
+    window.addEventListener("scroll", refreshOnceScrollSettles, { passive: true, capture: true });
     let snapshotTimer = null;
     const scheduleNextSnapshot = (delay) => {
       if (cancelled)
         return;
-      snapshotTimer = setTimeout(async () => {
-        await refreshPageAndComposite();
+      snapshotTimer = setTimeout(() => {
+        refreshPageAndComposite();
         scheduleNextSnapshot(SNAPSHOT_INTERVAL_MS);
       }, delay);
     };
     scheduleNextSnapshot(FIRST_SNAPSHOT_DELAY_MS);
     tickA11yPublish();
-    const a11yPublishTimer = setInterval(tickA11yPublish, A11Y_PUBLISH_INTERVAL_MS);
+    const a11yPublishTimer = setInterval(() => void tickA11yPublish(), A11Y_PUBLISH_INTERVAL_MS);
     freezeCaptureRef.current = () => {
       pausedRef.current = true;
       drawPausedOverlay(canvas, ctx);
     };
-    resumeCaptureRef.current = () => {
+    const repaintCleanFrameFromCachedPage = () => {
       pausedRef.current = false;
+      compositePageAndCursorOverlay(ctx, canvas, cachedPage, brandRef.current);
+    };
+    resumeCaptureRef.current = () => {
+      repaintCleanFrameFromCachedPage();
       tickA11yPublish();
       refreshPageAndComposite();
     };
+    clearPausedCaptureRef.current = repaintCleanFrameFromCachedPage;
     return () => {
       cancelled = true;
       didStartRef.current = false;
       freezeCaptureRef.current = null;
       resumeCaptureRef.current = null;
+      clearPausedCaptureRef.current = null;
       if (snapshotTimer)
         clearTimeout(snapshotTimer);
+      if (scrollSettleTimer)
+        clearTimeout(scrollSettleTimer);
+      window.removeEventListener("scroll", refreshOnceScrollSettles, { capture: true });
       clearInterval(a11yPublishTimer);
       if (overlayAnimationFrame !== null)
         cancelAnimationFrame(overlayAnimationFrame);
@@ -6101,7 +6284,7 @@ function DomCapture({ pushOrTapMicMode = false }) {
       for (const track of canvasStream.getTracks())
         track.stop();
     };
-  }, [connectionState, localParticipant, pushOrTapMicMode]);
+  }, [connectionState, localParticipant, pushOrTapMicMode, clearPausedCaptureRef]);
   useEffect22(() => {
     if (isHeld) {
       freezeCaptureRef.current?.();