@skippr/live-agent-sdk 0.96.0 → 0.98.0

package/dist/esm/lib-exports.js

@@ -394,6 +394,12 @@ var CHAT_SURFACE = {
   Room: "room"
 };
 var DEFAULT_CHAT_SURFACE = CHAT_SURFACE.Line;
+var ACTION_PLAN_MODE = {
+  Always: "always",
+  Auto: "auto",
+  Never: "never"
+};
+var DEFAULT_ACTION_PLAN_MODE = ACTION_PLAN_MODE.Auto;
 var LAUNCHER_MODE = {
   Avatar: "avatar",
   Strip: "strip",
@@ -2182,7 +2188,11 @@ function isBuddyActive(appearance) {
   return appearance?.buddy?.enabled === true;
 }
 function buddyEffectiveControls(base) {
-  const next = { ...base ?? {}, micMode: MIC_MODE.PushToTalk };
+  const next = {
+    ...base ?? {},
+    micMode: MIC_MODE.PushToTalk,
+    actionPlanMode: ACTION_PLAN_MODE.Never
+  };
   delete next.chat;
   return next;
 }
@@ -4058,6 +4068,9 @@ function useSession({
       enabledControls.highlight = true;
     if (agentControls?.actions === true)
       enabledControls.actions = true;
+    if (agentControls?.actionPlanMode) {
+      enabledControls.actionPlanMode = agentControls.actionPlanMode;
+    }
     const micMode = agentControls?.micMode;
     if (isRequestResponseMicMode(micMode))
       enabledControls.micMode = micMode;
@@ -4963,6 +4976,7 @@ function accessibleName(element, options = {}) {
 var REF_PREFIX = "r-";
 var VALUE_MAX_CHARS = 80;
 var HREF_MAX_CHARS = 60;
+var TABLE_ROW_MAX_CHARS = 160;
 var PAGE_CONTENT_MAX_BYTES = 14000;
 var NODE_LIMIT = 1500;
 var SENSITIVE_IFRAME_HOSTS = [
@@ -5308,8 +5322,9 @@ function ensureStableRef(element) {
 function formatNodeAsLine(entry) {
   const indent = "  ".repeat(entry.depth);
   const namePart = entry.name ? ` "${escapeAttributeValue(entry.name)}"` : "";
+  const refPart = entry.ref ? ` ${entry.ref}` : "";
   const attrsPart = entry.attrs.length > 0 ? ` ${entry.attrs.join(" ")}` : "";
-  return `${indent}${entry.role}${namePart} ${entry.ref}${attrsPart}`;
+  return `${indent}${entry.role}${namePart}${refPart}${attrsPart}`;
 }
 function shouldSkipSubtree(element) {
   if (element.id === WIDGET_ROOT_ID)
@@ -5337,13 +5352,66 @@ function getSameOriginIframeBody(element) {
     return null;
   }
 }
-function walkAndCollect(element, depth, entries) {
+function normalizeText(text) {
+  return (text ?? "").replace(/\s+/g, " ").trim();
+}
+function formatTableRowText(row) {
+  const cellValues = Array.from(row.children).filter((cell) => cell.tagName === "TD" || cell.tagName === "TH").map((cell) => normalizeText(cell.textContent).slice(0, VALUE_MAX_CHARS));
+  return cellValues.join(" | ").slice(0, TABLE_ROW_MAX_CHARS).trim();
+}
+function isTableHeaderRow(row) {
+  return Array.from(row.children).some((cell) => cell.tagName === "TH");
+}
+function directTextOfElement(element) {
+  let combinedText = "";
+  for (const childNode of Array.from(element.childNodes)) {
+    if (childNode.nodeType === Node.TEXT_NODE)
+      combinedText += `${childNode.textContent} `;
+  }
+  return normalizeText(combinedText);
+}
+var REFLESS_TEXT_ENTRY_REF = "";
+function collectRefLessTextLeaf(text, depth, rect, entries) {
+  entries.push({
+    depth,
+    role: "text",
+    name: text.slice(0, VALUE_MAX_CHARS),
+    ref: REFLESS_TEXT_ENTRY_REF,
+    attrs: [],
+    area: rect.width * rect.height
+  });
+}
+function collectTableRowAsSingleEntry(row, depth, rect, entries) {
+  const rowText = formatTableRowText(row);
+  if (!rowText)
+    return false;
+  entries.push({
+    depth,
+    role: "row",
+    name: rowText,
+    ref: ensureStableRef(row),
+    attrs: isTableHeaderRow(row) ? ["header"] : [],
+    area: rect.width * rect.height
+  });
+  for (const child of Array.from(row.children)) {
+    walkAndCollect(child, depth + 1, entries, true);
+  }
+  return true;
+}
+function walkAndCollect(element, depth, entries, suppressTextLeaves = false) {
   if (shouldSkipSubtree(element))
     return;
   const rect = element.getBoundingClientRect();
+  if (element.tagName === "TR") {
+    if (!isVisible(element, rect))
+      return;
+    if (collectTableRowAsSingleEntry(element, depth, rect, entries))
+      return;
+  }
   const role = inferRole(element) ?? (isCursorClickable(element, rect) ? "button" : null);
   const isVisibleAndEmittable = role !== null && isVisible(element, rect) && shouldEmitElement(element, role);
   let childDepth = depth;
+  let childSuppressTextLeaves = suppressTextLeaves;
   if (isVisibleAndEmittable && role) {
     const ref = ensureStableRef(element);
     const name = accessibleName(element);
@@ -5356,12 +5424,20 @@ function walkAndCollect(element, depth, entries) {
       area: rect.width * rect.height
     });
     childDepth = depth + 1;
+    const nameAlreadyCoversSubtreeText = name !== "";
+    if (nameAlreadyCoversSubtreeText)
+      childSuppressTextLeaves = true;
+  } else if (!suppressTextLeaves) {
+    const ownText = directTextOfElement(element);
+    if (ownText && isVisible(element, rect)) {
+      collectRefLessTextLeaf(ownText, depth, rect, entries);
+    }
   }
   if (role === "iframe") {
     const iframeBody = getSameOriginIframeBody(element);
     if (iframeBody) {
       for (const child of Array.from(iframeBody.children)) {
-        walkAndCollect(child, childDepth, entries);
+        walkAndCollect(child, childDepth, entries, childSuppressTextLeaves);
       }
     }
     return;
@@ -5369,15 +5445,19 @@ function walkAndCollect(element, depth, entries) {
   if (role && isBlindRegion(role))
     return;
   for (const child of Array.from(element.children)) {
-    walkAndCollect(child, childDepth, entries);
+    walkAndCollect(child, childDepth, entries, childSuppressTextLeaves);
   }
 }
-function dropSmallestAreaEntriesUntilUnderLimit(entries) {
+var TRUNCATION_DROP_ORDER_BY_ROLE = { text: 0, row: 1 };
+function truncationDropOrder(entry) {
+  return TRUNCATION_DROP_ORDER_BY_ROLE[entry.role] ?? 2;
+}
+function dropInformationalEntriesFirstUntilUnderLimit(entries) {
   if (entries.length <= NODE_LIMIT)
     return entries;
   const indexedEntries = entries.map((entry, index2) => ({ entry, index: index2 }));
-  const entriesByAreaAscending = indexedEntries.slice(1).sort((a, b) => a.entry.area - b.entry.area);
-  const indicesToDrop = new Set(entriesByAreaAscending.slice(0, entries.length - NODE_LIMIT).map((indexed) => indexed.index));
+  const entriesMostDroppableFirst = indexedEntries.slice(1).sort((a, b) => truncationDropOrder(a.entry) - truncationDropOrder(b.entry) || a.entry.area - b.entry.area);
+  const indicesToDrop = new Set(entriesMostDroppableFirst.slice(0, entries.length - NODE_LIMIT).map((indexed) => indexed.index));
   const keptEntries = [];
   for (let i = 0;i < entries.length; i++) {
     if (!indicesToDrop.has(i))
@@ -5402,7 +5482,7 @@ function buildAccessibilityTree() {
     }
   }
   const exceededNodeLimit = collectedEntries.length > NODE_LIMIT;
-  const finalEntries = exceededNodeLimit ? dropSmallestAreaEntriesUntilUnderLimit(collectedEntries) : collectedEntries;
+  const finalEntries = exceededNodeLimit ? dropInformationalEntriesFirstUntilUnderLimit(collectedEntries) : collectedEntries;
   const treeLines = finalEntries.map(formatNodeAsLine);
   let pageContent = treeLines.join(`
 `);
@@ -5777,21 +5857,46 @@ function paintCursorOverlay(ctx, transform, brand = DEFAULT_BRAND_PALETTE) {
 }
 
 // src/capture/snapdom.ts
+var UNLOADABLE_IMAGE_PLACEHOLDER_FILL = "#e2e8f0";
+function placeholderForUnloadableImage({
+  width = 100,
+  height = 100
+}) {
+  const placeholderSvg = `<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${height}"><rect width="100%" height="100%" fill="${UNLOADABLE_IMAGE_PLACEHOLDER_FILL}"/></svg>`;
+  return `data:image/svg+xml;charset=utf-8,${encodeURIComponent(placeholderSvg)}`;
+}
 var cachedSnapdomModule = null;
 function loadSnapdom() {
-  if (!cachedSnapdomModule)
+  if (!cachedSnapdomModule) {
     cachedSnapdomModule = import("@zumer/snapdom");
+  }
   return cachedSnapdomModule;
 }
+function preCacheSnapshotResources() {
+  loadSnapdom().then(({ preCache }) => preCache(document, { embedFonts: true })).catch(() => {
+    return;
+  });
+}
 async function snapToCanvas(element, options = {}) {
   const snapdomModule = await loadSnapdom();
-  return snapdomModule.snapdom.toCanvas(element, options);
+  const optionsWithEngineDefaults = {
+    embedFonts: true,
+    fallbackURL: placeholderForUnloadableImage,
+    ...options,
+    snap: snapdomModule.snapdom
+  };
+  return snapdomModule.snapdom.toCanvas(element, optionsWithEngineDefaults);
 }
 
 // src/components/DomCapture.tsx
 var SNAPSHOT_INTERVAL_MS = 3000;
 var A11Y_PUBLISH_INTERVAL_MS = 2000;
 var FIRST_SNAPSHOT_DELAY_MS = 400;
+var RASTERIZE_TIMEOUT_MS = 1e4;
+var FULL_QUALITY_TIMEOUTS_BEFORE_COOLDOWN = 1;
+var FAST_CAPTURES_BEFORE_FULL_QUALITY_RETRY = 10;
+var RASTERIZE_TIMED_OUT = Symbol("rasterize-timed-out");
+var SCROLL_SETTLE_MS = 300;
 var CAPTURE_PRESET = ScreenSharePresets2.h1080fps30;
 var CAPTURE_FPS = CAPTURE_PRESET.encoding.maxFramerate ?? 30;
 var CAPTURE_BITRATE = 4000000;
@@ -5799,15 +5904,20 @@ var DOM_SNAPSHOT_GZIP_THRESHOLD_BYTES = 14000;
 var MAX_CANVAS_DPR = 1.5;
 var CONSECUTIVE_FAILURES_BEFORE_REPORT = 3;
 var textEncoder4 = new TextEncoder;
+var SNAPSHOT_EXCLUDE_SELECTORS = [`#${WIDGET_ROOT_ID}`, `[${PRIVATE_ATTR}]`];
+function isCapturableIframe(element) {
+  return isSameOriginIframe(element) && !isSensitiveIframe(element);
+}
 function shouldIncludeInSnapshot(element) {
-  if (!(element instanceof Element))
-    return true;
-  if (element.id === WIDGET_ROOT_ID)
-    return false;
-  if (element.hasAttribute?.(PRIVATE_ATTR))
+  if (element instanceof HTMLIFrameElement && !isCapturableIframe(element))
     return false;
   return true;
 }
+function shouldIncludeInFirstFrame(element) {
+  if (element.tagName === "IMG")
+    return false;
+  return shouldIncludeInSnapshot(element);
+}
 function createPublishingCanvas() {
   const dprBoost = Math.min(window.devicePixelRatio || 1, MAX_CANVAS_DPR);
   const canvas = document.createElement("canvas");
@@ -5828,22 +5938,23 @@ function getCanvasCaptureStream(canvas, fps) {
   }
   return captureStreamFn.call(canvas, fps);
 }
-async function rasterizeViewportToPageLayer(width, height) {
-  const dpr = window.devicePixelRatio || 1;
+async function rasterizeViewportToPageLayer(width, height, { skipCrossOriginImageSettling, includeImages }) {
+  const requestedDpr = Math.min(window.devicePixelRatio || 1, MAX_CANVAS_DPR);
+  const viewportWidthAtCaptureStart = window.innerWidth;
   const snapshotCanvas = await snapToCanvas(document.documentElement, {
-    filter: shouldIncludeInSnapshot,
-    filterMode: "remove",
+    exclude: SNAPSHOT_EXCLUDE_SELECTORS,
+    excludeMode: "remove",
+    filter: includeImages ? shouldIncludeInSnapshot : shouldIncludeInFirstFrame,
+    filterMode: "hide",
     backgroundColor: "#ffffff",
-    fast: false,
-    dpr
+    fast: skipCrossOriginImageSettling,
+    dpr: requestedDpr,
+    clip: "viewport"
   });
-  const sourceX = window.scrollX * dpr;
-  const sourceY = window.scrollY * dpr;
-  const sourceWidth = window.innerWidth * dpr;
-  const sourceHeight = window.innerHeight * dpr;
-  const fitScale = Math.min(width / sourceWidth, height / sourceHeight);
-  const destWidth = sourceWidth * fitScale;
-  const destHeight = sourceHeight * fitScale;
+  const canvasPixelsPerCssPixel = viewportWidthAtCaptureStart > 0 ? snapshotCanvas.width / viewportWidthAtCaptureStart : requestedDpr;
+  const fitScale = Math.min(width / snapshotCanvas.width, height / snapshotCanvas.height);
+  const destWidth = snapshotCanvas.width * fitScale;
+  const destHeight = snapshotCanvas.height * fitScale;
   const destX = (width - destWidth) / 2;
   const destY = (height - destHeight) / 2;
   const layer = document.createElement("canvas");
@@ -5854,10 +5965,14 @@ async function rasterizeViewportToPageLayer(width, height) {
     throw new Error("Failed to get 2D context for page layer");
   layerCtx.fillStyle = "#ffffff";
   layerCtx.fillRect(0, 0, width, height);
-  layerCtx.drawImage(snapshotCanvas, sourceX, sourceY, sourceWidth, sourceHeight, destX, destY, destWidth, destHeight);
+  layerCtx.drawImage(snapshotCanvas, destX, destY, destWidth, destHeight);
   return {
     canvas: layer,
-    viewportToCanvas: { scale: dpr * fitScale, offsetX: destX, offsetY: destY }
+    viewportToCanvas: {
+      scale: canvasPixelsPerCssPixel * fitScale,
+      offsetX: destX,
+      offsetY: destY
+    }
   };
 }
 function compositePageAndCursorOverlay(ctx, canvas, cachedPage, brand) {
@@ -5881,6 +5996,12 @@ async function gzipIfBeneficial(snapshotBytes) {
     return { bytes: snapshotBytes, compressionFlag: 0 };
   }
 }
+function frameWithCompressionFlag(compressionFlag, payload) {
+  const frame = new Uint8Array(payload.byteLength + 1);
+  frame[0] = compressionFlag;
+  frame.set(payload, 1);
+  return frame;
+}
 async function buildDomSnapshotFrame() {
   const a11yTree = buildAccessibilityTree();
   const snapshotJson = JSON.stringify({
@@ -5893,14 +6014,12 @@ async function buildDomSnapshotFrame() {
   });
   const snapshotBytes = textEncoder4.encode(snapshotJson);
   const { bytes, compressionFlag } = await gzipIfBeneficial(snapshotBytes);
-  const frame = new Uint8Array(bytes.byteLength + 1);
-  frame[0] = compressionFlag;
-  frame.set(bytes, 1);
-  return frame;
+  return frameWithCompressionFlag(compressionFlag, bytes);
 }
 function reportCaptureError(localParticipant) {
   try {
-    localParticipant.publishData(textEncoder4.encode(JSON.stringify({ type: "capture_error" })), {
+    const payload = textEncoder4.encode(JSON.stringify({ type: "capture_error" }));
+    localParticipant.publishData(frameWithCompressionFlag(0, payload), {
       reliable: true,
       topic: DOM_SNAPSHOT_TOPIC
     }).catch(() => {
@@ -5943,12 +6062,18 @@ function DomCapture({ pushOrTapMicMode = false }) {
       return;
     videoTrack.contentHint = "text";
     didStartRef.current = true;
+    preCacheSnapshotResources();
     let cancelled = false;
     let snapshotInFlight = false;
     let a11yPublishInFlight = false;
     let a11yPublishPending = false;
     let consecutiveCaptureFailures = 0;
     let cachedPage = null;
+    let rasterizeSequence = 0;
+    let lastCommittedSequence = 0;
+    let firstFrameCommitted = false;
+    let capturesUntilFullQualityRetry = 0;
+    let consecutiveFullQualityTimeouts = 0;
     if (!pushOrTapMicMode && !localParticipant.isMicrophoneEnabled) {
       localParticipant.setMicrophoneEnabled(true).catch((error) => console.error("Failed to enable microphone:", error));
     }
@@ -5964,16 +6089,61 @@ function DomCapture({ pushOrTapMicMode = false }) {
       for (const track of canvasStream.getTracks())
         track.stop();
     });
-    const refreshPageAndComposite = async () => {
-      if (cancelled || snapshotInFlight || pausedRef.current)
+    const commitPageLayerIfNewest = (sequence, page) => {
+      if (cancelled || pausedRef.current || sequence <= lastCommittedSequence)
+        return false;
+      lastCommittedSequence = sequence;
+      cachedPage = page;
+      compositePageAndCursorOverlay(ctx, canvas, page, brandRef.current);
+      firstFrameCommitted = true;
+      return true;
+    };
+    const rasterizeAndCommitWaitingOutTimeout = async (options) => {
+      const sequence = ++rasterizeSequence;
+      const captureInProgress = rasterizeViewportToPageLayer(canvas.width, canvas.height, options);
+      const captureOrTimeout = await Promise.race([
+        captureInProgress,
+        new Promise((resolve) => setTimeout(() => resolve(RASTERIZE_TIMED_OUT), RASTERIZE_TIMEOUT_MS))
+      ]);
+      if (captureOrTimeout === RASTERIZE_TIMED_OUT) {
+        captureInProgress.then((lateCapture) => commitPageLayerIfNewest(sequence, lateCapture)).catch(() => {
+          return;
+        });
+        return { timedOut: true };
+      }
+      commitPageLayerIfNewest(sequence, captureOrTimeout);
+      return { timedOut: false };
+    };
+    let followUpRefreshQueued = false;
+    const refreshPageAndComposite = async ({ queueWhenBusy = false } = {}) => {
+      if (cancelled || snapshotInFlight || pausedRef.current) {
+        if (snapshotInFlight && !cancelled && queueWhenBusy)
+          followUpRefreshQueued = true;
         return;
+      }
       snapshotInFlight = true;
       try {
-        cachedPage = await rasterizeViewportToPageLayer(canvas.width, canvas.height);
-        if (cancelled || pausedRef.current)
-          return;
-        compositePageAndCursorOverlay(ctx, canvas, cachedPage, brandRef.current);
+        const isImagelessFirstFrame = !firstFrameCommitted;
+        const useFullQuality = firstFrameCommitted && capturesUntilFullQualityRetry === 0;
+        const { timedOut } = await rasterizeAndCommitWaitingOutTimeout({
+          skipCrossOriginImageSettling: !useFullQuality,
+          includeImages: !isImagelessFirstFrame
+        });
+        if (isImagelessFirstFrame && firstFrameCommitted)
+          followUpRefreshQueued = true;
         consecutiveCaptureFailures = 0;
+        if (useFullQuality) {
+          if (timedOut) {
+            consecutiveFullQualityTimeouts += 1;
+            if (consecutiveFullQualityTimeouts >= FULL_QUALITY_TIMEOUTS_BEFORE_COOLDOWN) {
+              capturesUntilFullQualityRetry = FAST_CAPTURES_BEFORE_FULL_QUALITY_RETRY;
+            }
+          } else {
+            consecutiveFullQualityTimeouts = 0;
+          }
+        } else if (capturesUntilFullQualityRetry > 0) {
+          capturesUntilFullQualityRetry -= 1;
+        }
       } catch {
         if (cancelled)
           return;
@@ -5983,6 +6153,10 @@ function DomCapture({ pushOrTapMicMode = false }) {
         }
       } finally {
         snapshotInFlight = false;
+        if (followUpRefreshQueued && !cancelled) {
+          followUpRefreshQueued = false;
+          refreshPageAndComposite();
+        }
       }
     };
     let overlayAnimationFrame = null;
@@ -6048,20 +6222,30 @@ function DomCapture({ pushOrTapMicMode = false }) {
     });
     const cleanupSnapshotRequest = onSnapshotRequest(() => {
       tickA11yPublish();
-      refreshPageAndComposite();
+      refreshPageAndComposite({ queueWhenBusy: true });
     });
+    let scrollSettleTimer = null;
+    const refreshOnceScrollSettles = () => {
+      if (scrollSettleTimer)
+        clearTimeout(scrollSettleTimer);
+      scrollSettleTimer = setTimeout(() => {
+        tickA11yPublish();
+        refreshPageAndComposite({ queueWhenBusy: true });
+      }, SCROLL_SETTLE_MS);
+    };
+    window.addEventListener("scroll", refreshOnceScrollSettles, { passive: true, capture: true });
     let snapshotTimer = null;
     const scheduleNextSnapshot = (delay) => {
       if (cancelled)
         return;
-      snapshotTimer = setTimeout(async () => {
-        await refreshPageAndComposite();
+      snapshotTimer = setTimeout(() => {
+        refreshPageAndComposite();
         scheduleNextSnapshot(SNAPSHOT_INTERVAL_MS);
       }, delay);
     };
     scheduleNextSnapshot(FIRST_SNAPSHOT_DELAY_MS);
     tickA11yPublish();
-    const a11yPublishTimer = setInterval(tickA11yPublish, A11Y_PUBLISH_INTERVAL_MS);
+    const a11yPublishTimer = setInterval(() => void tickA11yPublish(), A11Y_PUBLISH_INTERVAL_MS);
     freezeCaptureRef.current = () => {
       pausedRef.current = true;
       drawPausedOverlay(canvas, ctx);
@@ -6078,6 +6262,9 @@ function DomCapture({ pushOrTapMicMode = false }) {
       resumeCaptureRef.current = null;
       if (snapshotTimer)
         clearTimeout(snapshotTimer);
+      if (scrollSettleTimer)
+        clearTimeout(scrollSettleTimer);
+      window.removeEventListener("scroll", refreshOnceScrollSettles, { capture: true });
       clearInterval(a11yPublishTimer);
       if (overlayAnimationFrame !== null)
         cancelAnimationFrame(overlayAnimationFrame);