@qualified-salesforce/web 0.0.1-oidc-bootstrap

package/widget/multimodal_v2/stores/screen_share_store.js

@@ -0,0 +1,217 @@
+import { __webpack_require__ } from "../../../js/runtime.js";
+import { signals_core_module_E, signals_core_module_w } from "../../../js/multimodal_v2.js";
+import { getSentryClient } from "../../widget/sentry.js";
+import "../../utils/native_globals.js";
+
+/* provided dependency */ var screen_share_store_setInterval = __webpack_require__(68034).LK;
+/* provided dependency */ var screen_share_store_clearInterval = __webpack_require__(68034).wm;
+
+
+// Cap the longest side so the encoded JPEG stays small enough to travel through
+// ActionCable -> Redis -> the OpenAI websocket comfortably, while remaining
+// readable (text + faces) for the agent.
+const MAX_DIMENSION = 1280;
+const JPEG_QUALITY = 0.7;
+// Capping the longest side does not cap the encoded size: a dense, high-entropy slide can still
+// clear a megabyte at 1280px/0.7. Anything above coffee's budget is dropped there (and a frame
+// over 1 MiB never even leaves ActionCable), which silently freezes perception on a stale slide,
+// so re-encode down a quality-then-resolution ladder until it fits. Mirrors
+// coffee/src/new_openai_realtime/frames.py, which enforces the same budget on its own frames.
+const MAX_DATA_URL_BYTES = 900000;
+const QUALITY_LADDER = [
+    JPEG_QUALITY,
+    0.6,
+    0.5,
+    0.4
+];
+// How many times the ladder halves the resolution before accepting whatever it has.
+const MAX_SHRINK_ITERATIONS = 5;
+// coffee expires the slide perception slot after 30s without a slide frame refresh (so page
+// frames can reclaim the slot when a slide-clear never arrives, e.g. on a visitor reload).
+// While a slide stays on screen we therefore re-push the same frame comfortably under that
+// staleness window so a long-displayed slide doesn't go stale server-side.
+const SLIDE_KEEPALIVE_INTERVAL_MS = 15000;
+class ScreenShareStore {
+    dispose() {
+        this.disposeEffect == null ? void 0 : this.disposeEffect.call(this);
+        this.disposeEffect = null;
+        this.stopKeepalive();
+        // The store can go away while the channel and the call live on -- a session handed over to the
+        // legacy messenger tears the multimodal UI down and keeps both (see Multimodal#teardownUI). Stopping
+        // the keepalive is not enough then: coffee keeps the slide we last pushed for its full 30s staleness
+        // window and rejects page frames while it owns the slot, so hand the slot back on the way out.
+        this.clearPerception();
+    }
+    // Reactively push the current slide (on change) and clear perception when no slide is shown or
+    // the call ends.
+    sync() {
+        // Server-side perception is disabled without the flag, so don't capture or ship frames.
+        if (!this.rootStore.featureFlags.piperPerceptionSlides) return;
+        const slideUrl = this.currentSlideImageUrl.value;
+        const inCall = this.rootStore.callInProgress.value;
+        if (!inCall) {
+            this.clearPerception();
+            return;
+        }
+        if (slideUrl) {
+            if (slideUrl !== this.lastPushedKey && slideUrl !== this.pendingKey) {
+                void this.pushSlide(slideUrl);
+            }
+            return;
+        }
+        // No slide on screen: clear whatever slide we last pushed.
+        this.clearPerception();
+    }
+    async pushSlide(url) {
+        this.pendingKey = url;
+        try {
+            const image = await this.loadImage(url);
+            if (this.currentSlideImageUrl.peek() !== url) return; // superseded while loading
+            // The call may have ended while the image was loading; don't push after call end
+            // (the !inCall branch of sync() has already cleared perception).
+            if (!this.rootStore.callInProgress.peek()) return;
+            const imageData = this.encodeImage(image);
+            if (!imageData) {
+                this.retractOnFailure(url);
+                return;
+            }
+            this.push(imageData);
+            this.lastPushedKey = url;
+            this.lastPushedImageData = imageData;
+            this.startKeepalive();
+        } catch (error) {
+            var _getSentryClient;
+            (_getSentryClient = getSentryClient({
+                project: "multimodal"
+            })) == null ? void 0 : _getSentryClient.captureException(error);
+            this.retractOnFailure(url);
+        } finally{
+            if (this.pendingKey === url) this.pendingKey = null;
+        }
+    }
+    // A slide that failed to load or encode leaves the *previous* slide pushed, with the keepalive
+    // actively refreshing it — so the agent keeps describing content the visitor has already moved
+    // on from, and coffee's staleness expiry never fires because the frame keeps arriving. Retract
+    // it: showing the agent nothing beats showing it something confidently wrong.
+    retractOnFailure(url) {
+        if (this.currentSlideImageUrl.peek() !== url) return; // already superseded; leave it alone
+        this.clearPerception();
+    }
+    loadImage(url) {
+        return new Promise((resolve, reject)=>{
+            const image = new Image();
+            image.crossOrigin = "anonymous";
+            image.onload = ()=>resolve(image);
+            image.onerror = ()=>reject(new Error("Failed to load slide image: " + url));
+            image.src = url;
+        });
+    }
+    encodeImage(image) {
+        const imageWidth = image.width || MAX_DIMENSION;
+        const imageHeight = image.height || MAX_DIMENSION;
+        const scale = Math.min(MAX_DIMENSION / imageWidth, MAX_DIMENSION / imageHeight, 1);
+        const canvas = document.createElement("canvas");
+        canvas.width = Math.max(1, Math.round(imageWidth * scale));
+        canvas.height = Math.max(1, Math.round(imageHeight * scale));
+        const ctx = canvas.getContext("2d");
+        if (!ctx || typeof canvas.toDataURL !== "function") return null;
+        ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
+        // Drop quality first (cheap, keeps the slide legible), then resolution for the rare frame
+        // that is still too big at the lowest quality.
+        for (const quality of QUALITY_LADDER){
+            const dataUrl = canvas.toDataURL("image/jpeg", quality);
+            if (dataUrl.length <= MAX_DATA_URL_BYTES) return dataUrl;
+        }
+        let dataUrl = canvas.toDataURL("image/jpeg", QUALITY_LADDER[QUALITY_LADDER.length - 1]);
+        for(let i = 0; i < MAX_SHRINK_ITERATIONS && dataUrl.length > MAX_DATA_URL_BYTES; i++){
+            const halved = document.createElement("canvas");
+            halved.width = Math.max(1, Math.floor(canvas.width / 2));
+            halved.height = Math.max(1, Math.floor(canvas.height / 2));
+            const halvedCtx = halved.getContext("2d");
+            if (!halvedCtx) break;
+            halvedCtx.drawImage(canvas, 0, 0, halved.width, halved.height);
+            canvas.width = halved.width;
+            canvas.height = halved.height;
+            ctx.drawImage(halved, 0, 0);
+            dataUrl = canvas.toDataURL("image/jpeg", QUALITY_LADDER[QUALITY_LADDER.length - 1]);
+        }
+        if (dataUrl.length > MAX_DATA_URL_BYTES) return null;
+        return dataUrl;
+    }
+    push(imageData) {
+        this.rootStore.channel.perform("update_perception", {
+            image_data: imageData,
+            kind: "slide"
+        });
+    }
+    // Tell the agent no slide is on screen anymore (only if we'd sent something).
+    clearPerception() {
+        if (!this.lastPushedKey) return;
+        this.lastPushedKey = null;
+        this.lastPushedImageData = null;
+        this.stopKeepalive();
+        this.rootStore.channel.perform("update_perception", {
+            image_data: ""
+        });
+    }
+    // Periodically re-push the on-screen slide so coffee's slide perception slot doesn't expire
+    // (see SLIDE_KEEPALIVE_INTERVAL_MS). Deliberately bypasses the on-change dedupe: the point is
+    // to refresh the *same* frame. Stopped on slide removal / call end (clearPerception) and dispose.
+    startKeepalive() {
+        this.stopKeepalive();
+        this.keepaliveTimerId = screen_share_store_setInterval(()=>{
+            // Belt-and-braces: the timer is cleared on slide removal and call end, but re-check in
+            // case a tick races those transitions.
+            if (!this.lastPushedImageData || !this.rootStore.callInProgress.peek()) return;
+            this.push(this.lastPushedImageData);
+        }, SLIDE_KEEPALIVE_INTERVAL_MS);
+    }
+    stopKeepalive() {
+        if (this.keepaliveTimerId != null) screen_share_store_clearInterval(this.keepaliveTimerId);
+        this.keepaliveTimerId = null;
+    }
+    constructor(rootStore){
+        this.rootStore = rootStore;
+        // The slide image currently shown in the artifact overlay (deck or single slide),
+        // or null when no slide is being presented.
+        this.currentSlideImageUrl = signals_core_module_w(()=>{
+            var _deck_slideImageUrls_deck_currentIndex, _slide_url;
+            var _this_rootStore_decksStore_value, _this_rootStore_slidesStore_value;
+            const deck = (_this_rootStore_decksStore_value = this.rootStore.decksStore.value) == null ? void 0 : _this_rootStore_decksStore_value.currentDeck.value;
+            if (deck) return (_deck_slideImageUrls_deck_currentIndex = deck.slideImageUrls[deck.currentIndex]) != null ? _deck_slideImageUrls_deck_currentIndex : null;
+            const slide = (_this_rootStore_slidesStore_value = this.rootStore.slidesStore.value) == null ? void 0 : _this_rootStore_slidesStore_value.currentSlide.value;
+            if (slide) return (_slide_url = slide.url) != null ? _slide_url : null;
+            return null;
+        });
+        this.disposeEffect = null;
+        // The slide URL we last pushed, so we only send on change and know when to clear.
+        this.lastPushedKey = null;
+        // The slide URL currently loading/encoding for a push. sync() can re-run while pushSlide is
+        // still awaiting the image load; without this an in-flight URL would be pushed twice.
+        this.pendingKey = null;
+        // The encoded payload we last pushed, so the keepalive can re-send it without re-encoding.
+        this.lastPushedImageData = null;
+        this.keepaliveTimerId = null;
+        this.disposeEffect = signals_core_module_E(()=>this.sync());
+    }
+}
+/**
+ * Mirrors the slide/deck currently shown in the artifact overlay to the voice agent's perception
+ * loop, as a base64 JPEG pushed to the backend via `update_perception` (Rails relays it to coffee's
+ * Redis `perception_frame`, which injects it).
+ *
+ * This store only handles the *slide* source. Live-page perception is handled server-side by the
+ * headless liveview-renderer (which replays the visitor's co-browse DOM stream and screenshots it),
+ * so the widget no longer captures the page itself — that previously required a `getDisplayMedia`
+ * prompt and the browser's "sharing this tab" indicator.
+ *
+ * coffee keeps exactly one perception image (slides take precedence over page frames there), so this
+ * store pushes the current slide when it changes and pushes a blank payload to clear it when no slide
+ * is on screen.
+ *
+ * Gated on the `piper_perception_slides` feature flag (exposed to the widget as
+ * `piperPerceptionSlides`): when it's off the server drops perception frames anyway, so we skip the
+ * canvas encode + ~100-500KB base64 push entirely.
+ */ 
+
+export default ScreenShareStore;

package/widget/multimodal_v2/stores/slides_store.js

@@ -0,0 +1,21 @@
+import { signals_core_module_w } from "../../../js/multimodal_v2.js";
+
+
+class SlidesStore {
+    // eslint-disable-next-line no-useless-constructor
+    constructor(rootStore){
+        this.rootStore = rootStore;
+        // Derived state
+        this.currentSlide = signals_core_module_w(()=>{
+            const message = this.rootStore.artifact.message.value;
+            if (!message || message.type !== "present_content") return null;
+            return {
+                slideId: message.slideId,
+                url: message.url
+            };
+        });
+    }
+}
+
+
+export default SlidesStore;

package/widget/multimodal_v2/stores/translation_store.js

@@ -0,0 +1,185 @@
+import { signals_core_module_d, signals_core_module_w } from "../../../js/multimodal_v2.js";
+import { importLocale, isSupported } from "../locales/index.js";
+import { useRootStore } from "./root_store.js";
+
+
+
+
+const DEFAULT_LOCALE = "en";
+/**
+ * Flatten a nested translations object into a Map with dot-separated keys.
+ * { chat: { enterMessage: "..." } } -> Map { "chat.enterMessage" => "..." }
+ *
+ * Exported for use in test utilities.
+ */ function flattenTranslations(obj, prefix) {
+    if (prefix === void 0) prefix = "";
+    const result = new Map();
+    for (const [key, value] of Object.entries(obj)){
+        const fullKey = prefix ? prefix + "." + key : key;
+        if (typeof value === "string") {
+            result.set(fullKey, value);
+        } else if (value && typeof value === "object") {
+            // Recursively flatten nested objects
+            const nested = flattenTranslations(value, fullKey);
+            for (const [nestedKey, nestedValue] of nested){
+                result.set(nestedKey, nestedValue);
+            }
+        }
+    }
+    return result;
+}
+/**
+ * Translation store for the multimodal widget.
+ *
+ * Responsibilities:
+ * - Lazy loads locale files on demand
+ * - Computes effective locale from conversation → script → browser → default
+ * - Provides `t()` function for translating keys with field substitution
+ *
+ * @example
+ * // In a component:
+ * const { translationStore: { t } } = rootStore;
+ * t("chat.enterMessage")
+ * t("chat.askAnything", { name: "Ada" })
+ */ class TranslationStore {
+    // ---------------------------------------------------------------------------
+    // Actions
+    // ---------------------------------------------------------------------------
+    initialize() {
+        // Load default locale immediately
+        void this.loadLocale(DEFAULT_LOCALE);
+        // Subscribe to locale changes and load the appropriate translations
+        this.effectiveLocale.subscribe((locale)=>{
+            void this.loadLocale(locale);
+        });
+    }
+    /**
+   * Set the script locale (from Experience settings).
+   */ setScriptLocale(locale) {
+        this.scriptLocale.value = locale != null ? locale : null;
+    }
+    /**
+   * O(1) lookup of a translation key from the pre-normalized dictionary.
+   */ getTranslation(key) {
+        var _ref;
+        var _this_dictionary_value;
+        return (_ref = (_this_dictionary_value = this.dictionary.value) == null ? void 0 : _this_dictionary_value.get(key)) != null ? _ref : null;
+    }
+    /**
+   * Replace {{field}} placeholders with values.
+   * "Hello {{name}}" + { name: "Ada" } -> "Hello Ada"
+   */ substituteFields(template, fields) {
+        let result = template;
+        for (const [key, value] of Object.entries(fields)){
+            result = result.replace(new RegExp("\\{\\{" + key + "\\}\\}", "g"), String(value));
+        }
+        return result;
+    }
+    // ---------------------------------------------------------------------------
+    // Private Helpers
+    // ---------------------------------------------------------------------------
+    /**
+   * Normalize browser language code to our locale format.
+   * e.g., "en-US" -> "en", "zh-CN" -> "zh_CN"
+   */ browserLanguageToLocale(language) {
+        switch(language){
+            case "zh-CN":
+                return "zh_CN";
+            case "zh-TW":
+                return "zh_TW";
+            case "es-419":
+                return "es_LA";
+            case "pt-BR":
+                return "pt_BR";
+            case "en-GB":
+                return "en_GB";
+            case "en-AU":
+                return "en_AU";
+            default:
+                // Use first part of language code: "en-US" -> "en"
+                return language.split("-")[0];
+        }
+    }
+    async loadLocale(locale) {
+        // Use cached dictionary if available
+        if (this.cache.has(locale)) {
+            this.dictionary.value = this.cache.get(locale);
+            return;
+        }
+        // Prevent duplicate loading requests
+        if (this.loading.has(locale)) return;
+        this.loading.add(locale);
+        try {
+            const module = await importLocale(locale);
+            const flatDictionary = flattenTranslations(module.default);
+            this.cache.set(locale, flatDictionary);
+            // Only update if this is still the current locale
+            if (this.effectiveLocale.value === locale) {
+                this.dictionary.value = flatDictionary;
+            }
+        } finally{
+            this.loading.delete(locale);
+        }
+    }
+    constructor(rootStore){
+        // ---------------------------------------------------------------------------
+        // State
+        // ---------------------------------------------------------------------------
+        /** Cache of pre-normalized translation dictionaries for instant lookups */ this.cache = new Map();
+        /** Set of currently loading locales */ this.loading = new Set();
+        /** Script-configured locale (from Experience settings) */ this.scriptLocale = signals_core_module_d(null);
+        /** Current flattened translation dictionary (updates when locale changes) */ this.dictionary = signals_core_module_d(null);
+        // ---------------------------------------------------------------------------
+        // Derived State
+        // ---------------------------------------------------------------------------
+        /** Conversation locale (raw value, may not be supported) */ this.conversationLocale = signals_core_module_w(()=>{
+            var _ref;
+            var _this_rootStore_activeConversation_value;
+            return (_ref = (_this_rootStore_activeConversation_value = this.rootStore.activeConversation.value) == null ? void 0 : _this_rootStore_activeConversation_value.locale) != null ? _ref : null;
+        });
+        /** Browser locale (raw value, may not be supported) */ this.browserLocale = signals_core_module_w(()=>{
+            return this.browserLanguageToLocale(navigator.language);
+        });
+        /**
+   * Effective locale with priority (only uses supported locales):
+   * 1. Conversation locale (can change based on AI SDR conversation with a visitor)
+   * 2. Script locale (configured in Experience)
+   * 3. Browser locale (detected from browser settings)
+   * 4. Default locale (en)
+   */ this.effectiveLocale = signals_core_module_w(()=>{
+            const conversation_locale = this.conversationLocale.value;
+            if (conversation_locale && isSupported(conversation_locale)) return conversation_locale;
+            const script_locale = this.scriptLocale.value;
+            if (script_locale && isSupported(script_locale)) return script_locale;
+            const browser_locale = this.browserLocale.value;
+            if (isSupported(browser_locale)) return browser_locale;
+            return DEFAULT_LOCALE;
+        });
+        /**
+   * Translate a key with optional field substitution.
+   *
+   * @param key - Dot-separated key path (e.g., "chat.enterMessage")
+   * @param fields - Optional values to substitute (e.g., { name: "Ada" })
+   * @returns Translated string, or empty string if not found
+   *
+   * @example
+   * t("chat.enterMessage")           // "Enter a message"
+   * t("chat.askAnything", { name })  // "Ask Ada anything…"
+   */ this.t = (key, fields)=>{
+            const translation = this.getTranslation(key);
+            if (translation === null) return "";
+            return fields ? this.substituteFields(translation, fields) : translation;
+        };
+        this.rootStore = rootStore;
+    }
+}
+/* export default */ const translation_store = ((/* unused pure expression or super */ null && (TranslationStore)));
+function useTranslationStore() {
+    var _useRootStore;
+    return (_useRootStore = useRootStore()) == null ? void 0 : _useRootStore.translationStore;
+}
+
+var translation_store_default = /* unused export */ undefined;
+var translation_store_flattenTranslations = /* unused export */ undefined;
+export { DEFAULT_LOCALE, TranslationStore, translation_store_flattenTranslations as flattenTranslations, useTranslationStore };
+export default translation_store_default;