@snowcone-app/ui 0.4.9 → 0.4.11

package/src/patterns/RealtimeProvider.tsx

@@ -38,11 +38,20 @@ import React, {
 } from "react";
 import { useRealtimeMockup } from "@snowcone-app/sdk/react";
 import { resolveVariantId as resolveVariantIdUtil } from "@snowcone-app/sdk";
-import type { RealtimeRenderError } from "@snowcone-app/sdk";
+import type {
+  RealtimeRenderError,
+  MockupResult as SdkMockupResult,
+} from "@snowcone-app/sdk";
 import { readEnv } from "../lib/env";
 import { useProductOptional } from "./Product";
 import type { ReactProductContext } from "./Product";
 import { useMockupBackgroundOptional } from "./MockupBackground";
+import {
+  allBackgroundsSeeded,
+  compositeCroppedFrame,
+  isFullRect,
+  loadImageElement,
+} from "./croppedFrameComposite";
 
 // Default artboard size for export calculations (matches @snowcone-app/canvas default)
 const DEFAULT_ARTBOARD_SIZE = 2000;
@@ -276,6 +285,18 @@ export interface RealtimeProviderProps {
    * via a renewable token. Without them, the legacy seal path runs (shim).
    */
   grantUrl?: string;
+  /**
+   * Frame delivery strategy. `"cropped"` (default, the fast path): once every
+   * configured mockup has rendered one full frame (frozen internally as its
+   * background), the session flips to `overlayCrop: 'auto'` — the server ships
+   * only the product-region pixels per edit and the provider composites them
+   * over the frozen background before notifying consumers, so subscribers and
+   * the URL cache only ever see full-frame images. Opt out with
+   * `"full-frame"` to ship the whole scene on every render (the pre-crop
+   * behavior). Any composite failure falls back to full-frame delivery for
+   * the rest of the session automatically.
+   */
+  delivery?: "cropped" | "full-frame";
 }
 
 /**
@@ -292,6 +313,7 @@ export function RealtimeProvider({
   autoEnable = false,
   shop,
   grantUrl,
+  delivery = "cropped",
 }: RealtimeProviderProps) {
   const productContext = useProductOptional();
   // Page background for renders (see MockupBackground) — sent on the config so
@@ -472,74 +494,318 @@ export function RealtimeProvider({
   // Track which mockupIds are pending (derived from placements)
   const pendingMockupIdsRef = useRef<Set<string>>(new Set());
 
-  // Sync rawMockupResults to ref, update URL cache, and notify subscribers (NO STATE UPDATES)
-  useEffect(() => {
-    if (rawMockupResults.length === 0) {
-      if (mockupResultsRef.current.length > 0) {
-        mockupResultsRef.current = [];
-        mockupSubscribersRef.current.forEach((cb) => cb([]));
+  // ============================================================================
+  // CROPPED-FRAME DELIVERY (internal — consumers only ever see full frames)
+  // ============================================================================
+
+  // Frozen full-frame background per mockup (the composite base for cropped
+  // frames). Values are server URLs, never blob URLs.
+  const backgroundsRef = useRef<
+    Map<string, { url: string; imgPromise?: Promise<HTMLImageElement> }>
+  >(new Map());
+  // Whether the last-sent config included overlayCrop: 'auto'.
+  const croppedActiveRef = useRef(false);
+  // Composite kill-switch: any composite failure disables cropped delivery
+  // for the rest of the session (full-frame is always correct, just heavier).
+  const cropCapableRef = useRef(true);
+  // Last config payload sent, WITHOUT overlayCrop — flip/unflip re-sends it.
+  const lastConfigBaseRef = useRef<Record<string, unknown> | null>(null);
+  // Scene generation — bumped when a config change invalidates backgrounds
+  // (variant/bg). In-flight composites from an older scene are dropped.
+  const sceneGenRef = useRef(0);
+  // Frame URL last processed per mockup. The sdk hook's results array is
+  // cumulative state (not a delta), so this gates background seeding and
+  // composite work across effect re-runs. Cleared on scene invalidation —
+  // safe because the sdk clears mockupResults on every config send, so an
+  // old-scene frame is never re-observed after the config that replaced it;
+  // clearing is what lets a reconnect re-seed from identical (render-cached)
+  // frame URLs.
+  const processedFrameRef = useRef<Map<string, string>>(new Map());
+  // Latest-wins composite queue per mockup + in-flight tracking. Frames can
+  // stream faster than a device composites; only the newest matters.
+  const compositeQueueRef = useRef<Map<string, SdkMockupResult>>(new Map());
+  const compositeBusyRef = useRef<Set<string>>(new Set());
+  const compositeCanvasRef = useRef<HTMLCanvasElement | null>(null);
+
+  /**
+   * Register a blob URL with lifecycle management
+   */
+  const registerBlobUrl = useCallback((blobOrUrl: Blob | string, placementName?: string): string => {
+    let url: string;
+
+    if (blobOrUrl instanceof Blob) {
+      // Enforce blob URL limit before creating new one
+      if (registeredBlobUrlsRef.current.size >= MAX_CONCURRENT_BLOBS) {
+        // Revoke oldest blob URL (first entry in Set maintains insertion order)
+        const oldest = registeredBlobUrlsRef.current.values().next().value;
+        if (oldest) {
+          URL.revokeObjectURL(oldest);
+          registeredBlobUrlsRef.current.delete(oldest);
+
+          // Also remove from URL cache and the results snapshot if present, so
+          // late subscribers are never replayed a revoked blob URL.
+          mockupUrlCacheRef.current.forEach((value, key) => {
+            if (value.url === oldest) {
+              mockupUrlCacheRef.current.delete(key);
+            }
+          });
+          mockupResultsRef.current = mockupResultsRef.current.filter(
+            (r) => r.imageUrl !== oldest
+          );
+        }
       }
-      return;
+
+      url = URL.createObjectURL(blobOrUrl);
+      registeredBlobUrlsRef.current.add(url);
+    } else {
+      url = blobOrUrl;
     }
 
-    console.log(`[RTP] rawMockupResults changed: ${rawMockupResults.length} result(s)`, rawMockupResults.map(r => `${r.mockupId}${r.imageUrl ? ' ✓url' : ' ✗NO-URL'}`));
+    return url;
+  }, []);
 
-    // Update ref immediately
-    mockupResultsRef.current = rawMockupResults;
+  const croppedDeliveryEnabled = delivery !== "full-frame";
+
+  /** Re-send the last config with/without overlayCrop (delivery flip only). */
+  const sendCropConfig = useCallback(
+    (withCrop: boolean) => {
+      const base = lastConfigBaseRef.current;
+      if (!base) return;
+      croppedActiveRef.current = withCrop;
+      sendConfigRaw({
+        ...base,
+        placementSettings: { ...placementSettingsOverridesRef.current },
+        ...(withCrop ? { overlayCrop: "auto" as const } : {}),
+      } as Parameters<typeof sendConfigRaw>[0]);
+    },
+    [sendConfigRaw]
+  );
 
-    // Update centralized URL cache with new results
-    const now = Date.now();
-    rawMockupResults.forEach((result) => {
-      if (result.imageUrl) {
+  /** Drop back to full-frame delivery (always correct, just heavier). */
+  const unflipCrop = useCallback(() => {
+    if (!croppedActiveRef.current) return;
+    sendCropConfig(false);
+  }, [sendCropConfig]);
+
+  /** Flip to cropped delivery once every configured mockup has a frozen
+   *  background (a cropped frame can't seed one). */
+  const maybeFlipCrop = useCallback(() => {
+    if (!croppedDeliveryEnabled || !cropCapableRef.current) return;
+    if (croppedActiveRef.current) return;
+    const base = lastConfigBaseRef.current;
+    if (!base) return;
+    const ids = (base.mockupIds as string[] | undefined) ?? [];
+    const seeded = new Set(backgroundsRef.current.keys());
+    if (allBackgroundsSeeded(ids, seeded)) {
+      console.log(`[RTP] Cropped delivery ON — all ${ids.length} mockup background(s) seeded`);
+      sendCropConfig(true);
+    } else {
+      const missing = ids.filter((id) => !seeded.has(id));
+      console.log(`[RTP] Cropped delivery waiting — ${missing.length}/${ids.length} mockup(s) unseeded: [${missing.slice(0, 8).join(", ")}]`);
+    }
+  }, [croppedDeliveryEnabled, sendCropConfig]);
+
+  /** Scene changed (variant/bg) or session reset — every frozen background
+   *  is stale. */
+  const invalidateBackgrounds = useCallback(() => {
+    sceneGenRef.current += 1;
+    backgroundsRef.current.clear();
+    processedFrameRef.current.clear();
+    compositeQueueRef.current.clear();
+    croppedActiveRef.current = false;
+  }, []);
+
+  /**
+   * Deliver displayable (always full-frame) results: update the snapshot and
+   * URL cache, clear pending state, notify subscribers, check settle.
+   */
+  const deliverResults = useCallback(
+    (results: SdkMockupResult[]) => {
+      if (results.length === 0) return;
+      const now = Date.now();
+
+      // Upsert into the results snapshot (latest per mockup)
+      const merged = [...mockupResultsRef.current];
+      results.forEach((result) => {
+        const idx = merged.findIndex((r) => r.mockupId === result.mockupId);
+        if (idx >= 0) merged[idx] = result;
+        else merged.push(result);
+      });
+      mockupResultsRef.current = merged;
+
+      const clearedPlacements = new Set<string>();
+      results.forEach((result) => {
         const wasPending = pendingMockupIdsRef.current.has(result.mockupId);
         mockupUrlCacheRef.current.set(result.mockupId, {
           url: result.imageUrl,
-          isBlob: false, // Server URLs are not blob URLs
+          isBlob: result.imageUrl.startsWith("blob:"),
           timestamp: now,
         });
-
-        // Clear pending state for this mockup
         pendingMockupIdsRef.current.delete(result.mockupId);
         if (wasPending) {
           console.log(`[RTP] Cleared pending for ${result.mockupId} — url cached`);
         }
+        if (result.placement) clearedPlacements.add(result.placement);
+      });
+
+      // Clear pending placement state so notifyPendingExport can re-fire on
+      // the next edit. Placement-scoped when known, otherwise clear all.
+      if (clearedPlacements.size > 0) {
+        clearedPlacements.forEach((p) => pendingPlacementsRef.current.delete(p));
       } else {
-        console.log(`[RTP] Result for ${result.mockupId} has NO imageUrl — not caching, pending NOT cleared`);
+        pendingPlacementsRef.current.clear();
       }
-    });
 
-    // Clear pending placement state so notifyPendingExport can re-fire on next edit.
-    // We clear based on result placements when available, otherwise clear all.
-    const clearedPlacements = new Set<string>();
-    rawMockupResults.forEach((result) => {
-      if (result.placement) clearedPlacements.add(result.placement);
-    });
-    if (clearedPlacements.size > 0) {
-      clearedPlacements.forEach((p) => pendingPlacementsRef.current.delete(p));
-    } else {
-      pendingPlacementsRef.current.clear();
+      // Notify subscribers immediately (no throttle needed - no state updates!)
+      mockupSubscribersRef.current.forEach((cb) => cb(results));
+
+      // Notify filtered subscribers
+      results.forEach((result) => {
+        const subscribers = mockupSubscribersByIdRef.current.get(result.mockupId);
+        if (subscribers && subscribers.size > 0) {
+          console.log(`[RTP] Notifying ${subscribers.size} subscriber(s) for ${result.mockupId}`);
+          subscribers.forEach((cb) => cb(result));
+        } else {
+          console.log(`[RTP] No subscribers for ${result.mockupId} — result received but nobody listening`);
+        }
+      });
+
+      // Pipeline-settled check
+      if (hasBlobInFlightRef.current && !hasPendingBlobs() && !pipelineResetCooldownRef.current) {
+        hasBlobInFlightRef.current = false;
+        pipelineSettledSubscribersRef.current.forEach((cb) => cb());
+      }
+    },
+    [hasPendingBlobs]
+  );
+
+  /** Composite one cropped frame over its frozen background and deliver it. */
+  const runComposite = useCallback(
+    async (result: SdkMockupResult) => {
+      const gen = sceneGenRef.current;
+      const bg = backgroundsRef.current.get(result.mockupId);
+      if (!bg) {
+        // A cropped frame arrived for a mockup with no frozen background
+        // (e.g. a priority request raced the flip). Drop the session back to
+        // full-frame and re-render this mockup from the server's cached
+        // blobs so the edit isn't lost; the flip re-arms once it seeds.
+        console.error(
+          `[RealtimeProvider] E_OVERLAY_NO_BACKGROUND: cropped frame for ${result.mockupId} has no frozen background — falling back to full-frame and re-rendering it.`
+        );
+        unflipCrop();
+        updateMockupIdsRaw([result.mockupId]);
+        return;
+      }
+      try {
+        if (!bg.imgPromise) bg.imgPromise = loadImageElement(bg.url);
+        const background = await bg.imgPromise;
+        if (!compositeCanvasRef.current && typeof document !== "undefined") {
+          compositeCanvasRef.current = document.createElement("canvas");
+        }
+        const blob = await compositeCroppedFrame({
+          background,
+          frameUrl: result.imageUrl,
+          rect: result.cropRect!,
+          canvas: compositeCanvasRef.current ?? undefined,
+        });
+        // Scene changed while compositing (variant/bg flip) — stale, drop.
+        if (gen !== sceneGenRef.current) return;
+        deliverResults([{ ...result, imageUrl: registerBlobUrl(blob) }]);
+      } catch (err) {
+        console.error(
+          "[RealtimeProvider] E_OVERLAY_COMPOSITE_FAILED: cropped-frame composite failed — full-frame delivery for the rest of this session.",
+          err
+        );
+        cropCapableRef.current = false;
+        unflipCrop();
+        // Re-render the affected mockup full-frame so this edit isn't lost.
+        updateMockupIdsRaw([result.mockupId]);
+      }
+    },
+    [deliverResults, registerBlobUrl, unflipCrop, updateMockupIdsRaw]
+  );
+
+  /** Serial per-mockup composite pump (latest-wins). */
+  const pumpCompositeQueue = useCallback(
+    (mockupId: string) => {
+      if (compositeBusyRef.current.has(mockupId)) return;
+      const job = compositeQueueRef.current.get(mockupId);
+      if (!job) return;
+      compositeQueueRef.current.delete(mockupId);
+      compositeBusyRef.current.add(mockupId);
+      void runComposite(job).finally(() => {
+        compositeBusyRef.current.delete(mockupId);
+        pumpCompositeQueue(mockupId);
+      });
+    },
+    [runComposite]
+  );
+
+  // Sync sdk results into the delivery pipeline (NO STATE UPDATES).
+  // Full frames pass straight through (and seed/refresh their mockup's frozen
+  // background); cropped frames composite over that background first — so
+  // consumers only ever receive full-frame URLs, whatever the wire delivery.
+  useEffect(() => {
+    if (rawMockupResults.length === 0) {
+      if (mockupResultsRef.current.length > 0) {
+        mockupResultsRef.current = [];
+        mockupSubscribersRef.current.forEach((cb) => cb([]));
+      }
+      return;
     }
 
-    // Notify subscribers immediately (no throttle needed - no state updates!)
-    mockupSubscribersRef.current.forEach((cb) => cb(rawMockupResults));
+    console.log(`[RTP] rawMockupResults changed: ${rawMockupResults.length} result(s)`, rawMockupResults.map(r => `${r.mockupId}${r.imageUrl ? ' ✓url' : ' ✗NO-URL'}${r.cropRect ? ' [crop]' : ''}`));
 
-    // Notify filtered subscribers
+    const displayable: SdkMockupResult[] = [];
     rawMockupResults.forEach((result) => {
-      const subscribers = mockupSubscribersByIdRef.current.get(result.mockupId);
-      if (subscribers && subscribers.size > 0) {
-        console.log(`[RTP] Notifying ${subscribers.size} subscriber(s) for ${result.mockupId}`);
-        subscribers.forEach((cb) => cb(result));
-      } else {
-        console.log(`[RTP] No subscribers for ${result.mockupId} — result received but nobody listening`);
+      if (!result.imageUrl) {
+        console.log(`[RTP] Result for ${result.mockupId} has NO imageUrl — not caching, pending NOT cleared`);
+        return;
+      }
+      const firstSight =
+        processedFrameRef.current.get(result.mockupId) !== result.imageUrl;
+      // Only frames observed under a live session config may seed backgrounds
+      // or composite — a leftover pre-config array (e.g. across a reconnect)
+      // may belong to a scene this session hasn't configured yet.
+      const sessionLive = configSentRef.current;
+
+      if (isFullRect(result.cropRect)) {
+        if (firstSight && sessionLive) {
+          processedFrameRef.current.set(result.mockupId, result.imageUrl);
+          // Seed/refresh the frozen background — the freshest full frame is
+          // always the best composite base. Decode it NOW, not at composite
+          // time: with ws binary delivery `imageUrl` is the SDK's live blob:
+          // URL for this mockup, and the SDK revokes it the moment the next
+          // frame for the same mockup (the first cropped one) arrives. A
+          // lazily-loaded background would then fail with
+          // E_OVERLAY_FRAME_LOAD and kill cropped delivery for the session
+          // (observed: every edit cost 3 extra full renders). A decoded
+          // HTMLImageElement keeps its pixels after the URL is revoked.
+          const imgPromise = loadImageElement(result.imageUrl);
+          imgPromise.catch(() => {
+            /* surfaced by runComposite when it awaits the promise */
+          });
+          backgroundsRef.current.set(result.mockupId, { url: result.imageUrl, imgPromise });
+        }
+        // Full frames re-deliver on every run (matches pre-crop behavior —
+        // consumers dedupe by URL; delivery also clears pending state).
+        displayable.push(result);
+      } else if (firstSight && sessionLive) {
+        processedFrameRef.current.set(result.mockupId, result.imageUrl);
+        compositeQueueRef.current.set(result.mockupId, result);
+        pumpCompositeQueue(result.mockupId);
+      } else if (pendingMockupIdsRef.current.has(result.mockupId)) {
+        // Already-composited frame re-observed while the mockup is marked
+        // pending (e.g. a render that resolved to the identical frame).
+        // Re-deliver the composited URL so pending state can't stick.
+        const cached = mockupUrlCacheRef.current.get(result.mockupId);
+        if (cached) displayable.push({ ...result, imageUrl: cached.url });
       }
     });
 
-    // Pipeline-settled check
-    if (hasBlobInFlightRef.current && !hasPendingBlobs() && !pipelineResetCooldownRef.current) {
-      hasBlobInFlightRef.current = false;
-      pipelineSettledSubscribersRef.current.forEach((cb) => cb());
-    }
-  }, [rawMockupResults]);
+    deliverResults(displayable);
+    maybeFlipCrop();
+  }, [rawMockupResults, deliverResults, maybeFlipCrop, pumpCompositeQueue]);
 
   // Wrap sendCanvasBlob to track when blobs are sent
   const sendCanvasBlob = useCallback(
@@ -620,9 +886,12 @@ export function RealtimeProvider({
         lastSentBgRef.current = undefined;
         lastSentSpreadCropsRef.current = "null";
         lastSentColorsRef.current = {};
+        // A new session must re-seed backgrounds before cropped delivery.
+        invalidateBackgrounds();
+        lastConfigBaseRef.current = null;
       };
     }
-  }, [isEnabled, connect, disconnect, wsUrl]);
+  }, [isEnabled, connect, disconnect, wsUrl, invalidateBackgrounds]);
 
   // Clear pending state when mockup results arrive
   useEffect(() => {
@@ -664,6 +933,14 @@ export function RealtimeProvider({
       return; // Nothing changed, skip
     }
 
+    // Variant/bg changes change the SCENE — every frozen cropped-delivery
+    // background is stale. Drop back to full-frame delivery; the flip re-arms
+    // once every mockup has re-seeded. (Spread-crop changes only remap
+    // artwork inside the product region — backgrounds stay valid.)
+    if (variantChanged || bgChanged) {
+      invalidateBackgrounds();
+    }
+
     // Get mockup IDs from product.mockups
     const mockupIds: string[] = [];
     if (product.mockups && Array.isArray(product.mockups)) {
@@ -710,7 +987,15 @@ export function RealtimeProvider({
       });
     }
 
-    sendConfigRaw(configPayload);
+    // Remember the crop-less payload so delivery flips can re-send it, then
+    // send — preserving cropped delivery across re-sends that kept the scene
+    // (e.g. spread-crop changes).
+    lastConfigBaseRef.current = configPayload as unknown as Record<string, unknown>;
+    sendConfigRaw(
+      (croppedActiveRef.current
+        ? { ...configPayload, overlayCrop: "auto" as const }
+        : configPayload) as Parameters<typeof sendConfigRaw>[0]
+    );
 
     configSentRef.current = true;
     lastSentVariantIdRef.current = variantId;
@@ -735,6 +1020,7 @@ export function RealtimeProvider({
     productContext?.selection,
     sendConfigRaw,
     sendColorBlobTracked,
+    invalidateBackgrounds,
     mockupWidth,
     mockupBg,
     spreadCropsVersion,
@@ -772,6 +1058,23 @@ export function RealtimeProvider({
 
   // Canvas integration uses @snowcone-app/canvas with sendCanvasBlob directly
 
+  // Guard cropped delivery on priority requests: a mockup that has never
+  // rendered has no frozen background to composite onto, so drop the session
+  // to full-frame first — its full render seeds the background and the flip
+  // re-arms automatically once every requested mockup is seeded.
+  const updateMockupIdsGuarded = useCallback(
+    (mockupIds: string[]) => {
+      if (
+        croppedActiveRef.current &&
+        mockupIds.some((id) => !backgroundsRef.current.has(id))
+      ) {
+        unflipCrop();
+      }
+      return updateMockupIdsRaw(mockupIds);
+    },
+    [updateMockupIdsRaw, unflipCrop]
+  );
+
   // Wrap updatePlacementSettings to also track in ref (for config re-sends)
   const updatePlacementSettingsWrapped = useCallback(
     (settings: Record<string, any>) => {
@@ -1052,39 +1355,6 @@ export function RealtimeProvider({
     return pendingMockupIdsRef.current.has(mockupId);
   }, []);
 
-  /**
-   * Register a blob URL with lifecycle management
-   */
-  const registerBlobUrl = useCallback((blobOrUrl: Blob | string, placementName?: string): string => {
-    let url: string;
-
-    if (blobOrUrl instanceof Blob) {
-      // Enforce blob URL limit before creating new one
-      if (registeredBlobUrlsRef.current.size >= MAX_CONCURRENT_BLOBS) {
-        // Revoke oldest blob URL (first entry in Set maintains insertion order)
-        const oldest = registeredBlobUrlsRef.current.values().next().value;
-        if (oldest) {
-          URL.revokeObjectURL(oldest);
-          registeredBlobUrlsRef.current.delete(oldest);
-
-          // Also remove from URL cache if present
-          mockupUrlCacheRef.current.forEach((value, key) => {
-            if (value.url === oldest) {
-              mockupUrlCacheRef.current.delete(key);
-            }
-          });
-        }
-      }
-
-      url = URL.createObjectURL(blobOrUrl);
-      registeredBlobUrlsRef.current.add(url);
-    } else {
-      url = blobOrUrl;
-    }
-
-    return url;
-  }, []);
-
   // Store subscription functions in refs so they remain stable across context recreations
   // This prevents race conditions where notifications are lost during re-subscription
   const subscribeMockupResultsRef = useRef(subscribeMockupResults);
@@ -1179,7 +1449,7 @@ export function RealtimeProvider({
       disableRealtime,
       sendCanvasBlob,
       sendCanvasState,
-      updateMockupIds: updateMockupIdsRaw,
+      updateMockupIds: updateMockupIdsGuarded,
       updatePlacementSettings: updatePlacementSettingsWrapped,
       setSpreadCrops,
       getMockupResultsImmediate: () => mockupResultsRef.current,
@@ -1216,7 +1486,7 @@ export function RealtimeProvider({
       disableRealtime,
       sendCanvasBlob,
       sendCanvasState,
-      updateMockupIdsRaw,
+      updateMockupIdsGuarded,
       updatePlacementSettingsWrapped,
       setSpreadCrops,
       stableSubscribeMockupResults,