@agent-native/core 0.92.10 → 0.92.11

package/corpus/templates/design/app/components/design/design-canvas/annotation-snapshot.ts

@@ -0,0 +1,265 @@
+import { callAction } from "@agent-native/core/client";
+
+import type { DrawAnnotation } from "@/components/visual-editor";
+
+/**
+ * Draw-to-agent visual capture.
+ *
+ * DrawOverlay's `onSend` still hands the caller a text summary of stroke
+ * paths and text labels (see DesignCanvas's `onSend` wiring) — that stays,
+ * because raw coordinates remain useful for precision edits. This module
+ * additionally rasterizes the annotated screen into ONE PNG the agent can
+ * actually see, mirroring the Claude-design-style "draw over a screenshot"
+ * flow instead of forcing the agent to reconstruct a picture from
+ * coordinates alone.
+ *
+ * Pipeline: reuse `take-design-screenshot` (already renders the live screen
+ * server-side via headless Chromium — see that action's docblock) at the
+ * exact on-screen rect the user drew on, composite the serialized
+ * strokes/text on top client-side, then hand the finished PNG to the shared
+ * `upload-image` core action so only a durable URL — never base64 — reaches
+ * the agent chat message or any persisted state.
+ *
+ * Every step degrades silently to `null` (no thrown errors): a missing
+ * Chromium binary in hosted/serverless deploys, a slow render, a network
+ * blip fetching the screenshot, or an unconfigured upload provider should
+ * never block the user's drawing from reaching the agent as text — see
+ * `captureAnnotatedScreenshot`'s caller in DesignCanvas.tsx, which always
+ * falls through to the text-only summary.
+ */
+
+/** Bounds the whole capture pipeline so a slow/hung Chromium render or
+ * upload never leaves the user waiting indefinitely to see their message
+ * land in chat. Generous enough for a local Playwright cold start. */
+const CAPTURE_TIMEOUT_MS = 9000;
+
+const MIN_DIMENSION_PX = 200;
+const MAX_WIDTH_PX = 3840;
+const MAX_HEIGHT_PX = 4096;
+
+interface TakeDesignScreenshotResult {
+  ok: boolean;
+  reason?: string;
+  screenshots?: Array<{ url?: string }>;
+}
+
+interface UploadImageResult {
+  url?: string;
+  error?: string;
+}
+
+export interface CaptureAnnotatedScreenshotOptions {
+  /** Stable design id — used as fallback screen lookup and upload filename. */
+  designId?: string;
+  /** Specific design_files.id for the annotated screen. Preferred lookup key. */
+  fileId?: string;
+  /** Filename fallback when `fileId` is not available (paired with `designId`). */
+  filename?: string;
+  /** Only inline (SQL-backed HTML) screens can be rendered by take-design-screenshot. */
+  sourceType?: "inline" | "localhost" | "fusion";
+  annotations: DrawAnnotation[];
+  /** Layout-space canvas size the annotations were serialized against. */
+  canvasSize: { width: number; height: number };
+}
+
+function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      reject(
+        new Error(`annotation screenshot capture timed out after ${ms}ms`),
+      );
+    }, ms);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (error) => {
+        clearTimeout(timer);
+        reject(error);
+      },
+    );
+  });
+}
+
+function loadImageFromUrl(url: string): Promise<HTMLImageElement> {
+  return new Promise((resolve, reject) => {
+    const image = new Image();
+    image.onload = () => resolve(image);
+    image.onerror = () =>
+      reject(new Error("Failed to decode screenshot image"));
+    image.src = url;
+  });
+}
+
+/**
+ * Draw one annotation (a stroke path or a text label) onto a 2D canvas
+ * context, in the same layout-pixel coordinate space DrawOverlay used when
+ * it serialized `pathData`/`position` in its own `send()` — see
+ * DrawOverlay.tsx's coordinate-model docblock. Parses the "M x,y L x,y ..."
+ * syntax with a small regex instead of `new Path2D(...)`: this is the exact
+ * (and only) format DrawOverlay's `send()` emits, and avoiding Path2D keeps
+ * this rasterizer testable against a plain fake context object — Path2D
+ * isn't implemented in jsdom/happy-dom, only real browsers. Exported for unit
+ * testing against a fake context (canvas 2D isn't implemented in jsdom).
+ */
+export function drawAnnotationsOnContext(
+  ctx: Pick<
+    CanvasRenderingContext2D,
+    | "save"
+    | "restore"
+    | "beginPath"
+    | "moveTo"
+    | "lineTo"
+    | "stroke"
+    | "fillText"
+    | "strokeStyle"
+    | "fillStyle"
+    | "lineWidth"
+    | "lineCap"
+    | "lineJoin"
+    | "font"
+    | "textBaseline"
+  >,
+  annotations: DrawAnnotation[],
+): void {
+  for (const annotation of annotations) {
+    if (annotation.type === "path" && annotation.pathData) {
+      const points = parsePathDataPoints(annotation.pathData);
+      if (points.length < 2) continue;
+      ctx.save();
+      ctx.strokeStyle = annotation.color;
+      ctx.lineWidth = annotation.lineWidth;
+      ctx.lineCap = "round";
+      ctx.lineJoin = "round";
+      ctx.beginPath();
+      ctx.moveTo(points[0].x, points[0].y);
+      for (let i = 1; i < points.length; i++) {
+        ctx.lineTo(points[i].x, points[i].y);
+      }
+      ctx.stroke();
+      ctx.restore();
+    } else if (annotation.type === "text" && annotation.text) {
+      ctx.save();
+      ctx.fillStyle = annotation.color;
+      ctx.font = `600 ${14 + annotation.lineWidth}px sans-serif`;
+      ctx.textBaseline = "top";
+      ctx.fillText(
+        annotation.text,
+        annotation.position.x,
+        annotation.position.y,
+      );
+      ctx.restore();
+    }
+  }
+}
+
+const PATH_DATA_POINT_RE = /[ML]\s*(-?[\d.]+),(-?[\d.]+)/g;
+
+/** Parse DrawOverlay's `"M x,y L x,y L x,y"` path syntax into raw points. */
+function parsePathDataPoints(
+  pathData: string,
+): Array<{ x: number; y: number }> {
+  const points: Array<{ x: number; y: number }> = [];
+  for (const match of pathData.matchAll(PATH_DATA_POINT_RE)) {
+    points.push({ x: Number(match[1]), y: Number(match[2]) });
+  }
+  return points;
+}
+
+/** Fetch the screenshot bytes (avoids canvas cross-origin tainting — a
+ * same-process `fetch` + `Blob` object URL is never "foreign" to the canvas,
+ * regardless of the storage provider's CORS headers) and composite the
+ * annotations on top at the exact requested pixel size. */
+async function compositeScreenshotWithAnnotations(
+  screenshotUrl: string,
+  annotations: DrawAnnotation[],
+  size: { width: number; height: number },
+): Promise<string | null> {
+  const response = await fetch(screenshotUrl);
+  if (!response.ok) return null;
+  const blob = await response.blob();
+  const objectUrl = URL.createObjectURL(blob);
+  try {
+    const image = await loadImageFromUrl(objectUrl);
+    const canvas = document.createElement("canvas");
+    canvas.width = size.width;
+    canvas.height = size.height;
+    const ctx = canvas.getContext("2d");
+    if (!ctx) return null;
+    ctx.drawImage(image, 0, 0, size.width, size.height);
+    drawAnnotationsOnContext(ctx, annotations);
+    return canvas.toDataURL("image/png");
+  } finally {
+    URL.revokeObjectURL(objectUrl);
+  }
+}
+
+/**
+ * Best-effort: render the annotated screen, composite the drawing on top,
+ * and upload the result. Returns the durable image URL, or `null` on ANY
+ * failure (missing Chromium, no upload provider configured, network error,
+ * unsupported source type, timeout) — callers must treat `null` as "fall
+ * back to the text-only summary", never as a reason to drop the annotation.
+ */
+export async function captureAnnotatedScreenshot(
+  options: CaptureAnnotatedScreenshotOptions,
+): Promise<string | null> {
+  const { designId, fileId, filename, sourceType, annotations, canvasSize } =
+    options;
+
+  // take-design-screenshot renders stored SQL/collab HTML content; localhost
+  // and fusion screens hold a URL, not renderable HTML, so skip straight to
+  // the text-only fallback for those instead of paying for a doomed render.
+  if (sourceType && sourceType !== "inline") return null;
+  if (!fileId && !designId) return null;
+  if (typeof document === "undefined") return null;
+  if (
+    !Number.isFinite(canvasSize.width) ||
+    !Number.isFinite(canvasSize.height) ||
+    canvasSize.width < MIN_DIMENSION_PX ||
+    canvasSize.height < MIN_DIMENSION_PX
+  ) {
+    return null;
+  }
+
+  const widthPx = Math.min(MAX_WIDTH_PX, Math.round(canvasSize.width));
+  const heightPx = Math.min(MAX_HEIGHT_PX, Math.round(canvasSize.height));
+
+  try {
+    return await withTimeout(
+      (async () => {
+        const shot = await callAction<TakeDesignScreenshotResult>(
+          "take-design-screenshot",
+          {
+            ...(fileId ? { fileId } : { designId, filename }),
+            widths: [widthPx],
+            heights: [heightPx],
+          },
+        );
+        const screenshotUrl = shot?.ok ? shot.screenshots?.[0]?.url : undefined;
+        if (!screenshotUrl) return null;
+
+        const compositeDataUrl = await compositeScreenshotWithAnnotations(
+          screenshotUrl,
+          annotations,
+          { width: widthPx, height: heightPx },
+        );
+        if (!compositeDataUrl) return null;
+
+        const uploaded = await callAction<UploadImageResult>("upload-image", {
+          data: compositeDataUrl,
+          filename: `design-annotation-${designId ?? fileId}-${Date.now()}.png`,
+        });
+        return uploaded?.url ?? null;
+      })(),
+      CAPTURE_TIMEOUT_MS,
+    );
+  } catch (error) {
+    console.warn(
+      "[DesignCanvas] annotated screenshot capture failed; falling back to text-only:",
+      error,
+    );
+    return null;
+  }
+}

package/corpus/templates/design/app/components/design/design-canvas/annotation-submit.ts

@@ -1,7 +1,14 @@
 interface SubmitDesignAnnotationsOptions {
   message: string;
   hasQueuedPins: boolean;
-  send: (message: string) => void;
+  /**
+   * Deliver the message to the agent chat. Must reject (or throw) when
+   * delivery is not confirmed — e.g. `sendToAgentChatAndConfirm` resolving
+   * `delivered: false` should be turned into a thrown/rejected error here.
+   * A `send` that only fires-and-forgets (never throws on silent drops) will
+   * make this function report success even though nothing reached the agent.
+   */
+  send: (message: string) => void | Promise<void>;
   markQueuedPinsSubmitted: () => void;
   exitDrawMode: () => void;
   onError: (error: unknown) => void;
@@ -11,18 +18,22 @@ interface SubmitDesignAnnotationsOptions {
  * Submit a drawing/comment batch as one atomic UI transition.
  *
  * The overlay and queued pins must remain intact when chat handoff fails so the
- * user can retry without recreating their annotation work.
+ * user can retry without recreating their annotation work. `send` is awaited
+ * so a caller using an ack-confirmed delivery (e.g.
+ * `sendToAgentChatAndConfirm`) can reject when the message was silently
+ * dropped (no LLM/agent engine configured, panel never mounted, etc.) — the
+ * pins/draw-mode teardown below only runs once delivery is confirmed.
  */
-export function submitDesignAnnotations({
+export async function submitDesignAnnotations({
   message,
   hasQueuedPins,
   send,
   markQueuedPinsSubmitted,
   exitDrawMode,
   onError,
-}: SubmitDesignAnnotationsOptions): boolean {
+}: SubmitDesignAnnotationsOptions): Promise<boolean> {
   try {
-    send(message);
+    await send(message);
   } catch (error) {
     onError(error);
     return false;

package/corpus/templates/design/app/components/design/design-canvas/coordinate-transforms.ts

@@ -1,11 +1,25 @@
 /**
  * Converts a viewport point into the iframe's unscaled content coordinates.
+ *
+ * `scrollOffset` is the embedded screen's own internal document scroll
+ * (`iframe.contentWindow.scrollX/scrollY`), already expressed in the iframe's
+ * own unscaled content pixels — the SAME units as `iframeContentSize` and the
+ * function's return value. It is folded in AFTER the scale division, never
+ * multiplied/divided by `scaleX`/`scaleY`: the outer `scaleX`/`scaleY` factor
+ * only undoes the *host* canvas's zoom transform (which maps `iframeRect`,
+ * a host-viewport rect, onto `iframeContentSize`, the iframe's own layout
+ * box) and has nothing to do with how far the iframe's own document has been
+ * scrolled internally. Concretely: at 50% host zoom (scaleX = 0.5) with the
+ * embedded screen scrolled 600px down, a click at the iframe's visible top
+ * edge (rendered content y = 0) must land at content y = 600 — not 1200 (a
+ * mistaken `600 / 0.5`) and not 300 (a mistaken `600 * 0.5`).
  */
 export function getScreenContentPointFromClient(
   clientX: number,
   clientY: number,
   iframeRect: { left: number; top: number; width: number; height: number },
   iframeContentSize: { width: number; height: number },
+  scrollOffset: { left: number; top: number } = { left: 0, top: 0 },
 ): { x: number; y: number } {
   const scaleX =
     iframeContentSize.width > 0
@@ -16,8 +30,8 @@ export function getScreenContentPointFromClient(
       ? iframeRect.height / iframeContentSize.height
       : 1;
   return {
-    x: (clientX - iframeRect.left) / (scaleX || 1),
-    y: (clientY - iframeRect.top) / (scaleY || 1),
+    x: (clientX - iframeRect.left) / (scaleX || 1) + scrollOffset.left,
+    y: (clientY - iframeRect.top) / (scaleY || 1) + scrollOffset.top,
   };
 }
 

package/corpus/templates/design/app/components/design/design-canvas/embedded-frame.ts

@@ -25,9 +25,14 @@ export function getEmbeddedIframeBackgroundColor(args: {
     : (args.embeddedFrameBackground ?? "transparent");
 }
 
-export function embeddedContentOffsetStyle(x: number, y: number): string {
+export function embeddedContentOffsetCss(x: number, y: number): string {
   if (x === 0 && y === 0) return "";
-  return `<style data-agent-native-content-offset>body > [data-agent-native-node-id]{translate:${Math.round(x)}px ${Math.round(y)}px;}</style>`;
+  return `body > [data-agent-native-node-id]{translate:${Math.round(x)}px ${Math.round(y)}px;}`;
+}
+
+export function embeddedContentOffsetStyle(x: number, y: number): string {
+  const css = embeddedContentOffsetCss(x, y);
+  return css ? `<style data-agent-native-content-offset>${css}</style>` : "";
 }
 
 function injectEmbeddedFrameStyle(content: string, style: string): string {

package/corpus/templates/design/app/components/design/design-canvas/iframe-events.ts

@@ -1,4 +1,4 @@
-import type { ElementInfo } from "../types";
+import type { CanvasLayerHitCandidate, ElementInfo } from "../types";
 
 export interface IframeHotkeyPayload {
   key: string;
@@ -15,9 +15,11 @@ export interface IframeFigmaClipboardPastePayload {
 }
 
 export interface IframeContextMenuPayload {
+  screenId?: string;
   clientX: number;
   clientY: number;
   viewportClientX?: number;
   viewportClientY?: number;
   info?: ElementInfo | null;
+  layerCandidates?: CanvasLayerHitCandidate[];
 }

package/corpus/templates/design/app/components/design/edit-panel/code-inspect-helpers.tsx

@@ -19,6 +19,14 @@ const VOID_HTML_TAGS = new Set([
   "wbr",
 ]);
 
+const INSPECT_CODE_MAX_INLINE_TAG_LENGTH = 48;
+
+interface ParsedOpeningTag {
+  tagName: string;
+  attributes: string[];
+  closing: ">" | "/>";
+}
+
 export function normalizedElementTagName(
   tagName: string | null | undefined,
 ): string {
@@ -72,6 +80,58 @@ export function truncateOpeningTag(openTag: string, max = 32): string {
   );
 }
 
+function parseInspectCodeOpeningTag(openTag: string): ParsedOpeningTag | null {
+  const tagMatch = /^<([a-zA-Z][\w:-]*)([\s\S]*?)(\/?>)$/.exec(openTag.trim());
+  if (!tagMatch?.[1] || (tagMatch[3] !== ">" && tagMatch[3] !== "/>")) {
+    return null;
+  }
+
+  const attributes: string[] = [];
+  const attributePattern =
+    /\s+([^\s=/>]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s"'=<>`]+))?/g;
+  for (const match of (tagMatch[2] ?? "").matchAll(attributePattern)) {
+    const name = match[1];
+    if (!name) continue;
+    const normalizedName = name.toLowerCase();
+    if (
+      normalizedName === "style" ||
+      normalizedName.startsWith("data-agent-native-")
+    ) {
+      continue;
+    }
+    attributes.push(`${name}${match[2] ? `=${match[2]}` : ""}`);
+  }
+
+  return {
+    tagName: tagMatch[1],
+    attributes,
+    closing: tagMatch[3],
+  };
+}
+
+/**
+ * Remove Design's runtime-only attributes, then format retained attributes to
+ * fit the Inspect Code popover without routine horizontal scrolling.
+ */
+export function formatInspectCodeOpeningTag(
+  openTag: string,
+  maxInlineLength = INSPECT_CODE_MAX_INLINE_TAG_LENGTH,
+): string {
+  const parsed = parseInspectCodeOpeningTag(openTag);
+  if (!parsed) return openTag;
+
+  const inline = `<${parsed.tagName}${
+    parsed.attributes.length ? ` ${parsed.attributes.join(" ")}` : ""
+  }${parsed.closing}`;
+  if (!parsed.attributes.length || inline.length <= maxInlineLength) {
+    return inline;
+  }
+
+  return `<${parsed.tagName}\n  ${parsed.attributes.join("\n  ")}${
+    parsed.closing
+  }`;
+}
+
 function tagNameFromOpeningTag(openTag: string): string | null {
   const match = /^<\/?\s*([a-zA-Z][\w:-]*)/.exec(openTag.trim());
   return match?.[1]?.toLowerCase() ?? null;
@@ -113,7 +173,9 @@ export function elementHtmlPreview(
     data.classes?.some((item) => item.trim()),
   );
   if (!openingTag && !hasFallbackMetadata) return null;
-  const previewOpeningTag = openingTag ?? fallbackOpeningTag(data);
+  const previewOpeningTag = formatInspectCodeOpeningTag(
+    openingTag ?? fallbackOpeningTag(data),
+  );
   const tagName =
     tagNameFromOpeningTag(previewOpeningTag) ??
     normalizedElementTagName(data.tagName);
@@ -229,20 +291,28 @@ export function parseAlpineDataObject(
 
   const out: Record<string, string> = {};
   // Split on top-level commas only (no nesting / quotes inside values here).
+  // The quoted-value alternatives allow backslash-escaped quotes (`\\.`)
+  // inside the literal — without that, a value like `'it\'s ok'` truncates
+  // at the escaped quote (matching only `'it\'`), silently dropping the rest
+  // of the string. That mismatch used to slip past `canRebuildAlpineDataLosslessly`
+  // as a false positive: the truncated value round-tripped "stably" (in the
+  // sense of parse -> serialize -> parse staying self-consistent) while still
+  // being wrong relative to the original source.
   const pairRe =
-    /(?:^|,)\s*(?:'([^']+)'|"([^"]+)"|([A-Za-z_$][\w$]*))\s*:\s*('[^']*'|"[^"]*"|true|false|-?\d+(?:\.\d+)?)/g;
+    /(?:^|,)\s*(?:'([^']+)'|"([^"]+)"|([A-Za-z_$][\w$]*))\s*:\s*('(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*"|true|false|-?\d+(?:\.\d+)?)/g;
   let m: RegExpExecArray | null;
   let matched = false;
   while ((m = pairRe.exec(inner)) !== null) {
     matched = true;
     const key = m[1] ?? m[2] ?? m[3];
     let raw = m[4]!;
-    // Unwrap quotes for string literals; keep booleans / numbers verbatim.
-    if (
-      (raw.startsWith("'") && raw.endsWith("'")) ||
-      (raw.startsWith('"') && raw.endsWith('"'))
-    ) {
-      raw = raw.slice(1, -1);
+    // Unwrap quotes for string literals, un-escaping backslash-escaped quotes
+    // back to their literal form (the inverse of the escaping
+    // `serializeAlpineDataObject` applies); keep booleans / numbers verbatim.
+    if (raw.startsWith("'") && raw.endsWith("'")) {
+      raw = raw.slice(1, -1).replace(/\\'/g, "'");
+    } else if (raw.startsWith('"') && raw.endsWith('"')) {
+      raw = raw.slice(1, -1).replace(/\\"/g, '"');
     }
     if (key) out[key] = raw;
   }