@gakim-digital/dexter-bridge 0.11.9 → 0.11.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gakim-digital/dexter-bridge",
-  "version": "0.11.9",
+  "version": "0.11.10",
   "description": "Local bridge for InstaWebAI and Dexter — runs Codex, Claude Code, or OpenCode on your machine.",
   "type": "module",
   "bin": {

package/src/framerAgentTools.js

@@ -23,6 +23,8 @@ const MAX_MODEL_RESULT_BYTES = 120 * 1024;
 const MAX_SCREENSHOT_BYTES = 4 * 1024 * 1024;
 const MAX_SCREENSHOTS_PER_CALL = 4;
 const SCREENSHOT_FETCH_TIMEOUT_MS = 20_000;
+const SESSION_SCREENSHOT_TIMEOUT_MS = 60_000;
+const MAX_SCREENSHOT_FAILURES_REPORTED = 8;
 const SCREENSHOT_MIME_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp', 'image/gif']);
 const DEFAULT_COMMAND_TIMEOUT_MS = 5 * 60_000;
 const SESSION_TIMEOUT_MS = 10 * 60_000;
@@ -155,6 +157,8 @@ export const FRAMER_AGENT_TOOL_DEFINITIONS = [
           minItems: 1,
           uniqueItems: true,
           items: { type: 'string', enum: VERIFICATION_CHECKS },
+          description:
+            'The evidence you will produce before completion. Dexter enforces exactly this list and adds nothing to it. Include "visual" when the result must be judged by eye, such as layout or styling work or a build from a reference image, then inspect a screenshot after your final mutation.',
         },
         behaviors: {
           type: 'array',
@@ -706,16 +710,9 @@ function normalizeTaskPlan(rawArguments, inheritedRejectedMechanisms = []) {
       'Describe the understood task and declare at least one task domain.',
     );
   }
+  // The model's declared verification is authoritative; domains describe the
+  // work and never add checks on their own.
   const verification = new Set(['structural', ...requestedVerification]);
-  if (
-    domains.some((domain) =>
-      ['visual', 'interactions', 'responsive'].includes(domain))
-  ) verification.add('visual');
-  if (domains.includes('links')) verification.add('links');
-  if (domains.includes('interactions')) verification.add('interactions');
-  if (domains.includes('responsive')) verification.add('responsive');
-  if (domains.includes('code')) verification.add('code');
-  if (domains.includes('data')) verification.add('data');
   const behaviorPlan = normalizeBehaviorPlan(
     rawArguments,
     inheritedRejectedMechanisms,
@@ -906,13 +903,6 @@ function abortError() {
   });
 }
 
-function screenshotQueryCount(rawArguments) {
-  return (Array.isArray(rawArguments?.queries) ? rawArguments.queries : [])
-    .filter((query) => query?.type === 'screenshot')
-    .slice(0, MAX_SCREENSHOTS_PER_CALL)
-    .length;
-}
-
 function isFramerScreenshotUrl(value) {
   try {
     const url = new URL(String(value || ''));
@@ -924,32 +914,36 @@ function isFramerScreenshotUrl(value) {
   }
 }
 
-// Screenshot results are URLs the model cannot open, so the bridge downloads
-// them and hands the pixels to the model as image content.
-async function downloadScreenshotImages(parsed, fetchImpl) {
-  const urls = (Array.isArray(parsed?.results) ? parsed.results : [])
-    .map((entry) => entry?.image_url)
-    .filter(isFramerScreenshotUrl)
+function screenshotQueries(rawArguments) {
+  return (Array.isArray(rawArguments?.queries) ? rawArguments.queries : [])
+    .filter((query) => query?.type === 'screenshot')
     .slice(0, MAX_SCREENSHOTS_PER_CALL);
-  const images = [];
-  for (const url of urls) {
-    try {
-      const response = await fetchImpl(url, {
-        signal: AbortSignal.timeout(SCREENSHOT_FETCH_TIMEOUT_MS),
-      });
-      const mimeType = String(response.headers.get('content-type') || '')
-        .split(';')[0]
-        .trim()
-        .toLowerCase();
-      if (!response.ok || !SCREENSHOT_MIME_TYPES.has(mimeType)) continue;
-      const bytes = Buffer.from(await response.arrayBuffer());
-      if (!bytes.length || bytes.length > MAX_SCREENSHOT_BYTES) continue;
-      images.push({ url, mimeType, data: bytes.toString('base64') });
-    } catch {
-      // The URL stays in the text result; the model is told the image is missing.
-    }
+}
+
+// Screenshot results are URLs the model cannot open, so the bridge downloads
+// them and hands the pixels to the model as image content. Failures carry a
+// short reason so delivery problems are visible in run telemetry.
+async function downloadScreenshotImage(url, fetchImpl) {
+  if (!isFramerScreenshotUrl(url)) return { error: 'untrusted_url' };
+  try {
+    const response = await fetchImpl(url, {
+      signal: AbortSignal.timeout(SCREENSHOT_FETCH_TIMEOUT_MS),
+    });
+    if (!response.ok) return { error: `http_${response.status}` };
+    const mimeType = String(response.headers.get('content-type') || '')
+      .split(';')[0]
+      .trim()
+      .toLowerCase();
+    if (!SCREENSHOT_MIME_TYPES.has(mimeType)) return { error: 'unsupported_type' };
+    const bytes = Buffer.from(await response.arrayBuffer());
+    if (!bytes.length) return { error: 'empty' };
+    if (bytes.length > MAX_SCREENSHOT_BYTES) return { error: 'too_large' };
+    return { image: { url, mimeType, data: bytes.toString('base64') } };
+  } catch (error) {
+    return {
+      error: String(error?.cause?.code || error?.name || 'fetch_failed').slice(0, 60),
+    };
   }
-  return images;
 }
 
 function parseStructuredOutput(value) {
@@ -1239,8 +1233,12 @@ export function createFramerAgentToolRuntime({
   const representativeEvidenceBehaviorByNode = new Map();
   const behaviorVerificationAttempts = new Map();
   const mutationReceipts = [];
-  let screenshotBeforeMutation = false;
   let lastScreenshotSequence = 0;
+  let lastScreenshotAttemptSequence = 0;
+  let screenshotsViaDownload = 0;
+  let screenshotsViaSession = 0;
+  let screenshotsUndelivered = 0;
+  const screenshotFailures = [];
   let taskPlan = null;
   const verificationEvidence = new Map();
   let guidanceMetadata = {
@@ -1664,6 +1662,78 @@ export function createFramerAgentToolRuntime({
     }
   };
 
+  // Captures a node through the Framer session itself. This needs no access
+  // to the screenshot CDN, so it works where that download is blocked.
+  const captureSessionScreenshot = async (nodeId) => {
+    for (const scale of [1, 0.5]) {
+      try {
+        const result = await execCode(
+          [
+            `const shot = await framer.screenshot(${JSON.stringify(nodeId)}, { format: 'jpeg', quality: 80, scale: ${scale} });`,
+            'console.log(JSON.stringify({ mimeType: shot.mimeType, data: Buffer.from(shot.data).toString("base64") }));',
+          ].join('\n'),
+          { timeoutMs: SESSION_SCREENSHOT_TIMEOUT_MS },
+        );
+        const parsed = parseStructuredOutput(result.stdout);
+        const mimeType = String(parsed?.mimeType || '').toLowerCase();
+        const data = typeof parsed?.data === 'string' ? parsed.data : '';
+        if (!SCREENSHOT_MIME_TYPES.has(mimeType)) return { error: 'unsupported_type' };
+        const bytes = Buffer.from(data, 'base64').length;
+        if (!bytes) return { error: 'empty' };
+        if (bytes > MAX_SCREENSHOT_BYTES) return { error: 'too_large' };
+        return { image: { url: null, mimeType, data } };
+      } catch (error) {
+        if (error?.code === 'FRAMER_AGENT_OUTPUT_TOO_LARGE' && scale === 1) continue;
+        return { error: String(error?.code || 'capture_failed').slice(0, 60) };
+      }
+    }
+    return { error: 'too_large' };
+  };
+
+  const deliverScreenshots = async (queries, parsed) => {
+    const results = (Array.isArray(parsed?.results) ? parsed.results : [])
+      .filter((entry) => typeof entry?.image_url === 'string');
+    const images = [];
+    const failures = [];
+    let viaDownload = 0;
+    let viaSession = 0;
+    let undelivered = 0;
+    for (const [index, query] of queries.entries()) {
+      const nodeId = typeof query.id === 'string' ? query.id.trim() : '';
+      const entry =
+        (nodeId && results.find((candidate) => candidate.id === nodeId))
+        || results[index];
+      const downloaded = entry
+        ? await downloadScreenshotImage(entry.image_url, fetchImpl)
+        : { error: 'no_image_url' };
+      if (downloaded.image) {
+        images.push(downloaded.image);
+        viaDownload += 1;
+        continue;
+      }
+      const captured = nodeId
+        ? await captureSessionScreenshot(nodeId)
+        : { error: 'no_node_id' };
+      if (captured.image) {
+        images.push(captured.image);
+        viaSession += 1;
+        failures.push({
+          target: nodeId,
+          download: downloaded.error,
+          recoveredVia: 'session',
+        });
+        continue;
+      }
+      undelivered += 1;
+      failures.push({
+        target: nodeId || 'url',
+        download: downloaded.error,
+        capture: captured.error,
+      });
+    }
+    return { images, viaDownload, viaSession, undelivered, failures };
+  };
+
   const referenceAssetFiles = Array.isArray(assignment?.context?.referenceAssetFiles)
     ? assignment.context.referenceAssetFiles
     : [];
@@ -1764,16 +1834,20 @@ export function createFramerAgentToolRuntime({
     };
   };
 
-  const markInspection = ({ visual = false, verifies = [] } = {}) => {
+  const markInspection = ({
+    visual = false,
+    visualAttempted = false,
+    verifies = [],
+  } = {}) => {
     operationSequence += 1;
     lastInspectionSequence = operationSequence;
     lastInspectionAt = new Date().toISOString();
     verificationEvidence.set('structural', operationSequence);
+    if (visualAttempted) lastScreenshotAttemptSequence = operationSequence;
     if (visual) {
       screenshotCalls += 1;
       lastScreenshotSequence = operationSequence;
       verificationEvidence.set('visual', operationSequence);
-      if (lastMutationSequence === 0) screenshotBeforeMutation = true;
     }
     for (const check of normalizeVerificationValues(
       verifies,
@@ -1824,34 +1898,30 @@ export function createFramerAgentToolRuntime({
   const verificationCheckCompleted = (check) => {
     if (mutationCalls === 0) return true;
     if (check === 'structural') return synchronized();
-    if (check === 'visual') {
-      return (
-        screenshotBeforeMutation
-        && lastScreenshotSequence > lastMutationSequence
-      );
-    }
     return Number(verificationEvidence.get(check) || 0) > lastMutationSequence;
   };
   const visualVerificationRequired = () =>
-    applyChangesCalls > 0
-    || requiredVerificationChecks().includes('visual');
-  const visualVerificationCompleted = () =>
-    !visualVerificationRequired()
-    || mutationCalls === 0
-    || (
-      screenshotBeforeMutation
-      && lastScreenshotSequence > lastMutationSequence
-    );
+    requiredVerificationChecks().includes('visual');
+  // The model asked for the final canvas, but no screenshot could be handed to
+  // it. That is reported as unverified rather than blamed on the model.
+  const visualVerificationUnavailable = () =>
+    mutationCalls > 0
+    && lastScreenshotAttemptSequence > lastMutationSequence
+    && lastScreenshotSequence <= lastMutationSequence;
+  const visualVerificationStatus = () => {
+    if (!visualVerificationRequired() || mutationCalls === 0) return 'not_required';
+    if (verificationCheckCompleted('visual')) return 'seen';
+    return visualVerificationUnavailable() ? 'unavailable' : 'missing';
+  };
+  const requiredChecks = () => [
+    ...new Set(['structural', ...requiredVerificationChecks()]),
+  ];
   const missingVerificationChecks = () => {
     if (mutationCalls === 0) return [];
     const missing = [];
     if (!taskPlan) missing.push('task-plan');
-    const required = new Set([
-      'structural',
-      ...requiredVerificationChecks(),
-      ...(applyChangesCalls > 0 ? ['visual'] : []),
-    ]);
-    for (const check of required) {
+    for (const check of requiredChecks()) {
+      if (check === 'visual' && visualVerificationUnavailable()) continue;
       if (!verificationCheckCompleted(check)) missing.push(check);
     }
     return missing;
@@ -1862,8 +1932,18 @@ export function createFramerAgentToolRuntime({
     completed: missingVerificationChecks().length === 0,
     structuralCompleted: synchronized(),
     visualRequired: visualVerificationRequired() && mutationCalls > 0,
-    visualCompleted: visualVerificationCompleted(),
+    visualCompleted: visualVerificationStatus() !== 'missing',
+    visualStatus: visualVerificationStatus(),
     screenshotCalls,
+    // Every failed CDN download is listed; recoveredVia marks the ones the
+    // session capture still delivered to the model.
+    screenshotDelivery: {
+      delivered: screenshotsViaDownload + screenshotsViaSession,
+      viaDownload: screenshotsViaDownload,
+      viaSession: screenshotsViaSession,
+      undelivered: screenshotsUndelivered,
+      failures: screenshotFailures.map((failure) => ({ ...failure })),
+    },
     ...(requiredVerificationChecks().includes('interactions')
       || interactionVerificationCalls > 0
       ? {
@@ -1907,20 +1987,10 @@ export function createFramerAgentToolRuntime({
         }
       : {}),
     taskPlan: taskPlanSnapshot(),
-    requiredChecks: [
-      ...new Set([
-        'structural',
-        ...requiredVerificationChecks(),
-        ...(applyChangesCalls > 0 ? ['visual'] : []),
-      ]),
-    ],
-    completedChecks: [
-      ...new Set([
-        'structural',
-        ...requiredVerificationChecks(),
-        ...(applyChangesCalls > 0 ? ['visual'] : []),
-      ]),
-    ].filter(verificationCheckCompleted),
+    requiredChecks: requiredChecks(),
+    completedChecks: requiredChecks().filter(verificationCheckCompleted),
+    unavailableChecks:
+      visualVerificationStatus() === 'unavailable' ? ['visual'] : [],
     missingChecks: missingVerificationChecks(),
     lastMutationAt,
     lastInspectionAt,
@@ -1942,18 +2012,31 @@ export function createFramerAgentToolRuntime({
     };
   };
 
+  const withUnavailableVisualCheck = (completion) =>
+    visualVerificationStatus() === 'unavailable'
+      ? {
+          ...completion,
+          checks: [
+            ...(Array.isArray(completion?.checks) ? completion.checks : []),
+            {
+              command: 'Visual verification',
+              status: 'skipped',
+              output:
+                'Screenshots of the final canvas could not be delivered to the model, so the visual result is unverified.',
+            },
+          ],
+        }
+      : completion;
+
   const finalizeCompletion = (completion) => {
-    if (
-      completion?.status !== 'completed'
-      || mutationCalls === 0
-      || verification().completed
-    ) {
+    if (completion?.status !== 'completed' || mutationCalls === 0) {
       return completion;
     }
+    if (verification().completed) return withUnavailableVisualCheck(completion);
     const missing = missingVerificationChecks().map((check) => {
       if (check === 'task-plan') return 'a harness task and verification plan';
       if (check === 'structural') return 'a final structural read';
-      if (check === 'visual') return 'before-and-after screenshot verification';
+      if (check === 'visual') return 'post-mutation screenshot verification';
       return `post-mutation ${check} verification`;
     });
     const interactionDetails = interactionVerification.behaviors
@@ -1971,14 +2054,15 @@ export function createFramerAgentToolRuntime({
         ? [`Interaction evidence: ${interactionDetails.join('; ')}.`]
         : []),
     ].join(' ');
+    const failed = withUnavailableVisualCheck(completion);
     return {
-      ...completion,
+      ...failed,
       status: 'failed',
       summary:
         String(completion?.summary || '').trim()
         || 'The requested changes were applied.',
       checks: [
-        ...(Array.isArray(completion?.checks) ? completion.checks : []),
+        ...(Array.isArray(failed?.checks) ? failed.checks : []),
         {
           command: 'Harness-owned final verification',
           status: 'failed',
@@ -2002,9 +2086,7 @@ export function createFramerAgentToolRuntime({
         missing.push('perform a focused live read after the latest mutation');
       } else if (check === 'visual') {
         missing.push(
-          screenshotBeforeMutation
-            ? 'capture and inspect the affected canvas after the latest mutation'
-            : 'report that the required baseline screenshot was missed; do not claim visual verification',
+          'capture and inspect a screenshot of the affected canvas after the latest mutation',
         );
       } else {
         const interactionState = interactionVerification.behaviors
@@ -2084,10 +2166,7 @@ export function createFramerAgentToolRuntime({
     return uniqueStrings(commands.map((command) => command.nodeId));
   };
 
-  const requireTaskPlanForMutation = ({
-    canvas = false,
-    changes = '',
-  } = {}) => {
+  const requireTaskPlanForMutation = ({ changes = '' } = {}) => {
     if (!taskPlan) {
       throw toolError(
         'FRAMER_HARNESS_PLAN_REQUIRED',
@@ -2096,15 +2175,6 @@ export function createFramerAgentToolRuntime({
     }
     const repairTargets = representativeRepairTargets(changes);
     const isRepresentativeRepair = repairTargets.length > 0;
-    if (
-      (canvas || taskPlan.verification.includes('visual'))
-      && !screenshotBeforeMutation
-    ) {
-      throw toolError(
-        'FRAMER_HARNESS_BASELINE_SCREENSHOT_REQUIRED',
-        'Capture and inspect a screenshot of the affected canvas before the first visual mutation.',
-      );
-    }
     const rejectedSelectedMechanisms = taskPlan.behaviors.filter((behavior) => {
       const selected = taskPlan.mechanisms.find((decision) =>
         decision.behaviorId === behavior.id);
@@ -2319,16 +2389,6 @@ export function createFramerAgentToolRuntime({
           rawArguments,
           inheritedRejectedMechanisms,
         );
-        if (
-          mutationCalls > 0
-          && nextPlan.verification.includes('visual')
-          && !screenshotBeforeMutation
-        ) {
-          throw toolError(
-            'FRAMER_HARNESS_BASELINE_SCREENSHOT_REQUIRED',
-            'Visual verification cannot be added after mutations when no baseline screenshot was captured.',
-          );
-        }
         const previousMechanisms = new Map(
           (taskPlan?.mechanisms || []).map((decision) => [
             decision.behaviorId,
@@ -2644,12 +2704,25 @@ export function createFramerAgentToolRuntime({
             'console.log(JSON.stringify(result, null, 2));',
           ].join('\n'),
         );
-        modelImages = await downloadScreenshotImages(
-          parseStructuredOutput(result.stdout),
-          fetchImpl,
+        const requestedScreenshots = screenshotQueries(rawArguments);
+        const delivery = requestedScreenshots.length
+          ? await deliverScreenshots(
+              requestedScreenshots,
+              parseStructuredOutput(result.stdout),
+            )
+          : { images: [], viaDownload: 0, viaSession: 0, undelivered: 0, failures: [] };
+        modelImages = delivery.images;
+        screenshotsViaDownload += delivery.viaDownload;
+        screenshotsViaSession += delivery.viaSession;
+        screenshotsUndelivered += delivery.undelivered;
+        screenshotFailures.push(...delivery.failures);
+        screenshotFailures.splice(
+          0,
+          Math.max(0, screenshotFailures.length - MAX_SCREENSHOT_FAILURES_REPORTED),
         );
         markInspection({
           visual: modelImages.length > 0,
+          visualAttempted: requestedScreenshots.length > 0,
           verifies: rawArguments.verifies,
         });
       } else if (name === 'framer_query_images') {
@@ -2694,7 +2767,7 @@ export function createFramerAgentToolRuntime({
           );
         }
         const mutatedRepresentativeBehaviorIds =
-          requireTaskPlanForMutation({ canvas: true, changes });
+          requireTaskPlanForMutation({ changes });
         const pagePath = normalizePagePath(rawArguments.pagePath);
         result = await execCode(
           [
@@ -2770,7 +2843,7 @@ export function createFramerAgentToolRuntime({
       });
       const normalized = normalizedResult(result);
       if (name !== 'framer_read_project') return normalized;
-      const screenshotCount = screenshotQueryCount(rawArguments);
+      const screenshotCount = screenshotQueries(rawArguments).length;
       return {
         ...normalized,
         ...(modelImages.length ? { modelImages } : {}),

package/src/framerBehaviorContract.js

@@ -1455,6 +1455,18 @@ export function summarizeFramerChanges(changes) {
           sourceNodeId: variant[2],
         };
       }
+      const removed = command.match(/^DEL\s+([^\s;]+)/i);
+      if (removed) return { operation: 'delete', nodeId: removed[1] };
+      const duplicated = command.match(/^DUPE\s+([^\s]+).*?\bnewId="([^"]+)"/i);
+      if (duplicated) {
+        return {
+          operation: 'duplicate',
+          nodeId: duplicated[2],
+          sourceNodeId: duplicated[1],
+        };
+      }
+      const moved = command.match(/^MOVE\s+([^\s]+)/i);
+      if (moved) return { operation: 'move', nodeId: moved[1] };
       const operation = text(command.split(/\s+/)[0], 40).toLowerCase();
       return { operation: operation || 'unknown' };
     });

package/src/protocol.js

@@ -488,7 +488,7 @@ export function buildOutcomePrompt(outcome = {}, product, promptContext = {}) {
       'Do not publish or deploy unless the assignment explicitly says publishing is authorized.',
       'You own verification. Your framer_plan_task declaration determines the required evidence, and the harness enforces it before accepting completion.',
       'framer_read_project screenshots are returned to you as images; judge the result from what you see in them. If a result carries screenshotWarning, you did not see that screenshot, so do not describe it or claim it as visual verification.',
-      'Every framer_apply_changes canvas mutation requires a screenshot before the first mutation and after the final mutation. Use verifies on final reads to establish link, responsive, code, or data checks when your plan requires them.',
+      'You decide which evidence the task needs. Declare visual verification in framer_plan_task when the result must be judged by eye, such as layout or styling work or a build from a reference image, and then inspect a screenshot after your final mutation. When building from a reference attachment, compare that final screenshot against the reference. A screenshot before editing is optional; take one when you need to preserve or compare the current look. Use verifies on final reads to establish link, responsive, code, or data checks when your plan requires them.',
       'A generic read cannot verify interactions. framer_verify_interactions derives success requirements from the behavior contract and chosen mechanism; supply canonical evidence node IDs rather than defining your own success test. It returns verified, contradicted, or unknown. Unknown means the available representation could not prove the behavior; it is not a defect and must not trigger mutation. A grounded semanticAssessment may resolve unknown evidence when enabled, but it cannot override a deterministic contradiction or authorize writes. Native links are verified from href, URL, route, destination, or component-control evidence. Native effects are verified from reflected hoverEffect, tapEffect, or appearEffect properties and are structural evidence, not runtime playback.',
       'For repeated behavior, declare every targetNodeId and one representativeTargetNodeId in the mechanism plan. Implement and verify only that representative before fan-out, then verify complete target coverage after the final mutation. Only a contradicted result permits one evidence-scoped repair. Repeated unchanged evidence or exhausted verification budget must end as partial rather than starting another loop.',
       'Do not reuse a mechanism recorded as rejected by the user or repeated contradictory live evidence. Unknown evidence never rejects a mechanism.',