@reticlehq/server 3.8.2 → 3.10.0

package/dist/features/visual/desktop-capture.d.ts

@@ -0,0 +1,33 @@
+import { VisualReason } from '@reticlehq/core';
+import type { ToolDeps } from '../../surface/tools/tool-kit.js';
+/**
+ * Ask the desktop shell to photograph its own window.
+ *
+ * Returns undefined for a plain web page (no capture helper) and for a desktop app that did not
+ * install `@reticlehq/electron/main`. Deliberately NOT a screen-region capture: photographing
+ * the glass returns whatever window is on top, which would save a picture of the user's editor as a
+ * visual baseline — the exact false green Reticle exists to eliminate.
+ */
+export declare function desktopCapture(deps: ToolDeps, sessionId: string | undefined, fullPage: boolean): Promise<{
+    png?: Uint8Array;
+    reason?: VisualReason;
+    recommendation?: string;
+}>;
+/**
+ * Only read a capture the shell wrote. The path arrives from the PAGE, and the daemon must not
+ * become a file-read oracle for whatever a compromised renderer names.
+ *
+ * Two layouts are accepted, and the pair is the whole subtlety:
+ *
+ * - `<tmp>/<prefix><dir>/<prefix><n>.png` — the shell's own PRIVATE capture directory, created 0700
+ *   so no other local user can read the screenshot or pre-place a symlink at the path we write.
+ * - `<tmp>/<prefix><pid>-<n>.png` — the legacy flat layout. Kept because the shell ships as a
+ *   SEPARATE package on the user's own upgrade schedule: an app still on the older
+ *   `@reticlehq/electron` talking to a newer daemon must keep getting screenshots, not silently
+ *   start returning no image. It grants the daemon nothing it did not already have.
+ *
+ * Every comparison is an exact match against an UNRESOLVED path, which is what refuses traversal:
+ * `<tmp>/<prefix>x/../../etc/<prefix>passwd.png` has a grandparent of `<tmp>/<prefix>x/..`, not
+ * `<tmp>`. Exactly one directory level is allowed, so a deeper nesting is refused too.
+ */
+export declare function isCapturePath(path: string): boolean;

package/dist/features/visual/desktop-capture.js

@@ -0,0 +1,94 @@
+/**
+ * The desktop shell's photograph of its own window, read from the file the shell wrote.
+ *
+ * Shared by `reticle_screenshot`, the drive frames and the step shots. The step shots use it only
+ * for a page SDK before 3.7, which has no `render` command and declares `capture` instead.
+ */
+import { readFile, unlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { basename, dirname } from 'node:path';
+import { asRecord, asString, ReticleCommand, RETICLE_CAPTURE_FILE_PREFIX, RETICLE_TAURI_INERT_MARKER, TAURI_INERT_RECOMMENDATION, VisualReason, } from '@reticlehq/core';
+import { trackCaptureDirectory } from './capture-cleanup.js';
+/**
+ * Ask the desktop shell to photograph its own window.
+ *
+ * Returns undefined for a plain web page (no capture helper) and for a desktop app that did not
+ * install `@reticlehq/electron/main`. Deliberately NOT a screen-region capture: photographing
+ * the glass returns whatever window is on top, which would save a picture of the user's editor as a
+ * visual baseline — the exact false green Reticle exists to eliminate.
+ */
+export async function desktopCapture(deps, sessionId, fullPage) {
+    const session = deps.sessions.resolve(sessionId);
+    // A session that cannot take commands (no live browser behind it) simply has no pixels to give.
+    if (typeof session.command !== 'function') {
+        return {};
+    }
+    const res = await session.command(ReticleCommand.CAPTURE, { fullPage });
+    if (!res.ok) {
+        return {};
+    }
+    const r = asRecord(res.result);
+    if (r['ok'] !== true) {
+        // The shell refused a full-page capture it cannot do. That is a different answer from "capture
+        // broke", and the caller has to hear it — otherwise they read `fullPage` as honoured.
+        if (r['reason'] === VisualReason.FULL_PAGE_UNSUPPORTED) {
+            return { reason: VisualReason.FULL_PAGE_UNSUPPORTED };
+        }
+        // A Tauri release build with the capture compiled out: say that, never the web answer.
+        return true === asString(r['reason'])?.includes(RETICLE_TAURI_INERT_MARKER)
+            ? { reason: VisualReason.NO_PROVIDER, recommendation: TAURI_INERT_RECOMMENDATION }
+            : {};
+    }
+    const path = asString(r['path']);
+    if (path === undefined || !isCapturePath(path)) {
+        return {};
+    }
+    try {
+        const bytes = await readFile(path);
+        await unlink(path).catch(() => undefined);
+        trackCaptureDirectory(path);
+        return isCompletePng(bytes) ? { png: new Uint8Array(bytes) } : {};
+    }
+    catch {
+        return {};
+    }
+}
+/**
+ * Only read a capture the shell wrote. The path arrives from the PAGE, and the daemon must not
+ * become a file-read oracle for whatever a compromised renderer names.
+ *
+ * Two layouts are accepted, and the pair is the whole subtlety:
+ *
+ * - `<tmp>/<prefix><dir>/<prefix><n>.png` — the shell's own PRIVATE capture directory, created 0700
+ *   so no other local user can read the screenshot or pre-place a symlink at the path we write.
+ * - `<tmp>/<prefix><pid>-<n>.png` — the legacy flat layout. Kept because the shell ships as a
+ *   SEPARATE package on the user's own upgrade schedule: an app still on the older
+ *   `@reticlehq/electron` talking to a newer daemon must keep getting screenshots, not silently
+ *   start returning no image. It grants the daemon nothing it did not already have.
+ *
+ * Every comparison is an exact match against an UNRESOLVED path, which is what refuses traversal:
+ * `<tmp>/<prefix>x/../../etc/<prefix>passwd.png` has a grandparent of `<tmp>/<prefix>x/..`, not
+ * `<tmp>`. Exactly one directory level is allowed, so a deeper nesting is refused too.
+ */
+export function isCapturePath(path) {
+    if (!basename(path).startsWith(RETICLE_CAPTURE_FILE_PREFIX)) {
+        return false;
+    }
+    const parent = dirname(path);
+    if (parent === tmpdir()) {
+        return true;
+    }
+    return dirname(parent) === tmpdir() && basename(parent).startsWith(RETICLE_CAPTURE_FILE_PREFIX);
+}
+/**
+ * A PNG is complete only if it ends with IEND. This check exists because a truncated image once
+ * sailed through as a saved baseline: the tool answered `saved: true` with a path, and the file was
+ * undecodable. A screenshot that cannot be read must fail loudly, never bank a corrupt baseline that
+ * a later visual_diff will compare against.
+ */
+function isCompletePng(bytes) {
+    if (bytes.byteLength < 12) {
+        return false;
+    }
+    return 'IEND' === bytes.subarray(bytes.byteLength - 8, bytes.byteLength - 4).toString('ascii');
+}

package/dist/features/visual/step-shots.js

@@ -14,9 +14,10 @@
  * nothing at all, the next one is awaited, so the agent is told `{ error }` instead of a path to a
  * file that will never exist.
  *
- * ponytail: the desktop shell's own window capture is not a source here. It returns a device-pixel
- * PNG, and the daemon ships no JPEG encoder to shrink it to the step format; an Electron renderer
- * draws itself fine in-page. Add it when the shells can hand back a JPEG.
+ * A page SDK before 3.7 has no `render`. When such a page declares the desktop shell's `capture`,
+ * that command is asked instead. Its answer is a device-pixel PNG, and the daemon ships no JPEG
+ * encoder, so that picture is kept as a `.png` and stays on this machine (the platform takes only
+ * JPEG). A page that declares neither command is not asked: the result says to update its SDK.
  */
 import { createHash } from 'node:crypto';
 import { join } from 'node:path';
@@ -25,7 +26,12 @@ import { runIdOfCall } from '../../judgement/runs/drive-run.js';
 import { noteLanding } from '../../judgement/runs/artifact/run-store.js';
 import { isValidRunId, stepShotDir } from '../../memory/project/dir/reticle-dir.js';
 import { UPLOAD_TEXT_MAX } from '@reticlehq/core/artifacts';
+import { sdkBehindNote } from '../../command/version/version-skew.js';
+import { desktopCapture } from './desktop-capture.js';
 const JPEG_EXT = '.jpg';
+const PNG_EXT = '.png';
+/** Why a page that declared neither `render` nor `capture` was not asked for a picture. */
+const NO_PAGE_CAPTURE = 'this page declared no command that draws it (neither "render" nor "capture"), so nothing was sent';
 const TOOL_PREFIX = /^reticle_/;
 const SEQ_WIDTH = 4;
 /** A prior capture on this tab produced nothing — see the header. */
@@ -34,19 +40,58 @@ const NOTHING = 'none';
 const CDP_QUALITY = Math.round(STEP_SCREENSHOT.JPEG_QUALITY * 100);
 /** What worked last on each tab: the next result names that source. */
 const lastSource = new Map();
+/**
+ * The file type of the last picture of each tab: the next path is named with it.
+ *
+ * ponytail: a tab that changes type between two steps gets one predicted path with the old type;
+ * the run record names the real file. Wait for the capture if that ever matters.
+ */
+const lastExt = new Map();
 /** The next sequence number per run directory, read from its record the first time. */
 const nextSeq = new Map();
+/** The PNG signature's first four bytes, and where the IHDR chunk keeps the width and height. */
+const PNG_MAGIC = [0x89, 0x50, 0x4e, 0x47];
+const PNG_WIDTH_AT = 16;
+const PNG_HEIGHT_AT = 20;
+/** Width and height from a PNG's header, or undefined when the bytes are not a PNG. */
+function pngSize(bytes) {
+    if (bytes.length < PNG_HEIGHT_AT + 4 || PNG_MAGIC.some((b, i) => b !== bytes[i])) {
+        return undefined;
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const width = view.getUint32(PNG_WIDTH_AT);
+    const height = view.getUint32(PNG_HEIGHT_AT);
+    return width > 0 && height > 0 ? { width, height } : undefined;
+}
+/**
+ * The command that asks this page for its own picture, or undefined when it declared none.
+ *
+ * `render` came in SDK 3.7. An older SDK declares only the desktop shell's `capture`. A page that
+ * declared nothing is too old to say, so it is asked for `render` as before.
+ */
+function pageCaptureCommand(session) {
+    const declared = session.commands;
+    if (declared === undefined || declared.includes(ReticleCommand.RENDER)) {
+        return ReticleCommand.RENDER;
+    }
+    if (declared.includes(ReticleCommand.CAPTURE)) {
+        return ReticleCommand.CAPTURE;
+    }
+    return undefined;
+}
 /**
  * Width and height from a JPEG's frame header, or undefined when the bytes are not a JPEG.
  * Read rather than trusted: the run record and the upload both state the size.
  */
 export function jpegSize(bytes) {
-    if (bytes.length < 4 || 0xff !== bytes[0] || 0xd8 !== bytes[1])
+    if (bytes.length < 4 || 0xff !== bytes[0] || 0xd8 !== bytes[1]) {
         return undefined;
+    }
     let i = 2;
     while (i + 9 < bytes.length) {
-        if (0xff !== bytes[i])
+        if (0xff !== bytes[i]) {
             return undefined;
+        }
         const marker = bytes[i + 1] ?? 0;
         const length = ((bytes[i + 2] ?? 0) << 8) | (bytes[i + 3] ?? 0);
         // The start-of-frame markers carry the size; 0xC4, 0xC8 and 0xCC share the range and are not frames.
@@ -63,22 +108,32 @@ export function jpegSize(bytes) {
 async function browserPixels(deps, session) {
     const opts = { jpegQuality: CDP_QUALITY, cssScale: true };
     const leased = await deps.pool?.screenshotLease(session.id, opts);
-    if (leased !== undefined)
+    if (leased !== undefined) {
         return leased;
+    }
+    // A lease capture that failed still has a reason. Leaving it on the floor made the only text
+    // the page's "I cannot draw", which says nothing about the browser that was asked first.
+    const failed = deps.pool?.leaseCaptureError(session.id);
+    if (failed !== undefined) {
+        throw new Error(failed);
+    }
     // Only a provider driving THIS tab: a launched provider photographs the one page it owns whatever
     // url it is handed, which is how a screenshot of one tab was once filed under another (#1407).
-    if (true !== (await deps.realInput?.isAvailableFor(session.url)))
+    if (true !== (await deps.realInput?.isAvailableFor(session.url))) {
         return undefined;
+    }
     return deps.realInput?.screenshot?.(session.url, opts);
 }
 /** The page drawn by the SDK. The answer is base64 in chunks — see `ReticleCommand.RENDER`. */
 async function renderedPixels(session) {
     const res = await session.command(ReticleCommand.RENDER, {});
-    if (!res.ok)
+    if (!res.ok) {
         return res.error ?? 'the page did not answer';
+    }
     const r = asRecord(res.result);
-    if (true !== r['ok'])
+    if (true !== r['ok']) {
         return 'string' === typeof r['reason'] ? r['reason'] : 'render failed';
+    }
     const chunks = Array.isArray(r['chunks']) ? r['chunks'] : [];
     return new Uint8Array(Buffer.from(chunks.filter((c) => 'string' === typeof c).join(''), 'base64'));
 }
@@ -88,20 +143,47 @@ async function capture(deps, session) {
     try {
         const real = await browserPixels(deps, session);
         const size = real === undefined ? undefined : jpegSize(real);
-        if (real !== undefined && size !== undefined)
-            return { bytes: real, source: ScreenshotSource.BROWSER, ...size };
+        if (real !== undefined && size !== undefined) {
+            return { bytes: real, source: ScreenshotSource.BROWSER, ext: JPEG_EXT, ...size };
+        }
     }
     catch (error) {
         reasons.push(error instanceof Error ? error.message : String(error));
     }
+    const command = pageCaptureCommand(session);
+    if (command === undefined) {
+        // Nothing is sent: the page said it has no command for this.
+        reasons.push(NO_PAGE_CAPTURE);
+    }
+    if (command !== ReticleCommand.RENDER) {
+        if (command === ReticleCommand.CAPTURE) {
+            const png = (await desktopCapture(deps, session.id, false).catch(() => undefined))?.png;
+            const size = png === undefined ? undefined : pngSize(png);
+            if (png !== undefined && size !== undefined) {
+                return { bytes: png, source: ScreenshotSource.BROWSER, ext: PNG_EXT, ...size };
+            }
+            reasons.push('the desktop shell did not capture its window');
+        }
+        const behind = sdkBehindNote(session.sdkVersion);
+        if (behind !== undefined) {
+            reasons.push(behind);
+        }
+        return `no screenshot: ${reasons.join('; ')}`;
+    }
     try {
         const drawn = await renderedPixels(session);
-        if ('string' === typeof drawn)
+        if ('string' === typeof drawn) {
             reasons.push(`the page could not draw itself: ${drawn}`);
+            const behind = sdkBehindNote(session.sdkVersion);
+            if (behind !== undefined) {
+                reasons.push(behind);
+            }
+        }
         else {
             const size = jpegSize(drawn);
-            if (size !== undefined)
-                return { bytes: drawn, source: ScreenshotSource.RENDERED, ...size };
+            if (size !== undefined) {
+                return { bytes: drawn, source: ScreenshotSource.RENDERED, ext: JPEG_EXT, ...size };
+            }
             reasons.push('the page drew something that is not a JPEG');
         }
     }
@@ -113,12 +195,15 @@ async function capture(deps, session) {
 /** Which source this tab's next result names, or undefined when the last capture got nothing. */
 async function predict(deps, session) {
     const last = lastSource.get(session.id);
-    if (NOTHING === last)
+    if (NOTHING === last) {
         return undefined;
-    if (last !== undefined)
+    }
+    if (last !== undefined) {
         return last;
-    if (true === deps.pool?.leasedSessionIds().includes(session.id))
+    }
+    if (true === deps.pool?.leasedSessionIds().includes(session.id)) {
         return ScreenshotSource.BROWSER;
+    }
     const driven = deps.realInput?.screenshot !== undefined &&
         true === (await deps.realInput.isAvailableFor(session.url));
     return driven ? ScreenshotSource.BROWSER : ScreenshotSource.RENDERED;
@@ -168,8 +253,9 @@ async function shoot(deps, session, tool, { step, verdict }) {
     // Read before anything is awaited: the action this call's verdict was recorded on.
     const actionId = true === verdict ? session.lastActionId : undefined;
     const runId = runIdOfCall(session.id);
-    if (!isValidRunId(runId))
+    if (!isValidRunId(runId)) {
         return { error: `no screenshot: unusable run id ${runId}` };
+    }
     const dir = stepShotDir(root, runId);
     void filmTab(deps, session, dir);
     const seq = await allocate(deps.fs, dir);
@@ -185,14 +271,17 @@ async function shoot(deps, session, tool, { step, verdict }) {
     // Started before anything else is awaited: the picture is of THIS moment.
     const taken = capture(deps, session).then((pixels) => fresh && 'string' === typeof pixels ? capture(deps, session) : pixels);
     const predicted = fresh ? undefined : await predict(deps, session);
-    const file = `${String(seq).padStart(SEQ_WIDTH, '0')}-${tool.replace(TOOL_PREFIX, '')}${JPEG_EXT}`;
-    const path = join(dir, file);
+    const name = (ext) => `${String(seq).padStart(SEQ_WIDTH, '0')}-${tool.replace(TOOL_PREFIX, '')}${ext}`;
+    const predictedPath = join(dir, name(lastExt.get(session.id) ?? JPEG_EXT));
     const land = taken.then(async (pixels) => {
         if ('string' === typeof pixels) {
             lastSource.set(session.id, NOTHING);
             return;
         }
         lastSource.set(session.id, pixels.source);
+        lastExt.set(session.id, pixels.ext);
+        const file = name(pixels.ext);
+        const path = join(dir, file);
         try {
             await deps.fs.mkdir(dir);
             await deps.fs.writeFileBytes(path, pixels.bytes);
@@ -215,12 +304,15 @@ async function shoot(deps, session, tool, { step, verdict }) {
         }
     });
     noteLanding(dir, land);
-    if (predicted !== undefined)
-        return { path, source: predicted, pending: true };
+    if (predicted !== undefined) {
+        return { path: predictedPath, source: predicted, pending: true };
+    }
     // Nothing worked last time: wait, and tell the truth either way.
     const pixels = await taken;
     await land;
-    return 'string' === typeof pixels ? { error: pixels } : { path, source: pixels.source };
+    return 'string' === typeof pixels
+        ? { error: pixels }
+        : { path: join(dir, name(pixels.ext)), source: pixels.source };
 }
 /** The replay hook that gives every reported flow step its screenshot, filed as `<flow>#<step>`. */
 export function replayStepShooter(deps, sessionId) {
@@ -267,8 +359,9 @@ const films = new Map();
 const framesIn = new Map();
 /** Is this a tab Reticle drives, with a real browser behind it? */
 async function driven(deps, session) {
-    if (true === deps.pool?.leasedSessionIds().includes(session.id))
+    if (true === deps.pool?.leasedSessionIds().includes(session.id)) {
         return true;
+    }
     return (deps.realInput?.screenshot !== undefined &&
         true === (await deps.realInput.isAvailableFor(session.url)));
 }
@@ -281,10 +374,12 @@ async function filmTab(deps, session, dir) {
         film.still = 1;
         return;
     }
-    if (!(await driven(deps, session).catch(() => false)))
+    if (!(await driven(deps, session).catch(() => false))) {
         return;
-    if (films.has(session.id))
+    }
+    if (films.has(session.id)) {
         return;
+    }
     const timer = setInterval(() => {
         void filmNow(session.id);
     }, FRAME.EVERY_MS);
@@ -295,16 +390,18 @@ async function filmTab(deps, session, dir) {
 /** Stop filming a tab. Safe to call for a tab that is not filmed. */
 export function stopFilming(sessionId) {
     const film = films.get(sessionId);
-    if (film === undefined)
+    if (film === undefined) {
         return;
+    }
     clearInterval(film.timer);
     films.delete(sessionId);
 }
 /** One tick of a tab's film: one frame, when the budget, the gap and the page allow. Never throws. */
 export async function filmNow(sessionId) {
     const film = films.get(sessionId);
-    if (film === undefined || film.busy !== undefined)
+    if (film === undefined || film.busy !== undefined) {
         return;
+    }
     film.busy = frame(sessionId, film).finally(() => {
         film.busy = undefined;
     });
@@ -315,8 +412,9 @@ async function frame(sessionId, film) {
     try {
         session = film.deps.sessions.resolve(sessionId);
         // An ended tab has nothing more to show.
-        if (session.isEnded())
+        if (session.isEnded()) {
             session = undefined;
+        }
     }
     catch {
         session = undefined;
@@ -327,18 +425,21 @@ async function frame(sessionId, film) {
     }
     const dir = film.dir;
     const kept = framesIn.get(dir) ?? 0;
-    if (kept >= FRAME.MAX_PER_RUN)
+    if (kept >= FRAME.MAX_PER_RUN) {
         return;
+    }
     film.idle += 1;
     const gap = 2 ** Math.floor(kept / (FRAME.MAX_PER_RUN / FRAME_STAGES));
-    if (film.idle < Math.max(gap, film.still))
+    if (film.idle < Math.max(gap, film.still)) {
         return;
+    }
     film.idle = 0;
     try {
         const bytes = await framePixels(film.deps, session);
         const size = bytes === undefined ? undefined : jpegSize(bytes);
-        if (bytes === undefined || size === undefined)
+        if (bytes === undefined || size === undefined) {
             return;
+        }
         const sha256 = createHash('sha256').update(bytes).digest('hex');
         if (film.recent.includes(sha256)) {
             film.still = Math.min(film.still * 2, MAX_STILL_TICKS);
@@ -348,8 +449,9 @@ async function frame(sessionId, film) {
         film.still = 1;
         const seq = await allocate(film.deps.fs, dir);
         // The step shots' own limit is the platform's: a frame never takes the last of it.
-        if (seq > STEP_SCREENSHOT.MAX_PER_RUN)
+        if (seq > STEP_SCREENSHOT.MAX_PER_RUN) {
             return;
+        }
         framesIn.set(dir, kept + 1);
         const file = `${String(seq).padStart(SEQ_WIDTH, '0')}-${FRAME.TOOL}${JPEG_EXT}`;
         const record = {
@@ -377,9 +479,11 @@ async function frame(sessionId, film) {
 async function framePixels(deps, session) {
     const opts = { jpegQuality: FRAME.JPEG_QUALITY, cssScale: true };
     const leased = await deps.pool?.screenshotLease(session.id, opts);
-    if (leased !== undefined)
+    if (leased !== undefined) {
         return leased;
-    if (true !== (await deps.realInput?.isAvailableFor(session.url)))
+    }
+    if (true !== (await deps.realInput?.isAvailableFor(session.url))) {
         return undefined;
+    }
     return deps.realInput?.screenshot?.(session.url, opts);
 }

package/dist/features/visual/visual-tools.d.ts

@@ -8,24 +8,6 @@ import type { ToolDef, ToolDeps } from '../../surface/tools/tool-kit.js';
  * gone mid-drive must not break the drive.
  */
 export declare function driveFrame(deps: ToolDeps, sessionId: string, jpegQuality: number): Promise<Uint8Array | undefined>;
-/**
- * Only read a capture the shell wrote. The path arrives from the PAGE, and the daemon must not
- * become a file-read oracle for whatever a compromised renderer names.
- *
- * Two layouts are accepted, and the pair is the whole subtlety:
- *
- * - `<tmp>/<prefix><dir>/<prefix><n>.png` — the shell's own PRIVATE capture directory, created 0700
- *   so no other local user can read the screenshot or pre-place a symlink at the path we write.
- * - `<tmp>/<prefix><pid>-<n>.png` — the legacy flat layout. Kept because the shell ships as a
- *   SEPARATE package on the user's own upgrade schedule: an app still on the older
- *   `@reticlehq/electron` talking to a newer daemon must keep getting screenshots, not silently
- *   start returning no image. It grants the daemon nothing it did not already have.
- *
- * Every comparison is an exact match against an UNRESOLVED path, which is what refuses traversal:
- * `<tmp>/<prefix>x/../../etc/<prefix>passwd.png` has a grandparent of `<tmp>/<prefix>x/..`, not
- * `<tmp>`. Exactly one directory level is allowed, so a deeper nesting is refused too.
- */
-export declare function isCapturePath(path: string): boolean;
 /**
  * The opt-in pixel layer. Both tools require a DRIVEN browser (reticle drive /
  * RETICLE_CDP_URL) — the always-on SDK ships no screenshotter — and return a structured NO_PROVIDER