openmausbot 0.1.55

package/dist-server/server/computer-control.js

@@ -0,0 +1,149 @@
+// Who is driving a bot's computer — the person or the bot. One record per
+// bot, held in the harness because every consumer (the panel, the SSE
+// stream, and the per-turn computer proxies) already talks to the harness.
+//
+// The rules this module exists to enforce:
+//   - A bot can only ASK for hands (`requestHelp`); it can never take
+//     control, and it cannot clear a hold. Only the person takes and
+//     releases, from the computer panel.
+//   - While the person holds control, the bot's computer actions are
+//     REFUSED by the proxies, not queued. A queued click would land after
+//     the person has moved on, on whatever happens to be under it.
+//   - Releasing control also settles any open help request, so a bot
+//     waiting on `requestHelp` wakes up from the same state change the
+//     person made — there is no separate "done helping" step to forget.
+//
+// State is per-boot and in-memory on purpose: a hold is a live fact about
+// who is at the screen right now, and surviving a harness restart would
+// mean a stale hold silently bricking a bot's computer.
+const NO_CONTROL = { held: false, helpReason: null, heldSinceMs: null };
+/** Keep a shouted help reason card-sized; the transcript has the rest. */
+const MAX_REASON_CHARS = 280;
+export class ComputerControl {
+    entries = new Map();
+    onChange;
+    now;
+    requestSequence = 0;
+    constructor(onChange = () => { }, now = Date.now) {
+        this.onChange = onChange;
+        this.now = now;
+    }
+    snapshot(botId) {
+        const entry = this.entries.get(botId);
+        if (!entry)
+            return NO_CONTROL;
+        return {
+            held: entry.heldSinceMs !== null,
+            helpReason: entry.helpReason,
+            heldSinceMs: entry.heldSinceMs,
+        };
+    }
+    /** The person takes the wheel. Idempotent — a second click must not
+     * reset `heldSinceMs` and make the hold look newer than it is. */
+    take(botId) {
+        const entry = this.entries.get(botId);
+        if (entry?.heldSinceMs != null)
+            return this.snapshot(botId);
+        this.entries.set(botId, {
+            heldSinceMs: this.now(),
+            helpReason: entry?.helpReason ?? null,
+            helpRequestId: entry?.helpRequestId ?? null,
+            controlLeaseId: null,
+        });
+        return this.changed(botId);
+    }
+    /** Atomically take or re-check a workspace-owned hold. The opaque lease is
+     * never returned in a snapshot, broadcast, or API response. */
+    acquireLease(botId, controlLeaseId) {
+        const entry = this.entries.get(botId);
+        if (entry?.heldSinceMs != null) {
+            return {
+                snapshot: this.snapshot(botId),
+                owned: entry.controlLeaseId === controlLeaseId,
+                acquired: false,
+            };
+        }
+        this.entries.set(botId, {
+            heldSinceMs: this.now(),
+            helpReason: entry?.helpReason ?? null,
+            helpRequestId: entry?.helpRequestId ?? null,
+            controlLeaseId,
+        });
+        return { snapshot: this.changed(botId), owned: true, acquired: true };
+    }
+    /** The person hands the wheel back. Also settles any open help request —
+     * the waiting bot resumes from this one state change. */
+    release(botId) {
+        if (!this.entries.has(botId))
+            return NO_CONTROL;
+        this.entries.delete(botId);
+        return this.changed(botId);
+    }
+    /** Release only the hold created by this workspace lease. A newer or legacy
+     * holder is observed but never disturbed. */
+    releaseLease(botId, controlLeaseId) {
+        const entry = this.entries.get(botId);
+        if (!entry || entry.heldSinceMs === null || entry.controlLeaseId !== controlLeaseId) {
+            return { snapshot: this.snapshot(botId), released: false };
+        }
+        this.entries.delete(botId);
+        return { snapshot: this.changed(botId), released: true };
+    }
+    /** The bot asks the person to take over. Never grants anything by
+     * itself — it only surfaces the plea. A reason shouted while the person
+     * is already driving is kept, but must not clobber an earlier one they
+     * may still be reading. */
+    requestHelp(botId, reason) {
+        return this.requestHelpLease(botId, reason).snapshot;
+    }
+    /** Open a help request and return the lease that owns it. A proxy uses
+     * this id to expire only its own unanswered plea when its wait ends. */
+    requestHelpLease(botId, reason) {
+        const text = typeof reason === "string" ? reason.trim().slice(0, MAX_REASON_CHARS) : "";
+        const entry = this.entries.get(botId) ?? {
+            heldSinceMs: null,
+            helpReason: null,
+            helpRequestId: null,
+            controlLeaseId: null,
+        };
+        if (entry.helpReason === null) {
+            entry.helpReason = text || "the bot asked you to take over";
+            entry.helpRequestId = `${botId}-${++this.requestSequence}`;
+        }
+        this.entries.set(botId, entry);
+        return { snapshot: this.changed(botId), requestId: entry.helpRequestId };
+    }
+    /** The person declines without taking over; the waiting bot is told. */
+    dismissHelp(botId) {
+        const entry = this.entries.get(botId);
+        if (!entry || entry.helpReason === null)
+            return this.snapshot(botId);
+        entry.helpReason = null;
+        entry.helpRequestId = null;
+        if (entry.heldSinceMs === null)
+            this.entries.delete(botId);
+        return this.changed(botId);
+    }
+    /** Expire an unanswered plea. The id comparison prevents an old proxy's
+     * timeout from dismissing a newer request for the same bot. */
+    expireHelp(botId, requestId) {
+        const entry = this.entries.get(botId);
+        if (!entry || entry.helpRequestId !== requestId)
+            return this.snapshot(botId);
+        entry.helpReason = null;
+        entry.helpRequestId = null;
+        if (entry.heldSinceMs === null)
+            this.entries.delete(botId);
+        return this.changed(botId);
+    }
+    /** The bot is gone; a hold on a deleted bot's computer means nothing. */
+    forget(botId) {
+        if (this.entries.delete(botId))
+            this.onChange(botId, NO_CONTROL);
+    }
+    changed(botId) {
+        const snapshot = this.snapshot(botId);
+        this.onChange(botId, snapshot);
+        return snapshot;
+    }
+}

package/dist-server/server/computer-observation.js

@@ -0,0 +1,133 @@
+import { createHash } from "node:crypto";
+export const emptyObservationMetrics = () => ({
+    screenshotsCaptured: 0,
+    screenshotsSentToModel: 0,
+    fullScreenObservations: 0,
+    croppedObservations: 0,
+    structuredBrowserObservations: 0,
+    computerActions: 0,
+    retries: 0,
+    verificationSuccesses: 0,
+    verificationFailures: 0,
+});
+export function normalizeCrop(raw, maxWidth, maxHeight) {
+    if (!raw || typeof raw !== "object")
+        return null;
+    const value = raw;
+    const x = Math.round(Number(value.x));
+    const y = Math.round(Number(value.y));
+    const width = Math.round(Number(value.width));
+    const height = Math.round(Number(value.height));
+    if (![x, y, width, height, maxWidth, maxHeight].every(Number.isFinite) ||
+        maxWidth <= 0 ||
+        maxHeight <= 0 ||
+        x < 0 ||
+        y < 0 ||
+        width < 32 ||
+        height < 32) {
+        return null;
+    }
+    if (x + width > maxWidth || y + height > maxHeight)
+        return null;
+    return { x, y, width, height };
+}
+/** Canonical value for internal navigation checks. Credentials are never
+ * needed for equality and are removed here; query and fragment remain so
+ * two distinct application states cannot verify as the same destination. */
+export function normalizeBrowserUrl(value) {
+    if (typeof value !== "string" || !value || value.length > 8_192)
+        return null;
+    try {
+        const url = new URL(value);
+        if (!/^https?:$/.test(url.protocol))
+            return null;
+        url.username = "";
+        url.password = "";
+        return url.toString();
+    }
+    catch {
+        return null;
+    }
+}
+/** Removes credentials, query, and fragment before browser state reaches a model or log. */
+export function safeBrowserUrl(value) {
+    const normalized = normalizeBrowserUrl(value);
+    if (!normalized)
+        return null;
+    const url = new URL(normalized);
+    url.search = "";
+    url.hash = "";
+    const safe = url.toString();
+    return safe.length <= 2_048 ? safe : null;
+}
+/** Parses Chrome's /json/list response into a small, safe structured observation. */
+export function parseBrowserTargets(raw) {
+    if (raw.length > 1_000_000)
+        return [];
+    try {
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed))
+            return [];
+        return parsed.slice(0, 20).flatMap((item) => {
+            if (!item || typeof item !== "object")
+                return [];
+            const value = item;
+            const comparisonUrl = normalizeBrowserUrl(value.url);
+            const url = safeBrowserUrl(value.url);
+            if (value.type !== "page" || !url || !comparisonUrl || typeof value.id !== "string")
+                return [];
+            const title = typeof value.title === "string" ? value.title.replace(/\s+/g, " ").trim().slice(0, 200) : "";
+            return [{ id: value.id.slice(0, 100), title, url, comparisonUrl }];
+        });
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Keeps observations cheap without claiming the screen is immutable. Every
+ * requested observation still captures fresh pixels (pages can change without
+ * an input action), while byte-identical frames are not sent to the model twice.
+ */
+export class ObservationCoordinator {
+    metrics = emptyObservationMetrics();
+    lastObservation = null;
+    noteAction(count = 1) {
+        this.metrics.computerActions += Math.max(0, Math.trunc(count));
+    }
+    noteRetry() {
+        this.metrics.retries += 1;
+    }
+    /** canonicalFrame must describe the full screenshot, even when the image
+     * returned to the model is cropped. A box-provided full-frame hash works. */
+    observeFrame(canonicalFrame, crop) {
+        this.metrics.screenshotsCaptured += 1;
+        const hash = canonicalFrame
+            ? createHash("sha256").update(canonicalFrame).digest("hex")
+            : null;
+        const view = crop ? `${crop.x},${crop.y},${crop.width},${crop.height}` : "full";
+        const signature = hash ? `${hash}:${view}` : null;
+        // If the box cannot provide a full-frame hash, fail open and send the
+        // valid image. Suppressing a possibly-new crop would be worse.
+        const changed = signature === null || signature !== this.lastObservation;
+        if (signature)
+            this.lastObservation = signature;
+        if (changed) {
+            this.metrics.screenshotsSentToModel += 1;
+            if (crop)
+                this.metrics.croppedObservations += 1;
+            else
+                this.metrics.fullScreenObservations += 1;
+        }
+        return { changed, hash };
+    }
+    noteStructuredObservation() {
+        this.metrics.structuredBrowserObservations += 1;
+    }
+    noteVerification(ok) {
+        if (ok)
+            this.metrics.verificationSuccesses += 1;
+        else
+            this.metrics.verificationFailures += 1;
+    }
+}