@reticlehq/server 2.14.0 → 3.2.0

package/dist/portal/pool/browser-pool.js

@@ -0,0 +1,559 @@
+/**
+ * BrowserPool — one headless browser, many cheap isolated contexts.
+ *
+ * The "10 agents test 10 flows" scenario does NOT mean 10 Chromiums (each ~hundreds of MB). It means
+ * ONE launched browser and N `newContext` calls (each ~a few MB, fully isolated cookies/storage).
+ * The pool owns that single browser, hands out per-flow leases, caps concurrency so a machine is
+ * never overwhelmed (over-cap acquires queue FIFO), and transparently relaunches if the browser dies.
+ *
+ * The browser is injected via a `Launcher` so all lifecycle logic is testable without real Chromium;
+ * the thin Playwright adapter that satisfies these interfaces lives separately.
+ */
+import { unreachableUrlIn } from '@reticlehq/core';
+import { seedStorageInto } from './storage-seed.js';
+/** Default lease time-to-live: a lease untouched for this long is presumed orphaned and reclaimed. */
+export const DEFAULT_LEASE_TTL_MS = 5 * 60_000;
+/** Default cap on how long a lease's initial navigation may take before it fails (frees the slot). */
+const DEFAULT_NAV_TIMEOUT_MS = 30_000;
+/**
+ * How many reaped lease ids to remember. Only the most recent departure is ever asked about, so
+ * this is slack for a burst sweep, not a history.
+ */
+const MAX_REMEMBERED_REAPED_LEASES = 64;
+/**
+ * Owns a single browser and leases isolated contexts out of it, capped at `maxContexts`.
+ * Not exported as a singleton — the daemon owns one instance.
+ */
+export class BrowserPool {
+    #launch;
+    #max;
+    #genId;
+    #now;
+    #ttl;
+    #navTimeout;
+    #browser;
+    #launching;
+    #closed = false;
+    /** Leases reclaimed for going stale — see reapedLeaseCount(). */
+    #reapedLeases = 0;
+    /**
+     * The session ids of recently reaped leases, most recent last.
+     *
+     * The count above answers "has anything ever aged out", which is not a
+     * question any caller actually has: it latches true for the life of the
+     * daemon (#611). The ids answer the question the no-session diagnosis is
+     * really asking — was THIS session a lease that aged out — so a closed
+     * human tab is no longer blamed on an expired lease forever after.
+     *
+     * Bounded because a long-lived daemon reaps unboundedly many.
+     */
+    #reapedLeaseIds = [];
+    #active = new Map();
+    /** Registered-session-id → lease id, for apps that keep their own session name. See `alias`. */
+    #aliases = new Map();
+    /**
+     * Slots claimed-or-active — the real concurrency gate. Incremented SYNCHRONOUSLY the instant an
+     * acquire passes the cap check, BEFORE the async context creation, so a burst of concurrent
+     * acquires (the "10 agents at once" case) can't all slip through the gate before any has been
+     * recorded. `#active.size` only reflects fully-created leases; `#occupied` also counts in-flight
+     * ones, and is what the cap is enforced against.
+     */
+    #occupied = 0;
+    /** FIFO of acquires waiting for a slot; each resolves when one frees. */
+    #waiters = [];
+    constructor(launch, opts) {
+        if (opts.maxContexts < 1)
+            throw new Error('maxContexts must be >= 1');
+        this.#launch = launch;
+        this.#max = opts.maxContexts;
+        this.#genId = opts.genSessionId;
+        this.#now = opts.now ?? (() => Date.now());
+        this.#ttl = opts.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS;
+        this.#navTimeout = opts.navTimeoutMs ?? DEFAULT_NAV_TIMEOUT_MS;
+    }
+    /** The TTL configured for leases — how long an untouched lease lives before the reaper reclaims it. */
+    leaseTtlMs() {
+        return this.#ttl;
+    }
+    /** Max simultaneous leases — the ceiling the parallel suite sizes its concurrency to. */
+    capacity() {
+        return this.#max;
+    }
+    activeCount() {
+        return this.#active.size;
+    }
+    /** Acquires waiting for a free slot. */
+    queuedCount() {
+        return this.#waiters.length;
+    }
+    /** The sessionIds of every currently leased context — used to report/group leased sessions. */
+    leasedSessionIds() {
+        return [...this.#active.keys()];
+    }
+    /**
+     * The public session id of an active lease on this origin, if we already hold one.
+     *
+     * The lease TOOL reuses that id instead of minting a second context: a second acquire on the same
+     * origin used to leave both tabs connected and poison default session resolution (#600). The pool's
+     * own `acquire` still mints — the parallel suite needs isolation on purpose.
+     *
+     * Prefers the alias the agent was given, when the app registered under its own name.
+     */
+    leaseIdOnOrigin(origin) {
+        for (const [leaseId, lease] of this.#active) {
+            let leaseOrigin;
+            try {
+                leaseOrigin = new URL(lease.url).origin;
+            }
+            catch {
+                continue;
+            }
+            if (leaseOrigin !== origin)
+                continue;
+            for (const [alias, id] of this.#aliases) {
+                if (id === leaseId)
+                    return alias;
+            }
+            return leaseId;
+        }
+        return undefined;
+    }
+    /**
+     * Release a lease by its sessionId (closes the context, frees the slot). No-op if unknown.
+     *
+     * Resolves an alias first, because the agent releases with the id it was GIVEN, which for an app
+     * that named its own session is not the id the pool navigated with. Releasing by that name used to
+     * be a silent no-op that left the slot held until the reaper took it.
+     */
+    release(sessionId) {
+        return this.#release(this.#leaseIdOf(sessionId));
+    }
+    /**
+     * Record that a lease is also known by the name the APP registered.
+     *
+     * Leases are keyed by the id the pool navigated with, and the SDK normally adopts that id off the
+     * URL so the two agree. An app that passes an explicit `session` to `connect()` keeps its own name
+     * instead, which is legitimate, and the lease tool then hands the agent THAT name because it is the
+     * one it has to drive with. Every later touch and release arrives under it.
+     *
+     * Without this the touches hit nothing, the lease ages out despite continuous activity, and the
+     * reaper closes the context mid-flow — the failure in #157, where a session died on the very call
+     * that would have read a computed value.
+     */
+    alias(registeredId, leaseId) {
+        if (registeredId === leaseId)
+            return;
+        if (!this.#active.has(leaseId))
+            return;
+        this.#aliases.set(registeredId, leaseId);
+    }
+    /**
+     * The lease this id refers to: itself, or whatever it is an alias for.
+     *
+     * EVERY AGENT-FACING LOOKUP MUST GO THROUGH THIS. A leased page runs the SDK, which dials and
+     * registers under its OWN session id, so the id `reticle_sessions` shows an agent is routinely the
+     * alias rather than the lease key. Four capabilities used the raw map instead — screenshots,
+     * hover, network mocks and the dial-failure diagnostic — and every one of them refused for a
+     * lease that was alive and working, addressed by the only id the agent had been given.
+     *
+     * `reticle_network_mock` is where that surfaced: it reported `no-cdp-provider` in every
+     * configuration a reporter could reach, including after `reticle drive`, and the recommendation it
+     * printed ("start with `reticle drive <url>`") could not fix it because the provider was never the
+     * problem. Their conclusion was that error-path verification was impossible without shipping
+     * failure-injection code inside the application — which is exactly what an external mock exists to
+     * avoid.
+     *
+     * `acquire` and `#release` are the two exceptions and must stay raw: the first is registering the
+     * lease key itself before any alias can exist, and the second must delete that key rather than
+     * whatever it points at.
+     */
+    #leaseIdOf(sessionId) {
+        return this.#active.has(sessionId) ? sessionId : (this.#aliases.get(sessionId) ?? sessionId);
+    }
+    /** Mark a lease as still in use (called on agent activity) so the reaper doesn't reclaim it. */
+    touch(sessionId) {
+        const lease = this.#active.get(this.#leaseIdOf(sessionId));
+        if (lease !== undefined)
+            lease.touchedAt = this.#now();
+    }
+    /**
+     * Reclaim leases untouched for longer than the TTL — a hung/crashed agent never holds a slot
+     * forever. Returns the released sessionIds. This is the fault-tolerance backstop for the pool.
+     */
+    /**
+     * Capture a leased page, or undefined when this session is not a lease or cannot be captured.
+     *
+     * A lease IS a real browser page, so the pixels were always available — the visual tools simply had
+     * no route to them and answered "no provider" for every context an agent can actually get. That
+     * made visual regression impossible on exactly the isolated contexts the pool exists to hand out.
+     *
+     * Touches the lease like any other tool call, so capturing keeps it alive rather than letting the
+     * reaper take it out from under a diff.
+     */
+    async screenshotLease(sessionId, opts = {}) {
+        const lease = this.#active.get(this.#leaseIdOf(sessionId));
+        if (lease === undefined || lease.page.screenshot === undefined)
+            return undefined;
+        this.touch(sessionId);
+        try {
+            return await lease.page.screenshot(opts);
+        }
+        catch {
+            // A capture that fails is not a broken lease. Report "could not capture" and leave the lease
+            // usable — the alternative is losing a working context to one bad frame.
+            return undefined;
+        }
+    }
+    /**
+     * Move the real pointer on a leased page, or false when this session is not a lease or cannot
+     * hover. Same optional-capability contract as screenshotLease: absence must read as "could not
+     * hover", never as a successful dispatch of a synthetic mouseover.
+     *
+     * Touches the lease like any other tool call, so hovering keeps it alive.
+     */
+    async hoverLease(sessionId, x, y) {
+        const lease = this.#active.get(this.#leaseIdOf(sessionId));
+        if (lease === undefined || lease.page.hover === undefined)
+            return false;
+        this.touch(sessionId);
+        try {
+            await lease.page.hover(x, y);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Install network mocks on a leased page, or false when this session is not a lease or cannot
+     * intercept. Same optional-capability contract as screenshotLease: absence must read as "could not
+     * mock", never as a stub that applied to nothing.
+     *
+     * Touches the lease like any other tool call, so mocking keeps it alive.
+     */
+    /**
+     * Resize a leased page's viewport, or report that this page cannot be resized.
+     *
+     * Alias-resolved like every other agent-facing capability — see `#leaseIdOf`.
+     *
+     * Touches the lease like any other tool call, so resizing keeps it alive.
+     */
+    async setViewportLease(sessionId, size) {
+        const lease = this.#active.get(this.#leaseIdOf(sessionId));
+        if (lease === undefined || lease.page.setViewport === undefined)
+            return false;
+        this.touch(sessionId);
+        try {
+            await lease.page.setViewport(size);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async setMocksLease(sessionId, rules) {
+        const lease = this.#active.get(this.#leaseIdOf(sessionId));
+        if (lease === undefined || lease.page.installMocks === undefined)
+            return false;
+        this.touch(sessionId);
+        try {
+            await lease.page.installMocks(rules);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    async sweepExpired() {
+        const now = this.#now();
+        const expired = [...this.#active.entries()]
+            .filter(([, lease]) => now - lease.touchedAt > this.#ttl)
+            .map(([sessionId]) => sessionId);
+        for (const sessionId of expired)
+            await this.#release(sessionId);
+        this.#reapedLeases += expired.length;
+        for (const sessionId of expired) {
+            this.#reapedLeaseIds.push(sessionId);
+            if (this.#reapedLeaseIds.length > MAX_REMEMBERED_REAPED_LEASES)
+                this.#reapedLeaseIds.shift();
+        }
+        return expired;
+    }
+    /**
+     * How many leases this pool has reclaimed for going stale.
+     *
+     * Kept for reporting. NOT a basis for diagnosis: it is a lifetime total that never resets, so
+     * `count > 0` latches true forever after the first reap. Ask `wasReapedLease` instead, which
+     * answers about a specific session.
+     */
+    reapedLeaseCount() {
+        return this.#reapedLeases;
+    }
+    /**
+     * Did THIS session id belong to a lease this pool aged out?
+     *
+     * The question the no-session diagnosis actually has. It used to ask `reapedLeaseCount() > 0`,
+     * which meant that once any lease had ever aged out, every closed human tab for the rest of the
+     * daemon's life was reported as an expired lease — and the recovery that followed
+     * (`reticle_lease {acquire}`) would open an unauthenticated headless context and lose the app
+     * session (#611).
+     *
+     * Only the last {@link MAX_REMEMBERED_REAPED_LEASES} are remembered, so this answers `false` for
+     * a lease reaped long enough ago that thousands have followed. That direction is the safe one:
+     * it falls back to the generic "the tab was closed" wording rather than inventing a lease.
+     */
+    wasReapedLease(sessionId) {
+        return this.#reapedLeaseIds.includes(sessionId);
+    }
+    /**
+     * Lease an isolated context, navigate it to `url`, and return a handle. If the pool is at capacity,
+     * waits FIFO until a slot frees (or until `signal` aborts, if provided).
+     */
+    async acquire(url, opts = {}) {
+        if (this.#closed)
+            throw new Error('browser pool is shut down');
+        // #waitForSlot claims the slot synchronously (bumps #occupied) before returning, so the cap holds
+        // even when many acquires race through the gate in the same tick.
+        await this.#waitForSlot(opts.signal);
+        const sessionId = opts.sessionId ?? this.#genId();
+        let context;
+        let pending;
+        let pendingDialogMessage;
+        try {
+            const browser = await this.#ensureBrowser();
+            context = await browser.newContext();
+            const page = await context.newPage();
+            // Per-page crash isolation: if THIS renderer dies, reclaim only this lease — the shared browser
+            // and every other agent's context keep running. (A full browser death is handled by #onCrash.)
+            page.onCrash(() => {
+                void this.#release(sessionId);
+            });
+            // Listen BEFORE goto: the SDK's connect attempt races the navigation's own resolution, and a
+            // listener attached afterwards misses the first failures — the only ones logged before the
+            // retry backoff stretches out.
+            page.onConsole?.((text) => {
+                const dialled = unreachableUrlIn(text);
+                if (dialled === undefined)
+                    return;
+                const active = this.#active.get(sessionId);
+                if (active !== undefined)
+                    active.dialFailureUrl = dialled;
+                else
+                    pending = dialled; // logged during goto, before the lease is registered below
+            });
+            // Same "listen before goto" reasoning as onConsole above: a dialog opened during the initial
+            // navigation (an app that confirms something in an onload handler) must still be dismissed, not
+            // just ones opened after the lease is registered.
+            page.onDialog?.((dialog) => {
+                const active = this.#active.get(sessionId);
+                if (active !== undefined)
+                    active.lastDialogMessage = dialog.message;
+                else
+                    pendingDialogMessage = dialog.message;
+                void dialog.dismiss();
+            });
+            let seedHandle;
+            let checkSeedError;
+            if (opts.seedStorage !== undefined) {
+                const seedResult = await seedStorageInto(context, page, url, opts.seedStorage);
+                seedHandle = seedResult?.handle;
+                checkSeedError = seedResult?.checkError;
+            }
+            let navRes;
+            try {
+                navRes = await page.goto(url, { timeoutMs: this.#navTimeout });
+                checkSeedError?.();
+            }
+            finally {
+                if (seedHandle !== undefined) {
+                    await seedHandle.dispose().catch(() => undefined);
+                }
+            }
+            // The browser can crash WHILE goto is resolving; #onCrash then clears #active and zeroes
+            // #occupied. Registering the lease now would resurrect a dead entry against a crashed browser with
+            // the slot count out of sync (drifting below #active.size, eventually exceeding the cap). If we're
+            // no longer the live browser, bail — the catch below closes the context and returns the slot.
+            if (this.#browser !== browser)
+                throw new Error('browser crashed during navigation');
+            let navStatus;
+            if (null !== navRes && 'object' === typeof navRes && 'status' in navRes) {
+                const s = navRes.status;
+                if ('function' === typeof s) {
+                    try {
+                        const code = s.call(navRes);
+                        if ('number' === typeof code)
+                            navStatus = code;
+                    }
+                    catch {
+                        // ignore
+                    }
+                }
+                else if ('number' === typeof s) {
+                    navStatus = s;
+                }
+            }
+            this.#active.set(sessionId, {
+                context,
+                page,
+                url,
+                touchedAt: this.#now(),
+                ...(pending === undefined ? {} : { dialFailureUrl: pending }),
+                ...(pendingDialogMessage === undefined ? {} : { lastDialogMessage: pendingDialogMessage }),
+            });
+            return {
+                sessionId,
+                url,
+                ...(navStatus !== undefined ? { navStatus } : {}),
+                release: () => this.#release(sessionId),
+            };
+        }
+        catch (err) {
+            // Setup failed after we claimed the slot — give it back so a queued acquire isn't stuck, and
+            // close the half-open context.
+            if (context !== undefined)
+                await context.close().catch(() => undefined);
+            this.#releaseSlot();
+            throw err;
+        }
+    }
+    /**
+     * The websocket address this lease's page reported it could not reach, if it reported one.
+     *
+     * Undefined means the page said nothing — never that it dialled the right place.
+     */
+    dialFailureUrl(sessionId) {
+        return this.#active.get(this.#leaseIdOf(sessionId))?.dialFailureUrl;
+    }
+    /**
+     * The message of the most recent native dialog (`confirm`/`alert`/`prompt`) auto-dismissed on this
+     * lease, if any. Diagnostic only — the dialog itself is already gone by the time this is readable.
+     */
+    lastDialogMessage(sessionId) {
+        return this.#active.get(this.#leaseIdOf(sessionId))?.lastDialogMessage;
+    }
+    /** Close every context and the browser. Pending waiters are rejected (the pool is terminal now). */
+    async shutdown() {
+        this.#closed = true; // set first so any woken waiter rejects instead of relaunching a browser
+        const contexts = [...this.#active.values()].map((a) => a.context);
+        this.#active.clear();
+        this.#occupied = 0;
+        await Promise.all(contexts.map((c) => c.close().catch(() => undefined)));
+        // If a launch is in flight, await it so we can close the resulting browser — otherwise it
+        // resolves after shutdown returns and the Chromium process is orphaned (hundreds of MB leaked).
+        const inflight = this.#launching;
+        if (inflight !== undefined) {
+            const launched = await inflight.catch(() => undefined);
+            if (launched !== undefined && launched !== this.#browser) {
+                await launched.close().catch(() => undefined);
+            }
+        }
+        const browser = this.#browser;
+        this.#browser = undefined;
+        if (browser !== undefined)
+            await browser.close().catch(() => undefined);
+        for (const waiter of this.#waiters.splice(0))
+            waiter();
+    }
+    async #release(sessionId) {
+        const lease = this.#active.get(sessionId);
+        if (lease === undefined)
+            return; // already released or lost to a crash
+        this.#active.delete(sessionId);
+        // Drop any alias pointing here, or the map grows for the life of the daemon and a later lease
+        // reusing that registered name would resolve to a context that is already closed.
+        for (const [registered, leaseId] of this.#aliases) {
+            if (leaseId === sessionId)
+                this.#aliases.delete(registered);
+        }
+        await lease.context.close().catch(() => undefined);
+        this.#releaseSlot();
+    }
+    /** Synchronously claim a slot if under the cap. Returns true on success (caller proceeds). */
+    #tryClaim() {
+        if (this.#occupied < this.#max) {
+            this.#occupied += 1;
+            return true;
+        }
+        return false;
+    }
+    #waitForSlot(signal) {
+        if (this.#tryClaim())
+            return Promise.resolve();
+        return new Promise((resolve, reject) => {
+            const onFree = () => {
+                // The pool shut down while we waited → reject rather than relaunch a browser.
+                if (this.#closed) {
+                    if (signal !== undefined)
+                        signal.removeEventListener('abort', onAbort);
+                    reject(new Error('browser pool is shut down'));
+                    return;
+                }
+                // Claim the freed slot; if another woken waiter beat us to it, re-queue.
+                if (this.#tryClaim()) {
+                    if (signal !== undefined)
+                        signal.removeEventListener('abort', onAbort);
+                    resolve();
+                }
+                else {
+                    this.#waiters.push(onFree);
+                }
+            };
+            const onAbort = () => {
+                const i = this.#waiters.indexOf(onFree);
+                if (i >= 0)
+                    this.#waiters.splice(i, 1);
+                reject(new Error('acquire aborted'));
+            };
+            if (signal !== undefined) {
+                if (signal.aborted) {
+                    reject(new Error('acquire aborted'));
+                    return;
+                }
+                signal.addEventListener('abort', onAbort, { once: true });
+            }
+            this.#waiters.push(onFree);
+        });
+    }
+    /** Give back one occupied slot and hand it to the next waiter (which re-claims it synchronously). */
+    #releaseSlot() {
+        if (this.#occupied > 0)
+            this.#occupied -= 1;
+        const next = this.#waiters.shift();
+        if (next !== undefined)
+            next();
+    }
+    async #ensureBrowser() {
+        if (this.#closed)
+            throw new Error('browser pool is shut down');
+        if (this.#browser !== undefined && this.#browser.isConnected())
+            return this.#browser;
+        // De-dupe concurrent launches: the first acquire to find no browser starts one; the rest await it.
+        if (this.#launching === undefined) {
+            // The in-flight promise MUST be cleared on failure as well as success. It used to be cleared only
+            // in the success callback, so the first failed launch — Chromium not downloaded, a transient
+            // EAGAIN — left a permanently rejected promise here, and every subsequent acquire for the daemon's
+            // lifetime re-returned that same stale rejection. Running `npx playwright install` did not help;
+            // only restarting the daemon did. A retry must be allowed to actually retry.
+            this.#launching = this.#launch()
+                .then((b) => {
+                b.onDisconnected(() => this.#onCrash(b));
+                this.#browser = b;
+                return b;
+            })
+                .finally(() => {
+                this.#launching = undefined;
+            });
+        }
+        return this.#launching;
+    }
+    /** The browser process died: drop every lease (they're invalid) and clear so the next acquire relaunches. */
+    #onCrash(crashed) {
+        if (this.#browser !== crashed)
+            return; // a stale handler from a prior browser
+        this.#browser = undefined;
+        this.#active.clear();
+        this.#occupied = 0; // every slot is gone with the browser
+        for (const waiter of this.#waiters.splice(0))
+            waiter(); // let them re-claim + relaunch
+    }
+}

package/dist/portal/pool/lease-reaper.js

@@ -0,0 +1,40 @@
+/**
+ * LeaseReaper — periodic backstop that reclaims orphaned pool leases.
+ *
+ * A lease is "touched" on acquire and on every tool call that targets it. If an agent crashes or
+ * hangs mid-flow it stops touching its lease; after the pool's TTL the reaper closes that context and
+ * frees the slot for a queued acquire. This is what keeps "10 agents, headless" fault-tolerant: one
+ * dead agent can't starve the pool. Mirrors SessionReaper (unref'd interval, idempotent start).
+ */
+import { log } from '../../log.js';
+/** How often the reaper sweeps for expired leases. */
+const LEASE_REAP_INTERVAL_MS = 30_000;
+export class LeaseReaper {
+    #timer;
+    #pool;
+    #intervalMs;
+    constructor(pool, intervalMs = LEASE_REAP_INTERVAL_MS) {
+        this.#pool = pool;
+        this.#intervalMs = intervalMs;
+    }
+    start() {
+        if (this.#timer !== undefined)
+            return;
+        this.#timer = setInterval(() => {
+            this.#pool
+                .sweepExpired()
+                .then((reclaimed) => {
+                if (reclaimed.length > 0)
+                    log('lease_reaped_expired', { leases: reclaimed });
+            })
+                .catch(() => undefined);
+        }, this.#intervalMs);
+        this.#timer.unref();
+    }
+    stop() {
+        if (this.#timer === undefined)
+            return;
+        clearInterval(this.#timer);
+        this.#timer = undefined;
+    }
+}