@minionry/minion 0.7.36

package/dist/server/services/browser/driver-state.js

@@ -0,0 +1,546 @@
+// Copyright (c) 2025-present Minionry, Inc.
+// SPDX-License-Identifier: LicenseRef-Minionry-Software
+import { EventEmitter } from 'node:events';
+export const DRIVER_STATE_CONSTANTS = {
+    IDLE_GRACE_MS: 5000,
+    AUTOMATION_HOLD_MS: 2500,
+    HUMAN_HOLD_MS: 3000,
+    EDITABLE_GRACE_MS: 5000,
+    EMIT_COALESCE_MS: 250,
+    RECENT_KEYDOWN_MS: 1500,
+    RECENT_MOUSEMOVE_MS: 750,
+    RECENT_CLICK_MS: 3000,
+    RECENT_EDITABLE_INPUT_MS: 5000,
+    HELD_INPUT_MAX_MS: 30_000,
+};
+export const INPUT_MUTATING_BROWSER_TOOLS = new Set([
+    'browser_click',
+    'browser_type',
+    'browser_hover',
+    'browser_drag',
+    'browser_fill_form',
+    'browser_press_key',
+    'browser_select_option',
+    'browser_file_upload',
+    'browser_handle_dialog',
+    'browser_navigate',
+]);
+export const READ_ONLY_BROWSER_TOOLS = new Set([
+    'browser_snapshot',
+    'browser_take_screenshot',
+    'browser_console_messages',
+    'browser_network_request',
+    'browser_network_requests',
+    'browser_wait_for',
+    'browser_tabs',
+]);
+function newTabState(now) {
+    return {
+        state: 'idle',
+        mode: undefined,
+        parkedReason: undefined,
+        since: now,
+        lastInputAt: undefined,
+        lastInputActor: undefined,
+        lastUserInputAt: undefined,
+        lastFieldInputAt: undefined,
+        lastKeyDownAt: undefined,
+        lastMouseMoveAt: undefined,
+        lastClickAt: undefined,
+        pointerDownOpen: false,
+        imeCompositionOpen: false,
+        focusedElementIsEditable: false,
+        lastEmitAt: 0,
+        pendingEmitTimer: null,
+        decayTimer: null,
+        resumeWaiters: [],
+        ownerAgentId: undefined,
+        ownerAgentLabel: undefined,
+        planId: undefined,
+        leasedUntil: undefined,
+        ownerSessionId: undefined,
+    };
+}
+export class BrowserDriverState extends EventEmitter {
+    tabs = new Map();
+    now;
+    k;
+    constructor(opts = {}) {
+        super();
+        this.now = opts.now ?? (() => Date.now());
+        this.k = { ...DRIVER_STATE_CONSTANTS, ...(opts.constants ?? {}) };
+    }
+    ensureTab(tabId) {
+        let t = this.tabs.get(tabId);
+        if (!t) {
+            t = newTabState(this.now());
+            this.tabs.set(tabId, t);
+        }
+        return t;
+    }
+    closeTab(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return;
+        if (t.pendingEmitTimer)
+            clearTimeout(t.pendingEmitTimer);
+        if (t.decayTimer)
+            clearTimeout(t.decayTimer);
+        t.decayTimer = null;
+        for (const w of t.resumeWaiters)
+            w();
+        t.resumeWaiters.length = 0;
+        t.state = 'closed';
+        t.mode = undefined;
+        t.parkedReason = undefined;
+        t.since = this.now();
+        this.emitNow(tabId, t);
+        this.tabs.delete(tabId);
+    }
+    recordInput(tabId, actor, kind) {
+        const t = this.ensureTab(tabId);
+        const now = this.now();
+        t.lastInputAt = now;
+        t.lastInputActor = actor;
+        if (actor === 'user') {
+            t.lastUserInputAt = now;
+            if (kind === 'key')
+                t.lastKeyDownAt = now;
+            if (kind === 'mouse')
+                t.lastMouseMoveAt = now;
+            if (kind !== 'mouse' && kind !== 'wheel')
+                t.lastFieldInputAt = now;
+            this.transitionFromInput(tabId, t, 'user', now);
+        }
+        else {
+            this.transitionFromInput(tabId, t, 'agent', now);
+        }
+    }
+    recordUserClick(tabId) {
+        const t = this.ensureTab(tabId);
+        const now = this.now();
+        t.lastClickAt = now;
+        t.lastInputAt = now;
+        t.lastInputActor = 'user';
+        t.lastUserInputAt = now;
+        t.lastFieldInputAt = now;
+        this.transitionFromInput(tabId, t, 'user', now);
+    }
+    recordPointerDown(tabId, isDown) {
+        this.recordHold(tabId, 'pointerDownOpen', isDown);
+    }
+    recordImeComposition(tabId, open) {
+        this.recordHold(tabId, 'imeCompositionOpen', open);
+    }
+    recordHold(tabId, hold, open) {
+        const t = this.ensureTab(tabId);
+        t[hold] = open;
+        if (!open) {
+            this.scheduleEmit(tabId, t);
+            return;
+        }
+        const now = this.now();
+        t.lastUserInputAt = now;
+        this.transitionFromInput(tabId, t, 'user', now);
+    }
+    releaseHeldInput(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t || (!t.pointerDownOpen && !t.imeCompositionOpen))
+            return;
+        t.pointerDownOpen = false;
+        t.imeCompositionOpen = false;
+        this.scheduleEmit(tabId, t);
+    }
+    recordEditableFocus(tabId, isEditable) {
+        const t = this.ensureTab(tabId);
+        if (t.focusedElementIsEditable === isEditable)
+            return;
+        t.focusedElementIsEditable = isEditable;
+        this.scheduleEmit(tabId, t);
+    }
+    recordViewInput(tabId, envelope) {
+        const eventType = typeof envelope.eventType === 'string' ? envelope.eventType : undefined;
+        switch (envelope.kind) {
+            case 'mouse':
+                if (eventType === 'mousePressed') {
+                    this.recordUserClick(tabId);
+                    this.recordPointerDown(tabId, true);
+                }
+                else if (eventType === 'mouseReleased') {
+                    this.recordPointerDown(tabId, false);
+                }
+                else {
+                    this.recordInput(tabId, 'user', eventType === 'mouseWheel' ? 'wheel' : 'mouse');
+                }
+                return;
+            case 'key':
+            case 'text':
+                this.recordInput(tabId, 'user', 'key');
+                return;
+            case 'touch':
+                this.recordInput(tabId, 'user', 'touch');
+                return;
+            case 'ime':
+                this.recordImeComposition(tabId, eventType !== 'commit');
+                this.recordInput(tabId, 'user', 'ime');
+                return;
+        }
+    }
+    setOwner(tabId, owner) {
+        const t = this.ensureTab(tabId);
+        t.ownerAgentId = owner.ownerAgentId;
+        t.ownerAgentLabel = owner.ownerAgentLabel;
+        t.planId = owner.planId;
+        t.leasedUntil = owner.leasedUntil;
+        t.ownerSessionId = owner.ownerSessionId;
+        this.scheduleEmit(tabId, t);
+    }
+    getOwner(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t || t.ownerAgentId === undefined || t.leasedUntil === undefined)
+            return null;
+        return {
+            ownerAgentId: t.ownerAgentId,
+            ...(t.ownerAgentLabel !== undefined ? { ownerAgentLabel: t.ownerAgentLabel } : {}),
+            ...(t.planId !== undefined ? { planId: t.planId } : {}),
+            leasedUntil: t.leasedUntil,
+            ...(t.ownerSessionId !== undefined ? { ownerSessionId: t.ownerSessionId } : {}),
+        };
+    }
+    clearOwner(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return;
+        if (t.ownerAgentId === undefined &&
+            t.ownerAgentLabel === undefined &&
+            t.planId === undefined &&
+            t.leasedUntil === undefined &&
+            t.ownerSessionId === undefined) {
+            return;
+        }
+        t.ownerAgentId = undefined;
+        t.ownerAgentLabel = undefined;
+        t.planId = undefined;
+        t.leasedUntil = undefined;
+        t.ownerSessionId = undefined;
+        this.scheduleEmit(tabId, t);
+    }
+    park(tabId, reason) {
+        const t = this.ensureTab(tabId);
+        t.state = 'automation';
+        t.mode = 'parked';
+        t.parkedReason = reason;
+        t.since = this.now();
+        this.emitNow(tabId, t);
+    }
+    resume(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return false;
+        if (t.state !== 'automation' || t.mode !== 'parked')
+            return false;
+        t.state = 'automation';
+        t.mode = 'driving';
+        t.parkedReason = undefined;
+        t.since = this.now();
+        const waiters = t.resumeWaiters.slice();
+        t.resumeWaiters.length = 0;
+        this.emitNow(tabId, t);
+        for (const w of waiters)
+            w();
+        return true;
+    }
+    userIsActive(tabId, now = this.now()) {
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return false;
+        if (t.lastKeyDownAt && t.lastKeyDownAt > now - this.k.RECENT_KEYDOWN_MS)
+            return true;
+        if ((t.pointerDownOpen || t.imeCompositionOpen) && this.holdIsLive(t, now))
+            return true;
+        if (t.focusedElementIsEditable &&
+            t.lastFieldInputAt !== undefined &&
+            t.lastFieldInputAt > now - this.k.RECENT_EDITABLE_INPUT_MS) {
+            return true;
+        }
+        if (t.lastMouseMoveAt &&
+            t.lastClickAt &&
+            t.lastMouseMoveAt > now - this.k.RECENT_MOUSEMOVE_MS &&
+            t.lastClickAt > now - this.k.RECENT_CLICK_MS) {
+            return true;
+        }
+        return false;
+    }
+    holdIsLive(t, now) {
+        return t.lastUserInputAt !== undefined && t.lastUserInputAt > now - this.k.HELD_INPUT_MAX_MS;
+    }
+    gateSync(tabId, toolName) {
+        if (READ_ONLY_BROWSER_TOOLS.has(toolName))
+            return { allow: true };
+        if (!INPUT_MUTATING_BROWSER_TOOLS.has(toolName))
+            return { allow: true };
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return { allow: true };
+        const now = this.now();
+        if (t.state === 'automation' && t.mode === 'parked') {
+            return {
+                allow: false,
+                code: 'arbitration_user_active',
+                state: 'automation',
+                since: t.since,
+                hint: 'This tab is paused until the user resumes it from the browser view. Do not retry until then.',
+            };
+        }
+        if (this.userIsActive(tabId, now)) {
+            return {
+                allow: false,
+                code: 'arbitration_user_active',
+                state: 'human',
+                since: t.since,
+                hint: 'The user is using this tab right now. Wait a few seconds and take a fresh snapshot before retrying; do not blindly retry.',
+            };
+        }
+        return { allow: true };
+    }
+    gate(tabId, toolName) {
+        const decision = this.gateSync(tabId, toolName);
+        if (decision.allow) {
+            this.markAgentDriving(tabId);
+            return Promise.resolve(decision);
+        }
+        if (decision.state !== 'automation')
+            return Promise.resolve(decision);
+        const t = this.ensureTab(tabId);
+        return new Promise((resolve) => {
+            t.resumeWaiters.push(() => resolve({ allow: true }));
+        });
+    }
+    markAgentDriving(tabId) {
+        const t = this.ensureTab(tabId);
+        if (t.state === 'automation' && t.mode === 'driving') {
+            this.scheduleEmit(tabId, t);
+            return;
+        }
+        if (t.state === 'human' && this.userIsActive(tabId, this.now()))
+            return;
+        t.state = 'automation';
+        t.mode = 'driving';
+        t.parkedReason = undefined;
+        t.since = this.now();
+        this.scheduleEmit(tabId, t);
+    }
+    anyTabBlocksAgent() {
+        const now = this.now();
+        for (const [tabId, t] of this.tabs) {
+            if (t.state === 'closed')
+                continue;
+            if (t.state === 'automation' && t.mode === 'parked') {
+                return {
+                    allow: false,
+                    code: 'arbitration_user_active',
+                    state: 'automation',
+                    since: t.since,
+                    hint: `Tab ${tabId} is paused until the user resumes it from the browser view.`,
+                };
+            }
+            if (this.userIsActive(tabId, now)) {
+                return {
+                    allow: false,
+                    code: 'arbitration_user_active',
+                    state: 'human',
+                    since: t.since,
+                    hint: `The user is using tab ${tabId} right now. Wait a few seconds, then re-check the page before acting.`,
+                };
+            }
+        }
+        return { allow: true };
+    }
+    getSnapshot(tabId) {
+        const t = this.tabs.get(tabId);
+        if (!t)
+            return null;
+        return this.buildPayload(tabId, t);
+    }
+    getState(tabId) {
+        return this.tabs.get(tabId)?.state ?? 'idle';
+    }
+    getMode(tabId) {
+        return this.tabs.get(tabId)?.mode;
+    }
+    transitionFromInput(tabId, t, actor, now) {
+        if (actor === 'user') {
+            if (t.state === 'automation' && t.mode === 'parked') {
+                this.scheduleEmit(tabId, t);
+                return;
+            }
+            if (t.state !== 'human') {
+                t.state = 'human';
+                t.mode = undefined;
+                t.parkedReason = undefined;
+                t.since = now;
+            }
+            this.scheduleEmit(tabId, t);
+            return;
+        }
+        if (t.state === 'automation' && t.mode === 'parked') {
+            this.scheduleEmit(tabId, t);
+            return;
+        }
+        if (this.userIsActive(tabId, now)) {
+            this.scheduleEmit(tabId, t);
+            return;
+        }
+        if (t.state !== 'automation' || t.mode !== 'driving') {
+            t.state = 'automation';
+            t.mode = 'driving';
+            t.parkedReason = undefined;
+            t.since = now;
+        }
+        this.scheduleEmit(tabId, t);
+    }
+    buildPayload(tabId, t) {
+        const out = {
+            tabId,
+            state: t.state,
+            since: t.since,
+        };
+        if (t.state === 'automation') {
+            out.mode = t.mode;
+            if (t.mode === 'parked') {
+                out.pendingResume = true;
+                if (t.parkedReason)
+                    out.parkedReason = t.parkedReason;
+            }
+        }
+        if (t.lastInputAt !== undefined)
+            out.lastInputAt = t.lastInputAt;
+        if (t.lastInputActor !== undefined)
+            out.lastInputActor = t.lastInputActor;
+        out.presence = {
+            pointerDownOpen: t.pointerDownOpen,
+            imeCompositionOpen: t.imeCompositionOpen,
+            focusedElementIsEditable: t.focusedElementIsEditable,
+        };
+        if (t.lastKeyDownAt !== undefined)
+            out.presence.lastKeyDownAt = t.lastKeyDownAt;
+        if (t.lastMouseMoveAt !== undefined)
+            out.presence.lastMouseMoveAt = t.lastMouseMoveAt;
+        if (t.lastClickAt !== undefined)
+            out.presence.lastClickAt = t.lastClickAt;
+        if (t.ownerAgentId !== undefined)
+            out.ownerAgentId = t.ownerAgentId;
+        if (t.ownerAgentLabel !== undefined)
+            out.ownerAgentLabel = t.ownerAgentLabel;
+        if (t.planId !== undefined)
+            out.planId = t.planId;
+        if (t.leasedUntil !== undefined)
+            out.leasedUntil = t.leasedUntil;
+        return out;
+    }
+    decayDueAt(t) {
+        let hold;
+        if (t.state === 'human') {
+            if (t.pointerDownOpen || t.imeCompositionOpen) {
+                return (t.lastUserInputAt ?? t.since) + this.k.HELD_INPUT_MAX_MS;
+            }
+            hold = t.focusedElementIsEditable ? this.k.EDITABLE_GRACE_MS : this.k.HUMAN_HOLD_MS;
+        }
+        else if (t.state === 'automation' && t.mode === 'driving') {
+            hold = this.k.AUTOMATION_HOLD_MS;
+        }
+        else {
+            return null;
+        }
+        return Math.max(t.since + this.k.IDLE_GRACE_MS, (t.lastInputAt ?? t.since) + hold);
+    }
+    armDecay(tabId, t) {
+        if (t.decayTimer) {
+            clearTimeout(t.decayTimer);
+            t.decayTimer = null;
+        }
+        const due = this.decayDueAt(t);
+        if (due === null)
+            return;
+        t.decayTimer = setTimeout(() => {
+            t.decayTimer = null;
+            if (this.tabs.get(tabId) !== t)
+                return;
+            const lapseAt = this.decayDueAt(t);
+            if (lapseAt === null)
+                return;
+            const now = this.now();
+            if (now < lapseAt) {
+                this.armDecay(tabId, t);
+                return;
+            }
+            if (t.pointerDownOpen || t.imeCompositionOpen) {
+                t.pointerDownOpen = false;
+                t.imeCompositionOpen = false;
+                const next = this.decayDueAt(t);
+                if (next !== null && now < next) {
+                    this.emitNow(tabId, t);
+                    return;
+                }
+            }
+            t.state = 'idle';
+            t.mode = undefined;
+            t.parkedReason = undefined;
+            t.since = now;
+            this.emitNow(tabId, t);
+        }, Math.max(0, due - this.now()));
+        if (typeof t.decayTimer === 'object' && 'unref' in t.decayTimer) {
+            t.decayTimer.unref?.();
+        }
+    }
+    scheduleEmit(tabId, t) {
+        this.armDecay(tabId, t);
+        const now = this.now();
+        const elapsed = now - t.lastEmitAt;
+        if (elapsed >= this.k.EMIT_COALESCE_MS) {
+            this.emitNow(tabId, t);
+            return;
+        }
+        if (t.pendingEmitTimer)
+            return;
+        const wait = this.k.EMIT_COALESCE_MS - elapsed;
+        t.pendingEmitTimer = setTimeout(() => {
+            t.pendingEmitTimer = null;
+            const live = this.tabs.get(tabId);
+            if (live)
+                this.emitNow(tabId, live);
+        }, wait);
+        if (typeof t.pendingEmitTimer === 'object' && 'unref' in t.pendingEmitTimer) {
+            t.pendingEmitTimer.unref?.();
+        }
+    }
+    emitNow(tabId, t) {
+        this.armDecay(tabId, t);
+        t.lastEmitAt = this.now();
+        if (t.pendingEmitTimer) {
+            clearTimeout(t.pendingEmitTimer);
+            t.pendingEmitTimer = null;
+        }
+        this.emit('change', this.buildPayload(tabId, t));
+    }
+    dispose() {
+        for (const t of this.tabs.values()) {
+            if (t.pendingEmitTimer)
+                clearTimeout(t.pendingEmitTimer);
+            if (t.decayTimer)
+                clearTimeout(t.decayTimer);
+            t.pendingEmitTimer = null;
+            t.decayTimer = null;
+        }
+    }
+}
+let singleton = null;
+export function getBrowserDriverState() {
+    if (!singleton)
+        singleton = new BrowserDriverState();
+    return singleton;
+}
+export function _resetBrowserDriverStateForTests() {
+    singleton?.dispose();
+    singleton = null;
+}