@mercuryo-ai/agentbrowse 0.2.60 → 0.2.63

package/dist/commands/observe.js

@@ -7,7 +7,7 @@ import { ensureRuntimeState, replaceTargetsForPage } from '../runtime-state.js';
 import { incrementMetric } from '../runtime-metrics.js';
 import { bumpPageScopeEpoch, setCurrentPage } from '../runtime-page-state.js';
 import { getProtectedExposure } from '../runtime-protected-state.js';
-import { connectPlaywright, disconnectPlaywright, resolveCurrentPageContext, syncSessionPage, } from '../playwright-runtime.js';
+import { resolveCurrentPageContext, syncSessionPage } from '../playwright-runtime.js';
 import { tracedStepOperation, withApiTraceContext } from '../command-api-tracing.js';
 import { captureDiagnosticSnapshotBestEffort, finishDiagnosticStepBestEffort, recordCommandLifecycleEventBestEffort, startDiagnosticStep, } from '../diagnostics.js';
 import { outputContractFailure, outputJSON, } from '../output.js';
@@ -21,9 +21,11 @@ import { enrichDomTargetsWithAccessibility } from './observe-accessibility.js';
 import { collectPageSignals } from './observe-signals.js';
 import { attachObservedTargetOwners, linkObservedSurfaceGraph, reconcileObservedTargetsForPage, persistObservedSurfacesForPage, toDomDescriptor, } from './observe-persistence.js';
 import { clearProtectedFillableFormsForPage, markProtectedFillableFormsUnknownForPage, persistProtectedFillableFormsForPage, } from './observe-protected.js';
+import { withStickyOwnerBrowser } from '../sticky-owner.js';
+import { describeBrowserConnectionFailure } from './browser-connection-failure.js';
 import { classifyObservePageState, shouldSuppressFillableFormsForObserve, } from './observe-page-state.js';
-import { annotateDomTargets, compressSemanticallyDuplicateTargets, orderBySurfaceCompetition, prioritizeGoalActionTargets, } from './observe-semantics.js';
-import { buildGroupedObserveScopes, buildGoalProjectionScopeRefs, buildGoalObserveInventoryCandidates, compactFillableForms, compactSignals, expandWorkflowGraphTargets, projectPersistedTargetsForGoal, selectTargetsForGoalMatches, } from './observe-projection.js';
+import { normalizePostEnrichmentDomTargets, prioritizeGoalActionTargets, } from './observe-semantics.js';
+import { buildGroupedObserveScopes, buildGoalProjectionScopeRefs, buildGoalObserveInventoryCandidates, compactFillableForms, compactSignals, expandWorkflowGraphTargets, materializeObserveOutputTargets, projectPersistedTargetsForGoal, selectTargetsForGoalMatches, } from './observe-projection.js';
 import { collectSurfaceDescriptors, selectScopesForOutput } from './observe-surfaces.js';
 import { toStagehandDescriptor } from './observe-stagehand.js';
 import { rerankDomTargetsForGoal } from './semantic-observe.js';
@@ -174,7 +176,6 @@ export async function observeBrowser(session, instruction) {
     let pageRef = runtime.currentPageRef;
     let domPassError = null;
     let stagehandFallbackReason = null;
-    let browser = null;
     let observedScopes = [];
     const observeStep = startDiagnosticStep({
         runId: session.activeRunId,
@@ -219,50 +220,254 @@ export async function observeBrowser(session, instruction) {
                 stepId: observeStep?.stepId,
             });
         }
-        if (!instruction) {
-            try {
-                browser = await connectPlaywright(session.cdpUrl);
-                const resolvedPage = await resolveCurrentPageContext(browser, session);
-                pageRef = resolvedPage.pageRef;
-                const page = resolvedPage.page;
-                const previousPageUrl = session.runtime?.pages?.[pageRef]?.url;
-                const { url, title } = await syncSessionPage(session, pageRef, page);
-                const protectedExposure = getProtectedExposure(session, pageRef);
-                bumpPageScopeEpoch(session, pageRef);
-                setCurrentPage(session, pageRef);
-                const collectedTargets = await collectDomTargets(page);
-                let observeAccessibilityStats;
-                const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
-                    onStats: (stats) => {
-                        observeAccessibilityStats = stats;
-                    },
-                }))));
-                if (observeAccessibilityStats) {
-                    incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
-                    incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
-                    incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
+        try {
+            return await withStickyOwnerBrowser(session, async (browser) => {
+                if (!instruction) {
+                    try {
+                        const resolvedPage = await resolveCurrentPageContext(browser, session);
+                        pageRef = resolvedPage.pageRef;
+                        const page = resolvedPage.page;
+                        const previousPageUrl = session.runtime?.pages?.[pageRef]?.url;
+                        const { url, title } = await syncSessionPage(session, pageRef, page);
+                        const protectedExposure = getProtectedExposure(session, pageRef);
+                        bumpPageScopeEpoch(session, pageRef);
+                        setCurrentPage(session, pageRef);
+                        const collectedTargets = await collectDomTargets(page);
+                        let observeAccessibilityStats;
+                        const domTargets = normalizePostEnrichmentDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
+                            onStats: (stats) => {
+                                observeAccessibilityStats = stats;
+                            },
+                        }));
+                        if (observeAccessibilityStats) {
+                            incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
+                            incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
+                            incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
+                        }
+                        const pageSignals = await collectPageSignals(page).catch(() => []);
+                        const pageState = classifyObservePageState(pageSignals);
+                        const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets);
+                        observedScopes = persisted.observedScopes;
+                        const surfaceRefMap = persisted.surfaceRefMap;
+                        if (domTargets.length > 0) {
+                            const targets = replaceTargetsForPage(session, pageRef, domTargets.map((target) => toDomDescriptor(pageRef, target, surfaceRefMap)));
+                            reconcileObservedTargetsForPage(session, pageRef, targets);
+                            attachObservedTargetOwners(domTargets, targets);
+                            observedScopes = linkObservedSurfaceGraph(session, pageRef, domTargets, targets, observedScopes, surfaceRefMap);
+                            const fillableForms = shouldSuppressFillableFormsForObserve(pageState)
+                                ? clearProtectedFillableFormsForPage(session, pageRef)
+                                : await persistProtectedFillableFormsForPage(session, pageRef, url, targets, new Date().toISOString(), { previousPageUrl });
+                            const outputTargets = materializeObserveOutputTargets(domTargets, targets);
+                            return buildObserveSuccessResult(session, observeStep, {
+                                success: true,
+                                observationMode: 'deterministic_dom',
+                                pageRef,
+                                resolvedBy: 'dom',
+                                ...domRuntimeResolution(),
+                                scopes: buildGroupedObserveScopes({
+                                    pageRef,
+                                    title,
+                                    scopes: selectScopesForOutput(observedScopes, targets),
+                                    targets: outputTargets,
+                                }),
+                                signals: compactSignals(pageSignals),
+                                fillableForms: compactFillableForms(fillableForms),
+                                metrics: session.runtime?.metrics,
+                                message: targets.length === 0 ? 'This observe pass returned zero targets.' : undefined,
+                                ...buildObservePageMetadata({
+                                    url,
+                                    title,
+                                    protectedExposure,
+                                }),
+                            });
+                        }
+                        if (!allowAssistive) {
+                            return buildObserveSuccessResult(session, observeStep, {
+                                success: true,
+                                observationMode: 'deterministic_dom',
+                                pageRef,
+                                resolvedBy: 'dom',
+                                ...domRuntimeResolution(),
+                                scopes: [],
+                                signals: compactSignals(pageSignals),
+                                fillableForms: [],
+                                metrics: session.runtime?.metrics,
+                                message: 'This observe pass returned zero targets.',
+                                ...buildObservePageMetadata({
+                                    url,
+                                    title,
+                                    protectedExposure,
+                                }),
+                            });
+                        }
+                        stagehandFallbackReason = 'deterministic-observe-empty';
+                    }
+                    catch (err) {
+                        domPassError = err instanceof Error ? err.message : String(err);
+                        if (!allowAssistive) {
+                            return buildObserveContractFailureResult(session, {
+                                step: observeStep,
+                                error: 'observe_failed',
+                                outcomeType: 'blocked',
+                                message: 'Observe failed.',
+                                reason: domPassError,
+                                pageRef,
+                                runId: session.activeRunId,
+                                stepId: observeStep?.stepId,
+                            });
+                        }
+                        stagehandFallbackReason = 'deterministic-observe-failed';
+                    }
                 }
-                const pageSignals = await collectPageSignals(page).catch(() => []);
-                const pageState = classifyObservePageState(pageSignals);
-                const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets);
-                observedScopes = persisted.observedScopes;
-                const surfaceRefMap = persisted.surfaceRefMap;
-                if (domTargets.length > 0) {
-                    const targets = replaceTargetsForPage(session, pageRef, domTargets.map((target) => toDomDescriptor(pageRef, target, surfaceRefMap)));
-                    reconcileObservedTargetsForPage(session, pageRef, targets);
-                    attachObservedTargetOwners(domTargets, targets);
-                    observedScopes = linkObservedSurfaceGraph(session, pageRef, domTargets, targets, observedScopes, surfaceRefMap);
-                    const fillableForms = shouldSuppressFillableFormsForObserve(pageState)
-                        ? clearProtectedFillableFormsForPage(session, pageRef)
-                        : await persistProtectedFillableFormsForPage(session, pageRef, url, targets, new Date().toISOString(), { previousPageUrl });
-                    await disconnectPlaywright(browser);
-                    browser = null;
+                if (instruction) {
+                    try {
+                        const resolvedPage = await resolveCurrentPageContext(browser, session);
+                        pageRef = resolvedPage.pageRef;
+                        const page = resolvedPage.page;
+                        const previousPageUrl = session.runtime?.pages?.[pageRef]?.url;
+                        const { url, title } = await syncSessionPage(session, pageRef, page);
+                        const protectedExposure = getProtectedExposure(session, pageRef);
+                        if (protectedExposure) {
+                            return buildObserveContractFailureResult(session, {
+                                step: observeStep,
+                                ...buildProtectedObserveBlockedResult(protectedExposure, 'goal-rerank'),
+                                pageRef,
+                                runId: session.activeRunId,
+                                stepId: observeStep?.stepId,
+                            });
+                        }
+                        bumpPageScopeEpoch(session, pageRef);
+                        setCurrentPage(session, pageRef);
+                        const collectedTargets = await collectDomTargets(page, {
+                            includeActivationAffordances: true,
+                        });
+                        let observeAccessibilityStats;
+                        const domTargets = normalizePostEnrichmentDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
+                            onStats: (stats) => {
+                                observeAccessibilityStats = stats;
+                            },
+                        }));
+                        if (observeAccessibilityStats) {
+                            incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
+                            incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
+                            incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
+                        }
+                        const pageSignals = await collectPageSignals(page).catch(() => []);
+                        const pageState = classifyObservePageState(pageSignals);
+                        const surfaceInputs = collectSurfaceDescriptors(pageRef, domTargets);
+                        if (domTargets.length > 0) {
+                            const rerankedCandidates = await tracedStepOperation(() => rerankDomTargetsForGoal(instruction, buildGoalObserveInventoryCandidates(domTargets, surfaceInputs), { session }), {
+                                spanName: 'agentbrowse.observe.rerank_goal_candidates',
+                                attributes: {
+                                    ...observePhaseAttributes,
+                                    'agentbrowse.observe.target_count': domTargets.length,
+                                },
+                            });
+                            const { targets: goalMatchedTargets, selectedSurfaceIds } = selectTargetsForGoalMatches(domTargets, rerankedCandidates);
+                            const selectedTargets = prioritizeGoalActionTargets(instruction, expandWorkflowGraphTargets(domTargets, goalMatchedTargets, {
+                                selectedSurfaceIds,
+                            }));
+                            const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets, {
+                                allSurfaceInputs: surfaceInputs,
+                                explicitSurfaceIds: selectedSurfaceIds,
+                            });
+                            observedScopes = persisted.observedScopes;
+                            const surfaceRefMap = persisted.surfaceRefMap;
+                            const targets = replaceTargetsForPage(session, pageRef, domTargets.map((target) => toDomDescriptor(pageRef, target, surfaceRefMap)));
+                            reconcileObservedTargetsForPage(session, pageRef, targets);
+                            attachObservedTargetOwners(domTargets, targets);
+                            observedScopes = linkObservedSurfaceGraph(session, pageRef, domTargets, targets, observedScopes, surfaceRefMap);
+                            const fillableForms = shouldSuppressFillableFormsForObserve(pageState)
+                                ? clearProtectedFillableFormsForPage(session, pageRef)
+                                : await persistProtectedFillableFormsForPage(session, pageRef, url, targets, new Date().toISOString(), { previousPageUrl });
+                            if (selectedTargets.length > 0 || selectedSurfaceIds.size > 0) {
+                                const projectedTargets = projectPersistedTargetsForGoal(domTargets, targets, selectedTargets);
+                                const outputProjectedTargets = materializeObserveOutputTargets(domTargets, targets, projectedTargets);
+                                const explicitScopeRefs = buildGoalProjectionScopeRefs(outputProjectedTargets, selectedSurfaceIds, surfaceRefMap);
+                                return buildObserveSuccessResult(session, observeStep, {
+                                    success: true,
+                                    observationMode: canUseAssistiveLlm
+                                        ? 'goal_assistive_rerank'
+                                        : 'goal_heuristic_shortlist',
+                                    pageRef,
+                                    resolvedBy: 'dom-rerank',
+                                    ...domRuntimeResolution(),
+                                    scopes: buildGroupedObserveScopes({
+                                        pageRef,
+                                        title,
+                                        scopes: selectScopesForOutput(observedScopes, outputProjectedTargets, explicitScopeRefs),
+                                        targets: outputProjectedTargets,
+                                    }),
+                                    signals: compactSignals(pageSignals),
+                                    fillableForms: compactFillableForms(fillableForms),
+                                    metrics: session.runtime?.metrics,
+                                    url,
+                                    title,
+                                });
+                            }
+                            return buildObserveSuccessResult(session, observeStep, {
+                                success: true,
+                                observationMode: canUseAssistiveLlm
+                                    ? 'goal_assistive_rerank'
+                                    : 'goal_heuristic_shortlist',
+                                pageRef,
+                                resolvedBy: 'dom-rerank',
+                                ...domRuntimeResolution(),
+                                scopes: [],
+                                signals: compactSignals(pageSignals),
+                                fillableForms: compactFillableForms(fillableForms),
+                                metrics: session.runtime?.metrics,
+                                message: 'This goal-based observe pass returned zero matching targets.',
+                                url,
+                                title,
+                            });
+                        }
+                        stagehandFallbackReason = 'deterministic-observe-empty';
+                    }
+                    catch (err) {
+                        domPassError = err instanceof Error ? err.message : String(err);
+                        stagehandFallbackReason = 'deterministic-observe-failed';
+                    }
+                }
+                try {
+                    const resolvedPage = await resolveCurrentPageContext(browser, session);
+                    pageRef = resolvedPage.pageRef;
+                    const page = resolvedPage.page;
+                    const { url, title } = await syncSessionPage(session, pageRef, page);
+                    const protectedExposure = getProtectedExposure(session, pageRef);
+                    if (protectedExposure) {
+                        return buildObserveContractFailureResult(session, {
+                            step: observeStep,
+                            ...buildProtectedObserveBlockedResult(protectedExposure, 'stagehand-fallback'),
+                            fallbackReason: stagehandFallbackReason ?? undefined,
+                            deterministicObserveError: domPassError ?? undefined,
+                            pageRef,
+                            runId: session.activeRunId,
+                            stepId: observeStep?.stepId,
+                        });
+                    }
+                    bumpPageScopeEpoch(session, pageRef);
+                    setCurrentPage(session, pageRef);
+                    const pageSignals = await collectPageSignals(page).catch(() => []);
+                    const actions = await withStagehand(session, async (stagehand) => {
+                        incrementMetric(session, 'stagehandCalls');
+                        return instruction
+                            ? (await stagehand.observe(instruction, {
+                                page,
+                            }))
+                            : (await stagehand.observe({
+                                page,
+                            }));
+                    });
+                    const targets = replaceTargetsForPage(session, pageRef, await Promise.all(actions.map((action) => toStagehandDescriptor(pageRef, action, page, normalizePageSignature(url)))));
+                    const fillableForms = markProtectedFillableFormsUnknownForPage(session, pageRef);
                     return buildObserveSuccessResult(session, observeStep, {
                         success: true,
-                        observationMode: 'deterministic_dom',
+                        observationMode: 'goal_assistive_stagehand',
                         pageRef,
-                        resolvedBy: 'dom',
-                        ...domRuntimeResolution(),
+                        resolvedBy: 'stagehand-observe',
+                        ...stagehandRuntimeResolution(stagehandFallbackReason ?? 'deterministic-observe-failed'),
+                        deterministicObserveError: domPassError ?? undefined,
                         scopes: buildGroupedObserveScopes({
                             pageRef,
                             title,
@@ -273,259 +478,46 @@ export async function observeBrowser(session, instruction) {
                         fillableForms: compactFillableForms(fillableForms),
                         metrics: session.runtime?.metrics,
                         message: targets.length === 0 ? 'This observe pass returned zero targets.' : undefined,
-                        ...buildObservePageMetadata({
-                            url,
-                            title,
-                            protectedExposure,
-                        }),
-                    });
-                }
-                if (!allowAssistive) {
-                    await disconnectPlaywright(browser);
-                    browser = null;
-                    return buildObserveSuccessResult(session, observeStep, {
-                        success: true,
-                        observationMode: 'deterministic_dom',
-                        pageRef,
-                        resolvedBy: 'dom',
-                        ...domRuntimeResolution(),
-                        scopes: [],
-                        signals: compactSignals(pageSignals),
-                        fillableForms: [],
-                        metrics: session.runtime?.metrics,
-                        message: 'This observe pass returned zero targets.',
-                        ...buildObservePageMetadata({
-                            url,
-                            title,
-                            protectedExposure,
-                        }),
+                        url,
+                        title,
                     });
                 }
-                stagehandFallbackReason = 'deterministic-observe-empty';
-            }
-            catch (err) {
-                domPassError = err instanceof Error ? err.message : String(err);
-                if (!allowAssistive) {
+                catch (err) {
+                    const stagehandError = err instanceof Error ? err.message : String(err);
+                    const details = domPassError
+                        ? `${stagehandError} (deterministic observe failed earlier: ${domPassError})`
+                        : stagehandError;
                     return buildObserveContractFailureResult(session, {
                         step: observeStep,
                         error: 'observe_failed',
                         outcomeType: 'blocked',
                         message: 'Observe failed.',
-                        reason: domPassError,
+                        reason: details,
                         pageRef,
                         runId: session.activeRunId,
                         stepId: observeStep?.stepId,
                     });
                 }
-                stagehandFallbackReason = 'deterministic-observe-failed';
-            }
-        }
-        if (instruction) {
-            try {
-                if (!browser) {
-                    browser = await connectPlaywright(session.cdpUrl);
-                }
-                const resolvedPage = await resolveCurrentPageContext(browser, session);
-                pageRef = resolvedPage.pageRef;
-                const page = resolvedPage.page;
-                const previousPageUrl = session.runtime?.pages?.[pageRef]?.url;
-                const { url, title } = await syncSessionPage(session, pageRef, page);
-                const protectedExposure = getProtectedExposure(session, pageRef);
-                if (protectedExposure) {
-                    return buildObserveContractFailureResult(session, {
-                        step: observeStep,
-                        ...buildProtectedObserveBlockedResult(protectedExposure, 'goal-rerank'),
-                        pageRef,
-                        runId: session.activeRunId,
-                        stepId: observeStep?.stepId,
-                    });
-                }
-                bumpPageScopeEpoch(session, pageRef);
-                setCurrentPage(session, pageRef);
-                const collectedTargets = await collectDomTargets(page, {
-                    includeActivationAffordances: true,
-                });
-                let observeAccessibilityStats;
-                const domTargets = compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(await enrichDomTargetsWithAccessibility(page, collectedTargets, {
-                    onStats: (stats) => {
-                        observeAccessibilityStats = stats;
-                    },
-                }))));
-                if (observeAccessibilityStats) {
-                    incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
-                    incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
-                    incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
-                }
-                const pageSignals = await collectPageSignals(page).catch(() => []);
-                const pageState = classifyObservePageState(pageSignals);
-                const surfaceInputs = collectSurfaceDescriptors(pageRef, domTargets);
-                if (domTargets.length > 0) {
-                    const rerankedCandidates = await tracedStepOperation(() => rerankDomTargetsForGoal(instruction, buildGoalObserveInventoryCandidates(domTargets, surfaceInputs), { session }), {
-                        spanName: 'agentbrowse.observe.rerank_goal_candidates',
-                        attributes: {
-                            ...observePhaseAttributes,
-                            'agentbrowse.observe.target_count': domTargets.length,
-                        },
-                    });
-                    const { targets: goalMatchedTargets, selectedSurfaceIds } = selectTargetsForGoalMatches(domTargets, rerankedCandidates);
-                    const selectedTargets = prioritizeGoalActionTargets(instruction, expandWorkflowGraphTargets(domTargets, goalMatchedTargets, {
-                        selectedSurfaceIds,
-                    }));
-                    const persisted = persistObservedSurfacesForPage(session, pageRef, domTargets, {
-                        allSurfaceInputs: surfaceInputs,
-                        explicitSurfaceIds: selectedSurfaceIds,
-                    });
-                    observedScopes = persisted.observedScopes;
-                    const surfaceRefMap = persisted.surfaceRefMap;
-                    const targets = replaceTargetsForPage(session, pageRef, domTargets.map((target) => toDomDescriptor(pageRef, target, surfaceRefMap)));
-                    reconcileObservedTargetsForPage(session, pageRef, targets);
-                    attachObservedTargetOwners(domTargets, targets);
-                    observedScopes = linkObservedSurfaceGraph(session, pageRef, domTargets, targets, observedScopes, surfaceRefMap);
-                    const fillableForms = shouldSuppressFillableFormsForObserve(pageState)
-                        ? clearProtectedFillableFormsForPage(session, pageRef)
-                        : await persistProtectedFillableFormsForPage(session, pageRef, url, targets, new Date().toISOString(), { previousPageUrl });
-                    if (selectedTargets.length > 0 || selectedSurfaceIds.size > 0) {
-                        const projectedTargets = projectPersistedTargetsForGoal(domTargets, targets, selectedTargets);
-                        const explicitScopeRefs = buildGoalProjectionScopeRefs(projectedTargets, selectedSurfaceIds, surfaceRefMap);
-                        await disconnectPlaywright(browser);
-                        browser = null;
-                        return buildObserveSuccessResult(session, observeStep, {
-                            success: true,
-                            observationMode: canUseAssistiveLlm
-                                ? 'goal_assistive_rerank'
-                                : 'goal_heuristic_shortlist',
-                            pageRef,
-                            resolvedBy: 'dom-rerank',
-                            ...domRuntimeResolution(),
-                            scopes: buildGroupedObserveScopes({
-                                pageRef,
-                                title,
-                                scopes: selectScopesForOutput(observedScopes, projectedTargets, explicitScopeRefs),
-                                targets: projectedTargets,
-                            }),
-                            signals: compactSignals(pageSignals),
-                            fillableForms: compactFillableForms(fillableForms),
-                            metrics: session.runtime?.metrics,
-                            url,
-                            title,
-                        });
-                    }
-                    await disconnectPlaywright(browser);
-                    browser = null;
-                    return buildObserveSuccessResult(session, observeStep, {
-                        success: true,
-                        observationMode: canUseAssistiveLlm
-                            ? 'goal_assistive_rerank'
-                            : 'goal_heuristic_shortlist',
-                        pageRef,
-                        resolvedBy: 'dom-rerank',
-                        ...domRuntimeResolution(),
-                        scopes: [],
-                        signals: compactSignals(pageSignals),
-                        fillableForms: compactFillableForms(fillableForms),
-                        metrics: session.runtime?.metrics,
-                        message: 'This goal-based observe pass returned zero matching targets.',
-                        url,
-                        title,
-                    });
-                }
-                stagehandFallbackReason = 'deterministic-observe-empty';
-            }
-            catch (err) {
-                domPassError = err instanceof Error ? err.message : String(err);
-                stagehandFallbackReason = 'deterministic-observe-failed';
-            }
-        }
-        try {
-            if (!browser) {
-                browser = await connectPlaywright(session.cdpUrl);
-            }
-        }
-        catch (err) {
-            const domFailure = domPassError && domPassError.length > 0 ? `; dom observe failed: ${domPassError}` : '';
-            return buildObserveContractFailureResult(session, {
-                step: observeStep,
-                error: 'browser_connection_failed',
-                outcomeType: 'blocked',
-                message: 'Observe could not start because AgentBrowse failed to connect to the browser.',
-                reason: `${err instanceof Error ? err.message : String(err)}${domFailure}`,
-                pageRef,
-                runId: session.activeRunId,
-                stepId: observeStep?.stepId,
-            });
-        }
-        try {
-            const resolvedPage = await resolveCurrentPageContext(browser, session);
-            pageRef = resolvedPage.pageRef;
-            const page = resolvedPage.page;
-            const { url, title } = await syncSessionPage(session, pageRef, page);
-            const protectedExposure = getProtectedExposure(session, pageRef);
-            if (protectedExposure) {
-                return buildObserveContractFailureResult(session, {
-                    step: observeStep,
-                    ...buildProtectedObserveBlockedResult(protectedExposure, 'stagehand-fallback'),
-                    fallbackReason: stagehandFallbackReason ?? undefined,
-                    deterministicObserveError: domPassError ?? undefined,
-                    pageRef,
-                    runId: session.activeRunId,
-                    stepId: observeStep?.stepId,
-                });
-            }
-            bumpPageScopeEpoch(session, pageRef);
-            setCurrentPage(session, pageRef);
-            const pageSignals = await collectPageSignals(page).catch(() => []);
-            const actions = await withStagehand(session, async (stagehand) => {
-                incrementMetric(session, 'stagehandCalls');
-                return instruction
-                    ? (await stagehand.observe(instruction, {
-                        page,
-                    }))
-                    : (await stagehand.observe({ page }));
-            });
-            const targets = replaceTargetsForPage(session, pageRef, await Promise.all(actions.map((action) => toStagehandDescriptor(pageRef, action, page, normalizePageSignature(url)))));
-            const fillableForms = markProtectedFillableFormsUnknownForPage(session, pageRef);
-            return buildObserveSuccessResult(session, observeStep, {
-                success: true,
-                observationMode: 'goal_assistive_stagehand',
-                pageRef,
-                resolvedBy: 'stagehand-observe',
-                ...stagehandRuntimeResolution(stagehandFallbackReason ?? 'deterministic-observe-failed'),
-                deterministicObserveError: domPassError ?? undefined,
-                scopes: buildGroupedObserveScopes({
-                    pageRef,
-                    title,
-                    scopes: selectScopesForOutput(observedScopes, targets),
-                    targets,
-                }),
-                signals: compactSignals(pageSignals),
-                fillableForms: compactFillableForms(fillableForms),
-                metrics: session.runtime?.metrics,
-                message: targets.length === 0 ? 'This observe pass returned zero targets.' : undefined,
-                url,
-                title,
             });
         }
         catch (err) {
-            const stagehandError = err instanceof Error ? err.message : String(err);
-            const details = domPassError
-                ? `${stagehandError} (deterministic observe failed earlier: ${domPassError})`
-                : stagehandError;
+            const browserConnectionFailure = describeBrowserConnectionFailure(err, {
+                defaultMessage: instruction
+                    ? 'Observe could not start because AgentBrowse failed to connect to the browser.'
+                    : 'Observe failed.',
+                unrecoverableSessionMessage: 'Observe could not start because the previous browser session is no longer reachable.',
+            });
             return buildObserveContractFailureResult(session, {
                 step: observeStep,
-                error: 'observe_failed',
+                error: instruction ? 'browser_connection_failed' : 'observe_failed',
                 outcomeType: 'blocked',
-                message: 'Observe failed.',
-                reason: details,
+                message: browserConnectionFailure.message,
+                reason: browserConnectionFailure.reason,
                 pageRef,
                 runId: session.activeRunId,
                 stepId: observeStep?.stepId,
             });
         }
-        finally {
-            if (browser) {
-                await disconnectPlaywright(browser);
-            }
-        }
     });
 }
 /** CLI wrapper for `observeBrowser(...)` that persists the observed runtime state. */

package/dist/commands/screenshot.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAqBzE,kEAAkE;AAClE,eAAO,MAAM,sBAAsB,6FAIzB,CAAC;AAEX,8DAA8D;AAC9D,eAAO,MAAM,wBAAwB,0EAI3B,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AAC1E,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9E,4CAA4C;AAC5C,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EAAE,mBAAmB,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC,qBAAqB,EAAE,SAAS,GAAG,2BAA2B,CAAC,CAAC;IACrF,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,uBAAuB,GAAG,uBAAuB,CAAC;AAkJjF,0DAA0D;AAC1D,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,CAAC,CAwH3B;AAED,qGAAqG;AACrG,wBAAsB,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBjG"}
+{"version":3,"file":"screenshot.d.ts","sourceRoot":"","sources":["../../src/commands/screenshot.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,6BAA6B,CAAC;AAkBzE,kEAAkE;AAClE,eAAO,MAAM,sBAAsB,6FAIzB,CAAC;AAEX,8DAA8D;AAC9D,eAAO,MAAM,wBAAwB,0EAI3B,CAAC;AAEX,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,sBAAsB,CAAC,CAAC,MAAM,CAAC,CAAC;AAC1E,MAAM,MAAM,qBAAqB,GAAG,CAAC,OAAO,wBAAwB,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9E,4CAA4C;AAC5C,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wCAAwC;AACxC,MAAM,MAAM,uBAAuB,GAAG;IACpC,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EAAE,mBAAmB,CAAC;IAC3B,WAAW,EAAE,OAAO,CAAC,qBAAqB,EAAE,SAAS,GAAG,2BAA2B,CAAC,CAAC;IACrF,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,uBAAuB,GAAG,uBAAuB,CAAC;AAkJjF,0DAA0D;AAC1D,wBAAsB,iBAAiB,CACrC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,gBAAgB,CAAC,CA2G3B;AAED,qGAAqG;AACrG,wBAAsB,UAAU,CAAC,OAAO,EAAE,qBAAqB,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAmBjG"}