@nuanu-ai/agentbrowse 0.2.46 → 0.2.48

package/dist/commands/screenshot.js

@@ -1,16 +1,165 @@
 /**
  * browse screenshot [--path <file>] — Capture a screenshot of the current page.
  */
-import { getProtectedExposure, setCurrentPage } from '../runtime-state.js';
+import { setCurrentPage } from '../runtime-page-state.js';
+import { getProtectedExposure } from '../runtime-protected-state.js';
 import { saveSession } from '../session.js';
 import { outputFailure, outputJSON, outputContractFailure, info } from '../output.js';
 import { connectPlaywright, disconnectPlaywright, resolveCurrentPageContext, syncSessionPage, } from '../playwright-runtime.js';
+import { finishRunStep, startRunStep } from '../run-store.js';
+import { appendCommandLifecycleEventBestEffort, captureStepSnapshotBestEffort, } from '../run-observability.js';
+import { exportRunStepToOtlpHttpJsonBestEffort } from '../otel-exporter.js';
 import { buildProtectedScreenshotBlockedResult } from '../secrets/protected-artifact-guard.js';
+function finalizeScreenshotStepBestEffort(runId, stepId, options) {
+    if (!runId || !stepId) {
+        return;
+    }
+    try {
+        finishRunStep({
+            runId,
+            stepId,
+            ...options,
+        });
+    }
+    catch {
+    }
+}
+async function emitScreenshotSuccess(params) {
+    const step = params.runId && params.stepId
+        ? { runId: params.runId, stepId: params.stepId, command: 'screenshot' }
+        : null;
+    captureStepSnapshotBestEffort({
+        session: params.session,
+        step,
+        phase: 'after',
+        pageRef: params.pageRef,
+        url: params.url,
+        title: params.title,
+        artifactRefs: {
+            screenshotPath: params.path,
+        },
+    });
+    appendCommandLifecycleEventBestEffort({
+        step,
+        phase: 'completed',
+        attributes: {
+            outcomeType: 'screenshot_captured',
+            pageRef: params.pageRef,
+            outputPath: params.path,
+        },
+    });
+    finalizeScreenshotStepBestEffort(params.runId, params.stepId, {
+        success: true,
+        outcomeType: 'screenshot_captured',
+        message: 'Screenshot captured.',
+    });
+    await exportRunStepToOtlpHttpJsonBestEffort(params.runId, params.stepId);
+    return outputJSON({
+        success: true,
+        pageRef: params.pageRef,
+        path: params.path,
+        url: params.url,
+        title: params.title,
+    });
+}
+async function emitScreenshotFailure(session, params) {
+    const step = params.runId && params.stepId
+        ? { runId: params.runId, stepId: params.stepId, command: 'screenshot' }
+        : null;
+    captureStepSnapshotBestEffort({
+        session,
+        step,
+        phase: 'point-in-time',
+        pageRef: typeof params.pageRef === 'string' ? params.pageRef : session.runtime?.currentPageRef,
+    });
+    appendCommandLifecycleEventBestEffort({
+        step,
+        phase: 'failed',
+        attributes: {
+            outcomeType: typeof params.outcomeType === 'string' ? params.outcomeType : params.error,
+            pageRef: typeof params.pageRef === 'string' ? params.pageRef : undefined,
+            reason: typeof params.reason === 'string' ? params.reason : params.error,
+        },
+    });
+    finalizeScreenshotStepBestEffort(params.runId, params.stepId, {
+        success: false,
+        outcomeType: typeof params.outcomeType === 'string' ? params.outcomeType : params.error,
+        message: typeof params.message === 'string' ? params.message : params.error,
+        reason: typeof params.reason === 'string' ? params.reason : params.error,
+    });
+    await exportRunStepToOtlpHttpJsonBestEffort(params.runId, params.stepId);
+    const { runId: _runId, stepId: _stepId, ...result } = params;
+    return outputFailure(result);
+}
+async function emitScreenshotContractFailure(session, params) {
+    const step = params.runId && params.stepId
+        ? { runId: params.runId, stepId: params.stepId, command: 'screenshot' }
+        : null;
+    captureStepSnapshotBestEffort({
+        session,
+        step,
+        phase: 'point-in-time',
+        pageRef: typeof params.pageRef === 'string' ? params.pageRef : session.runtime?.currentPageRef,
+    });
+    appendCommandLifecycleEventBestEffort({
+        step,
+        phase: 'failed',
+        attributes: {
+            outcomeType: params.outcomeType,
+            pageRef: typeof params.pageRef === 'string' ? params.pageRef : undefined,
+            reason: params.reason,
+        },
+    });
+    finalizeScreenshotStepBestEffort(params.runId, params.stepId, {
+        success: false,
+        outcomeType: params.outcomeType,
+        message: params.message,
+        reason: params.reason,
+    });
+    await exportRunStepToOtlpHttpJsonBestEffort(params.runId, params.stepId);
+    const { runId: _runId, stepId: _stepId, ...result } = params;
+    return outputContractFailure(result);
+}
 export async function screenshot(session, filePath) {
     const outputPath = filePath ?? `/tmp/browse-screenshot-${Date.now()}.png`;
     const initialPageRef = session.runtime?.currentPageRef ?? 'p0';
     let pageRef = initialPageRef;
     const initialProtectedExposure = getProtectedExposure(session, initialPageRef);
+    const screenshotStep = session.activeRunId
+        ? startRunStep({
+            runId: session.activeRunId,
+            command: 'screenshot',
+            input: {
+                outputPath,
+            },
+            refs: {
+                pageRef: initialPageRef,
+            },
+            protectedStep: Boolean(initialProtectedExposure),
+        })
+        : null;
+    const screenshotStepHandle = session.activeRunId && screenshotStep?.stepId
+        ? {
+            runId: session.activeRunId,
+            stepId: screenshotStep.stepId,
+            command: 'screenshot',
+        }
+        : null;
+    captureStepSnapshotBestEffort({
+        session,
+        step: screenshotStepHandle,
+        phase: 'before',
+        pageRef: initialPageRef,
+    });
+    appendCommandLifecycleEventBestEffort({
+        step: screenshotStepHandle,
+        phase: 'started',
+        attributes: {
+            outputPath,
+            pageRef: initialPageRef,
+            protectedStep: Boolean(initialProtectedExposure),
+        },
+    });
     let browser = null;
     let failureMessage = null;
     let page = null;
@@ -18,11 +167,13 @@ export async function screenshot(session, filePath) {
         browser = await connectPlaywright(session.cdpUrl);
     }
     catch (err) {
-        return outputContractFailure({
+        return emitScreenshotContractFailure(session, {
             error: 'browser_connection_failed',
             outcomeType: 'blocked',
             message: 'Screenshot capture could not start because AgentBrowse failed to connect to the browser.',
             reason: err instanceof Error ? err.message : String(err),
+            runId: session.activeRunId,
+            stepId: screenshotStep?.stepId,
         });
     }
     try {
@@ -40,10 +191,14 @@ export async function screenshot(session, filePath) {
             saveSession(session);
         }
         if (browser) {
-            disconnectPlaywright(browser);
+            await disconnectPlaywright(browser);
             browser = null;
         }
-        return outputFailure(buildProtectedScreenshotBlockedResult(recoveredProtectedExposure));
+        return emitScreenshotFailure(session, {
+            ...buildProtectedScreenshotBlockedResult(recoveredProtectedExposure),
+            runId: session.activeRunId,
+            stepId: screenshotStep?.stepId,
+        });
     }
     try {
         if (!page) {
@@ -54,22 +209,32 @@ export async function screenshot(session, filePath) {
         const { url, title } = await syncSessionPage(session, pageRef, page);
         setCurrentPage(session, pageRef);
         saveSession(session);
-        outputJSON({ success: true, pageRef, path: outputPath, url, title });
+        return emitScreenshotSuccess({
+            session,
+            runId: session.activeRunId,
+            stepId: screenshotStep?.stepId,
+            pageRef,
+            path: outputPath,
+            url,
+            title,
+        });
     }
     catch (err) {
         failureMessage = `Screenshot failed: ${err instanceof Error ? err.message : String(err)}`;
     }
     finally {
         if (browser) {
-            disconnectPlaywright(browser);
+            await disconnectPlaywright(browser);
         }
     }
     if (failureMessage) {
-        outputContractFailure({
+        return emitScreenshotContractFailure(session, {
             error: 'screenshot_failed',
             outcomeType: 'blocked',
             message: 'Screenshot capture failed.',
             reason: failureMessage.replace(/^Screenshot failed:\s*/, ''),
+            runId: session.activeRunId,
+            stepId: screenshotStep?.stepId,
         });
     }
 }

package/dist/commands/select-action-executor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"select-action-executor.d.ts","sourceRoot":"","sources":["../../src/commands/select-action-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAA2B,KAAK,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAEnG,KAAK,WAAW,GAAG,IAAI,GAAG,YAAY,GAAG,OAAO,CAAC;AAajD,KAAK,mBAAmB,GAAG;IACzB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,0BAA0B,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrF,CAAC;AAgKF,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,OAAO,CAAC,CAsElB"}
+{"version":3,"file":"select-action-executor.d.ts","sourceRoot":"","sources":["../../src/commands/select-action-executor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AACnE,OAAO,EAA2B,KAAK,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AAEnG,KAAK,WAAW,GAAG,IAAI,GAAG,YAAY,GAAG,OAAO,CAAC;AAajD,KAAK,mBAAmB,GAAG;IACzB,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAClF,0BAA0B,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CACrF,CAAC;AAmKF,wBAAsB,qBAAqB,CACzC,IAAI,EAAE,IAAI,EACV,IAAI,EAAE,WAAW,EACjB,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAAE,EAClB,OAAO,EAAE,mBAAmB,EAC5B,MAAM,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,OAAO,CAAC,CAsElB"}

package/dist/commands/semantic-observe.d.ts

@@ -3,8 +3,10 @@ import type { TargetAcceptancePolicy, TargetAllowedAction, TargetCapability, Tar
 type ObserveInventoryTarget = {
     kind?: string;
     label?: string;
+    displayLabel?: string;
     role?: string;
     interactionHint?: 'click';
+    text?: string;
     placeholder?: string;
     title?: string;
     states?: Record<string, string | boolean | number>;
@@ -24,6 +26,8 @@ type ObserveInventoryTarget = {
     pageSignature?: string;
     goalInventoryType?: 'target' | 'scope';
     goalSurfaceId?: string;
+    goalLabel?: string;
+    goalAliases?: string[];
     controlsSurfaceSelector?: string;
 };
 export declare function rerankDomTargetsForGoal<T extends ObserveInventoryTarget>(instruction: string, targets: ReadonlyArray<T>, options?: {

package/dist/commands/semantic-observe.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-observe.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAY7B,KAAK,sBAAsB,GAAG;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,CAAC;AA8oCF,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,sBAAsB,EAC5E,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EACzB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAO,GACxC,OAAO,CAAC,CAAC,EAAE,CAAC,CA6Ed"}
+{"version":3,"file":"semantic-observe.d.ts","sourceRoot":"","sources":["../../src/commands/semantic-observe.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,KAAK,EACV,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAY7B,KAAK,sBAAsB,GAAG;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;IACnD,OAAO,CAAC,EAAE,aAAa,CAAC;IACxB,UAAU,CAAC,EAAE,gBAAgB,CAAC;IAC9B,cAAc,CAAC,EAAE,mBAAmB,EAAE,CAAC;IACvC,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;IAC1C,aAAa,CAAC,EAAE,mBAAmB,CAAC;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,EAAE,eAAe,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,iBAAiB,CAAC,EAAE,QAAQ,GAAG,OAAO,CAAC;IACvC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,uBAAuB,CAAC,EAAE,MAAM,CAAC;CAClC,CAAC;AAuoDF,wBAAsB,uBAAuB,CAAC,CAAC,SAAS,sBAAsB,EAC5E,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,EACzB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,aAAa,CAAA;CAAO,GACxC,OAAO,CAAC,CAAC,EAAE,CAAC,CAqGd"}