@mercuryo-ai/agentbrowse 0.2.50 → 0.2.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -1
- package/dist/browser-session-state.d.ts +8 -8
- package/dist/browser-session-state.d.ts.map +1 -1
- package/dist/browser-session-state.js +41 -47
- package/dist/commands/action-acceptance.d.ts.map +1 -1
- package/dist/commands/observe-projection.d.ts.map +1 -1
- package/dist/commands/observe.d.ts +8 -8
- package/dist/commands/observe.d.ts.map +1 -1
- package/dist/commands/observe.js +98 -114
- package/dist/commands/user-actionable.d.ts.map +1 -1
- package/dist/protected-fill.d.ts +12 -12
- package/dist/protected-fill.d.ts.map +1 -1
- package/dist/protected-fill.js +3 -3
- package/dist/runtime-state.d.ts +28 -28
- package/dist/runtime-state.d.ts.map +1 -1
- package/dist/runtime-state.js +50 -59
- package/dist/secrets/form-matcher.d.ts.map +1 -1
- package/dist/secrets/form-matcher.js +1 -3
- package/dist/secrets/protected-exact-value-redaction.d.ts.map +1 -1
- package/dist/secrets/protected-exact-value-redaction.js +3 -1
- package/package.json +4 -4
package/dist/commands/observe.js
CHANGED
|
@@ -1,50 +1,50 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* browse observe ["<instruction>"] — Discover available actions on the page.
|
|
3
3
|
*/
|
|
4
|
-
import { canUseAgentbrowseAssistiveLlmClient } from
|
|
5
|
-
import { saveSession } from
|
|
6
|
-
import { ensureRuntimeState, replaceTargetsForPage } from
|
|
7
|
-
import { incrementMetric } from
|
|
8
|
-
import { bumpPageScopeEpoch, setCurrentPage } from
|
|
9
|
-
import { getProtectedExposure } from
|
|
10
|
-
import { connectPlaywright, disconnectPlaywright, resolveCurrentPageContext, syncSessionPage, } from
|
|
11
|
-
import { tracedStepOperation, withApiTraceContext
|
|
12
|
-
import { captureDiagnosticSnapshotBestEffort, finishDiagnosticStepBestEffort, recordCommandLifecycleEventBestEffort, startDiagnosticStep, } from
|
|
13
|
-
import { outputContractFailure, outputJSON, } from
|
|
14
|
-
import { domRuntimeResolution, stagehandRuntimeResolution
|
|
15
|
-
import { withStagehand } from
|
|
16
|
-
import { buildProtectedObserveBlockedResult } from
|
|
17
|
-
import { scrubProtectedExactValues } from
|
|
18
|
-
import { normalizePageSignature } from
|
|
19
|
-
import { collectDomTargets, __testDomTargetCollection as inventoryDomTargetCollection, __testStagehandDescriptor as inventoryStagehandDescriptor, } from
|
|
20
|
-
import { enrichDomTargetsWithAccessibility } from
|
|
21
|
-
import { collectPageSignals } from
|
|
22
|
-
import { attachObservedTargetOwners, linkObservedSurfaceGraph, reconcileObservedTargetsForPage, persistObservedSurfacesForPage, toDomDescriptor, } from
|
|
23
|
-
import { clearProtectedFillableFormsForPage, markProtectedFillableFormsUnknownForPage, persistProtectedFillableFormsForPage, } from
|
|
24
|
-
import { classifyObservePageState, shouldSuppressFillableFormsForObserve, } from
|
|
25
|
-
import { annotateDomTargets, compressSemanticallyDuplicateTargets, orderBySurfaceCompetition, prioritizeGoalActionTargets, } from
|
|
26
|
-
import { buildGroupedObserveScopes, buildGoalProjectionScopeRefs, buildGoalObserveInventoryCandidates, compactFillableForms, compactSignals, expandWorkflowGraphTargets, projectPersistedTargetsForGoal, selectTargetsForGoalMatches, } from
|
|
27
|
-
import { collectSurfaceDescriptors, selectScopesForOutput
|
|
28
|
-
import { toStagehandDescriptor } from
|
|
29
|
-
import { rerankDomTargetsForGoal } from
|
|
4
|
+
import { canUseAgentbrowseAssistiveLlmClient } from '../assistive-runtime.js';
|
|
5
|
+
import { saveSession } from '../session.js';
|
|
6
|
+
import { ensureRuntimeState, replaceTargetsForPage } from '../runtime-state.js';
|
|
7
|
+
import { incrementMetric } from '../runtime-metrics.js';
|
|
8
|
+
import { bumpPageScopeEpoch, setCurrentPage } from '../runtime-page-state.js';
|
|
9
|
+
import { getProtectedExposure } from '../runtime-protected-state.js';
|
|
10
|
+
import { connectPlaywright, disconnectPlaywright, resolveCurrentPageContext, syncSessionPage, } from '../playwright-runtime.js';
|
|
11
|
+
import { tracedStepOperation, withApiTraceContext } from '../command-api-tracing.js';
|
|
12
|
+
import { captureDiagnosticSnapshotBestEffort, finishDiagnosticStepBestEffort, recordCommandLifecycleEventBestEffort, startDiagnosticStep, } from '../diagnostics.js';
|
|
13
|
+
import { outputContractFailure, outputJSON, } from '../output.js';
|
|
14
|
+
import { domRuntimeResolution, stagehandRuntimeResolution } from '../runtime-resolution.js';
|
|
15
|
+
import { withStagehand } from '../stagehand-runtime.js';
|
|
16
|
+
import { buildProtectedObserveBlockedResult } from '../secrets/protected-artifact-guard.js';
|
|
17
|
+
import { scrubProtectedExactValues } from '../secrets/protected-exact-value-redaction.js';
|
|
18
|
+
import { normalizePageSignature } from './descriptor-validation.js';
|
|
19
|
+
import { collectDomTargets, __testDomTargetCollection as inventoryDomTargetCollection, __testStagehandDescriptor as inventoryStagehandDescriptor, } from './observe-inventory.js';
|
|
20
|
+
import { enrichDomTargetsWithAccessibility } from './observe-accessibility.js';
|
|
21
|
+
import { collectPageSignals } from './observe-signals.js';
|
|
22
|
+
import { attachObservedTargetOwners, linkObservedSurfaceGraph, reconcileObservedTargetsForPage, persistObservedSurfacesForPage, toDomDescriptor, } from './observe-persistence.js';
|
|
23
|
+
import { clearProtectedFillableFormsForPage, markProtectedFillableFormsUnknownForPage, persistProtectedFillableFormsForPage, } from './observe-protected.js';
|
|
24
|
+
import { classifyObservePageState, shouldSuppressFillableFormsForObserve, } from './observe-page-state.js';
|
|
25
|
+
import { annotateDomTargets, compressSemanticallyDuplicateTargets, orderBySurfaceCompetition, prioritizeGoalActionTargets, } from './observe-semantics.js';
|
|
26
|
+
import { buildGroupedObserveScopes, buildGoalProjectionScopeRefs, buildGoalObserveInventoryCandidates, compactFillableForms, compactSignals, expandWorkflowGraphTargets, projectPersistedTargetsForGoal, selectTargetsForGoalMatches, } from './observe-projection.js';
|
|
27
|
+
import { collectSurfaceDescriptors, selectScopesForOutput } from './observe-surfaces.js';
|
|
28
|
+
import { toStagehandDescriptor } from './observe-stagehand.js';
|
|
29
|
+
import { rerankDomTargetsForGoal } from './semantic-observe.js';
|
|
30
30
|
/** Stable top-level error codes returned by `observe(...)`. */
|
|
31
31
|
export const OBSERVE_ERROR_CODES = [
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
'browser_connection_failed',
|
|
33
|
+
'observe_failed',
|
|
34
|
+
'protected_observe_blocked',
|
|
35
35
|
];
|
|
36
36
|
/** Stable outcome categories emitted by `observe(...)`. */
|
|
37
37
|
export const OBSERVE_OUTCOME_TYPES = [
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
'blocked',
|
|
39
|
+
'observation_completed',
|
|
40
|
+
'protected_exposure_active',
|
|
41
41
|
];
|
|
42
42
|
function flattenObserveTargets(scopes) {
|
|
43
43
|
if (!Array.isArray(scopes)) {
|
|
44
44
|
return [];
|
|
45
45
|
}
|
|
46
46
|
return scopes.flatMap((scope) => {
|
|
47
|
-
if (!scope || typeof scope !==
|
|
47
|
+
if (!scope || typeof scope !== 'object' || Array.isArray(scope)) {
|
|
48
48
|
return [];
|
|
49
49
|
}
|
|
50
50
|
const typedScope = scope;
|
|
@@ -62,9 +62,7 @@ function normalizeObserveSuccessPayload(payload) {
|
|
|
62
62
|
scopes,
|
|
63
63
|
targets,
|
|
64
64
|
signals: Array.isArray(payload.signals) ? payload.signals : [],
|
|
65
|
-
fillableForms: Array.isArray(payload.fillableForms)
|
|
66
|
-
? payload.fillableForms
|
|
67
|
-
: [],
|
|
65
|
+
fillableForms: Array.isArray(payload.fillableForms) ? payload.fillableForms : [],
|
|
68
66
|
targetCount: targets.length,
|
|
69
67
|
scopeCount: scopes.length,
|
|
70
68
|
projectedTargetCount: targets.length,
|
|
@@ -98,36 +96,32 @@ async function buildObserveSuccessResult(session, step, payload) {
|
|
|
98
96
|
captureDiagnosticSnapshotBestEffort({
|
|
99
97
|
session,
|
|
100
98
|
step,
|
|
101
|
-
phase:
|
|
102
|
-
pageRef: typeof normalizedPayload.pageRef ===
|
|
99
|
+
phase: 'after',
|
|
100
|
+
pageRef: typeof normalizedPayload.pageRef === 'string'
|
|
103
101
|
? normalizedPayload.pageRef
|
|
104
102
|
: session.runtime?.currentPageRef,
|
|
105
|
-
url: typeof normalizedPayload.url ===
|
|
106
|
-
|
|
107
|
-
: undefined,
|
|
108
|
-
title: typeof normalizedPayload.title === "string"
|
|
109
|
-
? normalizedPayload.title
|
|
110
|
-
: undefined,
|
|
103
|
+
url: typeof normalizedPayload.url === 'string' ? normalizedPayload.url : undefined,
|
|
104
|
+
title: typeof normalizedPayload.title === 'string' ? normalizedPayload.title : undefined,
|
|
111
105
|
});
|
|
112
106
|
recordCommandLifecycleEventBestEffort({
|
|
113
107
|
step,
|
|
114
|
-
phase:
|
|
108
|
+
phase: 'completed',
|
|
115
109
|
attributes: {
|
|
116
|
-
outcomeType:
|
|
117
|
-
...(typeof normalizedPayload.pageRef ===
|
|
110
|
+
outcomeType: 'observation_completed',
|
|
111
|
+
...(typeof normalizedPayload.pageRef === 'string'
|
|
118
112
|
? { pageRef: normalizedPayload.pageRef }
|
|
119
113
|
: {}),
|
|
120
|
-
...(typeof normalizedPayload.resolvedBy ===
|
|
114
|
+
...(typeof normalizedPayload.resolvedBy === 'string'
|
|
121
115
|
? { resolvedBy: normalizedPayload.resolvedBy }
|
|
122
116
|
: {}),
|
|
123
117
|
},
|
|
124
118
|
});
|
|
125
119
|
await finalizeObserveStepBestEffort(step, {
|
|
126
120
|
success: true,
|
|
127
|
-
outcomeType:
|
|
128
|
-
message: typeof normalizedPayload.message ===
|
|
121
|
+
outcomeType: 'observation_completed',
|
|
122
|
+
message: typeof normalizedPayload.message === 'string'
|
|
129
123
|
? normalizedPayload.message
|
|
130
|
-
:
|
|
124
|
+
: 'Observe completed.',
|
|
131
125
|
});
|
|
132
126
|
return normalizedPayload;
|
|
133
127
|
}
|
|
@@ -136,17 +130,15 @@ async function buildObserveContractFailureResult(session, params) {
|
|
|
136
130
|
captureDiagnosticSnapshotBestEffort({
|
|
137
131
|
session,
|
|
138
132
|
step,
|
|
139
|
-
phase:
|
|
140
|
-
pageRef: typeof params.pageRef ===
|
|
133
|
+
phase: 'point-in-time',
|
|
134
|
+
pageRef: typeof params.pageRef === 'string' ? params.pageRef : undefined,
|
|
141
135
|
});
|
|
142
136
|
recordCommandLifecycleEventBestEffort({
|
|
143
137
|
step,
|
|
144
|
-
phase:
|
|
138
|
+
phase: 'failed',
|
|
145
139
|
attributes: {
|
|
146
140
|
...(params.outcomeType ? { outcomeType: params.outcomeType } : {}),
|
|
147
|
-
...(typeof params.pageRef ===
|
|
148
|
-
? { pageRef: params.pageRef }
|
|
149
|
-
: {}),
|
|
141
|
+
...(typeof params.pageRef === 'string' ? { pageRef: params.pageRef } : {}),
|
|
150
142
|
...(params.reason ? { reason: params.reason } : {}),
|
|
151
143
|
},
|
|
152
144
|
});
|
|
@@ -186,7 +178,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
186
178
|
let observedScopes = [];
|
|
187
179
|
const observeStep = startDiagnosticStep({
|
|
188
180
|
runId: session.activeRunId,
|
|
189
|
-
command:
|
|
181
|
+
command: 'observe',
|
|
190
182
|
input: {
|
|
191
183
|
...(instruction ? { instruction } : {}),
|
|
192
184
|
},
|
|
@@ -198,12 +190,12 @@ export async function observeBrowser(session, instruction) {
|
|
|
198
190
|
captureDiagnosticSnapshotBestEffort({
|
|
199
191
|
session,
|
|
200
192
|
step: observeStep,
|
|
201
|
-
phase:
|
|
193
|
+
phase: 'before',
|
|
202
194
|
pageRef,
|
|
203
195
|
});
|
|
204
196
|
recordCommandLifecycleEventBestEffort({
|
|
205
197
|
step: observeStep,
|
|
206
|
-
phase:
|
|
198
|
+
phase: 'started',
|
|
207
199
|
attributes: {
|
|
208
200
|
...(instruction ? { instruction } : {}),
|
|
209
201
|
pageRef,
|
|
@@ -212,18 +204,16 @@ export async function observeBrowser(session, instruction) {
|
|
|
212
204
|
return withApiTraceContext({
|
|
213
205
|
runId: session.activeRunId,
|
|
214
206
|
stepId: observeStep?.stepId,
|
|
215
|
-
command:
|
|
207
|
+
command: 'observe',
|
|
216
208
|
}, async () => {
|
|
217
209
|
const observePhaseAttributes = {
|
|
218
|
-
|
|
210
|
+
'agentbrowse.observe.goal_based': Boolean(instruction),
|
|
219
211
|
};
|
|
220
|
-
const initialProtectedExposure = instruction
|
|
221
|
-
? getProtectedExposure(session, pageRef)
|
|
222
|
-
: null;
|
|
212
|
+
const initialProtectedExposure = instruction ? getProtectedExposure(session, pageRef) : null;
|
|
223
213
|
if (instruction && initialProtectedExposure) {
|
|
224
214
|
return buildObserveContractFailureResult(session, {
|
|
225
215
|
step: observeStep,
|
|
226
|
-
...buildProtectedObserveBlockedResult(initialProtectedExposure,
|
|
216
|
+
...buildProtectedObserveBlockedResult(initialProtectedExposure, 'goal-rerank'),
|
|
227
217
|
pageRef,
|
|
228
218
|
runId: session.activeRunId,
|
|
229
219
|
stepId: observeStep?.stepId,
|
|
@@ -247,9 +237,9 @@ export async function observeBrowser(session, instruction) {
|
|
|
247
237
|
},
|
|
248
238
|
}))));
|
|
249
239
|
if (observeAccessibilityStats) {
|
|
250
|
-
incrementMetric(session,
|
|
251
|
-
incrementMetric(session,
|
|
252
|
-
incrementMetric(session,
|
|
240
|
+
incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
|
|
241
|
+
incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
|
|
242
|
+
incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
|
|
253
243
|
}
|
|
254
244
|
const pageSignals = await collectPageSignals(page).catch(() => []);
|
|
255
245
|
const pageState = classifyObservePageState(pageSignals);
|
|
@@ -268,9 +258,9 @@ export async function observeBrowser(session, instruction) {
|
|
|
268
258
|
browser = null;
|
|
269
259
|
return buildObserveSuccessResult(session, observeStep, {
|
|
270
260
|
success: true,
|
|
271
|
-
observationMode:
|
|
261
|
+
observationMode: 'deterministic_dom',
|
|
272
262
|
pageRef,
|
|
273
|
-
resolvedBy:
|
|
263
|
+
resolvedBy: 'dom',
|
|
274
264
|
...domRuntimeResolution(),
|
|
275
265
|
scopes: buildGroupedObserveScopes({
|
|
276
266
|
pageRef,
|
|
@@ -281,9 +271,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
281
271
|
signals: compactSignals(pageSignals),
|
|
282
272
|
fillableForms: compactFillableForms(fillableForms),
|
|
283
273
|
metrics: session.runtime?.metrics,
|
|
284
|
-
message: targets.length === 0
|
|
285
|
-
? "This observe pass returned zero targets."
|
|
286
|
-
: undefined,
|
|
274
|
+
message: targets.length === 0 ? 'This observe pass returned zero targets.' : undefined,
|
|
287
275
|
...buildObservePageMetadata({
|
|
288
276
|
url,
|
|
289
277
|
title,
|
|
@@ -296,15 +284,15 @@ export async function observeBrowser(session, instruction) {
|
|
|
296
284
|
browser = null;
|
|
297
285
|
return buildObserveSuccessResult(session, observeStep, {
|
|
298
286
|
success: true,
|
|
299
|
-
observationMode:
|
|
287
|
+
observationMode: 'deterministic_dom',
|
|
300
288
|
pageRef,
|
|
301
|
-
resolvedBy:
|
|
289
|
+
resolvedBy: 'dom',
|
|
302
290
|
...domRuntimeResolution(),
|
|
303
291
|
scopes: [],
|
|
304
292
|
signals: compactSignals(pageSignals),
|
|
305
293
|
fillableForms: [],
|
|
306
294
|
metrics: session.runtime?.metrics,
|
|
307
|
-
message:
|
|
295
|
+
message: 'This observe pass returned zero targets.',
|
|
308
296
|
...buildObservePageMetadata({
|
|
309
297
|
url,
|
|
310
298
|
title,
|
|
@@ -312,23 +300,23 @@ export async function observeBrowser(session, instruction) {
|
|
|
312
300
|
}),
|
|
313
301
|
});
|
|
314
302
|
}
|
|
315
|
-
stagehandFallbackReason =
|
|
303
|
+
stagehandFallbackReason = 'deterministic-observe-empty';
|
|
316
304
|
}
|
|
317
305
|
catch (err) {
|
|
318
306
|
domPassError = err instanceof Error ? err.message : String(err);
|
|
319
307
|
if (!allowAssistive) {
|
|
320
308
|
return buildObserveContractFailureResult(session, {
|
|
321
309
|
step: observeStep,
|
|
322
|
-
error:
|
|
323
|
-
outcomeType:
|
|
324
|
-
message:
|
|
310
|
+
error: 'observe_failed',
|
|
311
|
+
outcomeType: 'blocked',
|
|
312
|
+
message: 'Observe failed.',
|
|
325
313
|
reason: domPassError,
|
|
326
314
|
pageRef,
|
|
327
315
|
runId: session.activeRunId,
|
|
328
316
|
stepId: observeStep?.stepId,
|
|
329
317
|
});
|
|
330
318
|
}
|
|
331
|
-
stagehandFallbackReason =
|
|
319
|
+
stagehandFallbackReason = 'deterministic-observe-failed';
|
|
332
320
|
}
|
|
333
321
|
}
|
|
334
322
|
if (instruction) {
|
|
@@ -344,7 +332,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
344
332
|
if (protectedExposure) {
|
|
345
333
|
return buildObserveContractFailureResult(session, {
|
|
346
334
|
step: observeStep,
|
|
347
|
-
...buildProtectedObserveBlockedResult(protectedExposure,
|
|
335
|
+
...buildProtectedObserveBlockedResult(protectedExposure, 'goal-rerank'),
|
|
348
336
|
pageRef,
|
|
349
337
|
runId: session.activeRunId,
|
|
350
338
|
stepId: observeStep?.stepId,
|
|
@@ -362,19 +350,19 @@ export async function observeBrowser(session, instruction) {
|
|
|
362
350
|
},
|
|
363
351
|
}))));
|
|
364
352
|
if (observeAccessibilityStats) {
|
|
365
|
-
incrementMetric(session,
|
|
366
|
-
incrementMetric(session,
|
|
367
|
-
incrementMetric(session,
|
|
353
|
+
incrementMetric(session, 'observeAxAttempts', observeAccessibilityStats.axAttempts);
|
|
354
|
+
incrementMetric(session, 'observeAxHits', observeAccessibilityStats.axHits);
|
|
355
|
+
incrementMetric(session, 'observeFallbackUses', observeAccessibilityStats.fallbackUses);
|
|
368
356
|
}
|
|
369
357
|
const pageSignals = await collectPageSignals(page).catch(() => []);
|
|
370
358
|
const pageState = classifyObservePageState(pageSignals);
|
|
371
359
|
const surfaceInputs = collectSurfaceDescriptors(pageRef, domTargets);
|
|
372
360
|
if (domTargets.length > 0) {
|
|
373
361
|
const rerankedCandidates = await tracedStepOperation(() => rerankDomTargetsForGoal(instruction, buildGoalObserveInventoryCandidates(domTargets, surfaceInputs), { session }), {
|
|
374
|
-
spanName:
|
|
362
|
+
spanName: 'agentbrowse.observe.rerank_goal_candidates',
|
|
375
363
|
attributes: {
|
|
376
364
|
...observePhaseAttributes,
|
|
377
|
-
|
|
365
|
+
'agentbrowse.observe.target_count': domTargets.length,
|
|
378
366
|
},
|
|
379
367
|
});
|
|
380
368
|
const { targets: goalMatchedTargets, selectedSurfaceIds } = selectTargetsForGoalMatches(domTargets, rerankedCandidates);
|
|
@@ -402,10 +390,10 @@ export async function observeBrowser(session, instruction) {
|
|
|
402
390
|
return buildObserveSuccessResult(session, observeStep, {
|
|
403
391
|
success: true,
|
|
404
392
|
observationMode: canUseAssistiveLlm
|
|
405
|
-
?
|
|
406
|
-
:
|
|
393
|
+
? 'goal_assistive_rerank'
|
|
394
|
+
: 'goal_heuristic_shortlist',
|
|
407
395
|
pageRef,
|
|
408
|
-
resolvedBy:
|
|
396
|
+
resolvedBy: 'dom-rerank',
|
|
409
397
|
...domRuntimeResolution(),
|
|
410
398
|
scopes: buildGroupedObserveScopes({
|
|
411
399
|
pageRef,
|
|
@@ -425,25 +413,25 @@ export async function observeBrowser(session, instruction) {
|
|
|
425
413
|
return buildObserveSuccessResult(session, observeStep, {
|
|
426
414
|
success: true,
|
|
427
415
|
observationMode: canUseAssistiveLlm
|
|
428
|
-
?
|
|
429
|
-
:
|
|
416
|
+
? 'goal_assistive_rerank'
|
|
417
|
+
: 'goal_heuristic_shortlist',
|
|
430
418
|
pageRef,
|
|
431
|
-
resolvedBy:
|
|
419
|
+
resolvedBy: 'dom-rerank',
|
|
432
420
|
...domRuntimeResolution(),
|
|
433
421
|
scopes: [],
|
|
434
422
|
signals: compactSignals(pageSignals),
|
|
435
423
|
fillableForms: compactFillableForms(fillableForms),
|
|
436
424
|
metrics: session.runtime?.metrics,
|
|
437
|
-
message:
|
|
425
|
+
message: 'This goal-based observe pass returned zero matching targets.',
|
|
438
426
|
url,
|
|
439
427
|
title,
|
|
440
428
|
});
|
|
441
429
|
}
|
|
442
|
-
stagehandFallbackReason =
|
|
430
|
+
stagehandFallbackReason = 'deterministic-observe-empty';
|
|
443
431
|
}
|
|
444
432
|
catch (err) {
|
|
445
433
|
domPassError = err instanceof Error ? err.message : String(err);
|
|
446
|
-
stagehandFallbackReason =
|
|
434
|
+
stagehandFallbackReason = 'deterministic-observe-failed';
|
|
447
435
|
}
|
|
448
436
|
}
|
|
449
437
|
try {
|
|
@@ -452,14 +440,12 @@ export async function observeBrowser(session, instruction) {
|
|
|
452
440
|
}
|
|
453
441
|
}
|
|
454
442
|
catch (err) {
|
|
455
|
-
const domFailure = domPassError && domPassError.length > 0
|
|
456
|
-
? `; dom observe failed: ${domPassError}`
|
|
457
|
-
: "";
|
|
443
|
+
const domFailure = domPassError && domPassError.length > 0 ? `; dom observe failed: ${domPassError}` : '';
|
|
458
444
|
return buildObserveContractFailureResult(session, {
|
|
459
445
|
step: observeStep,
|
|
460
|
-
error:
|
|
461
|
-
outcomeType:
|
|
462
|
-
message:
|
|
446
|
+
error: 'browser_connection_failed',
|
|
447
|
+
outcomeType: 'blocked',
|
|
448
|
+
message: 'Observe could not start because AgentBrowse failed to connect to the browser.',
|
|
463
449
|
reason: `${err instanceof Error ? err.message : String(err)}${domFailure}`,
|
|
464
450
|
pageRef,
|
|
465
451
|
runId: session.activeRunId,
|
|
@@ -475,7 +461,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
475
461
|
if (protectedExposure) {
|
|
476
462
|
return buildObserveContractFailureResult(session, {
|
|
477
463
|
step: observeStep,
|
|
478
|
-
...buildProtectedObserveBlockedResult(protectedExposure,
|
|
464
|
+
...buildProtectedObserveBlockedResult(protectedExposure, 'stagehand-fallback'),
|
|
479
465
|
fallbackReason: stagehandFallbackReason ?? undefined,
|
|
480
466
|
deterministicObserveError: domPassError ?? undefined,
|
|
481
467
|
pageRef,
|
|
@@ -487,7 +473,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
487
473
|
setCurrentPage(session, pageRef);
|
|
488
474
|
const pageSignals = await collectPageSignals(page).catch(() => []);
|
|
489
475
|
const actions = await withStagehand(session, async (stagehand) => {
|
|
490
|
-
incrementMetric(session,
|
|
476
|
+
incrementMetric(session, 'stagehandCalls');
|
|
491
477
|
return instruction
|
|
492
478
|
? (await stagehand.observe(instruction, {
|
|
493
479
|
page,
|
|
@@ -498,10 +484,10 @@ export async function observeBrowser(session, instruction) {
|
|
|
498
484
|
const fillableForms = markProtectedFillableFormsUnknownForPage(session, pageRef);
|
|
499
485
|
return buildObserveSuccessResult(session, observeStep, {
|
|
500
486
|
success: true,
|
|
501
|
-
observationMode:
|
|
487
|
+
observationMode: 'goal_assistive_stagehand',
|
|
502
488
|
pageRef,
|
|
503
|
-
resolvedBy:
|
|
504
|
-
...stagehandRuntimeResolution(stagehandFallbackReason ??
|
|
489
|
+
resolvedBy: 'stagehand-observe',
|
|
490
|
+
...stagehandRuntimeResolution(stagehandFallbackReason ?? 'deterministic-observe-failed'),
|
|
505
491
|
deterministicObserveError: domPassError ?? undefined,
|
|
506
492
|
scopes: buildGroupedObserveScopes({
|
|
507
493
|
pageRef,
|
|
@@ -512,9 +498,7 @@ export async function observeBrowser(session, instruction) {
|
|
|
512
498
|
signals: compactSignals(pageSignals),
|
|
513
499
|
fillableForms: compactFillableForms(fillableForms),
|
|
514
500
|
metrics: session.runtime?.metrics,
|
|
515
|
-
message: targets.length === 0
|
|
516
|
-
? "This observe pass returned zero targets."
|
|
517
|
-
: undefined,
|
|
501
|
+
message: targets.length === 0 ? 'This observe pass returned zero targets.' : undefined,
|
|
518
502
|
url,
|
|
519
503
|
title,
|
|
520
504
|
});
|
|
@@ -526,9 +510,9 @@ export async function observeBrowser(session, instruction) {
|
|
|
526
510
|
: stagehandError;
|
|
527
511
|
return buildObserveContractFailureResult(session, {
|
|
528
512
|
step: observeStep,
|
|
529
|
-
error:
|
|
530
|
-
outcomeType:
|
|
531
|
-
message:
|
|
513
|
+
error: 'observe_failed',
|
|
514
|
+
outcomeType: 'blocked',
|
|
515
|
+
message: 'Observe failed.',
|
|
532
516
|
reason: details,
|
|
533
517
|
pageRef,
|
|
534
518
|
runId: session.activeRunId,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"user-actionable.d.ts","sourceRoot":"","sources":["../../src/commands/user-actionable.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,eAAO,MAAM,4CAA4C,QAoHxD,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"user-actionable.d.ts","sourceRoot":"","sources":["../../src/commands/user-actionable.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAE/C,eAAO,MAAM,4CAA4C,QAoHxD,CAAC;AAEF,wBAAsB,uBAAuB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAoIhF"}
|
package/dist/protected-fill.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
import type { BrowserSessionState } from
|
|
2
|
-
import type { FillableFormFieldBinding, FillableFormPresence, ProtectedBindingValueHint, ProtectedFieldPolicy, StoredSecretFieldKey, StoredSecretFieldPolicies } from
|
|
3
|
-
import type { ProtectedExactValueProfile } from
|
|
1
|
+
import type { BrowserSessionState } from './browser-session-state.js';
|
|
2
|
+
import type { FillableFormFieldBinding, FillableFormPresence, ProtectedBindingValueHint, ProtectedFieldPolicy, StoredSecretFieldKey, StoredSecretFieldPolicies } from './secrets/types.js';
|
|
3
|
+
import type { ProtectedExactValueProfile } from './runtime-state.js';
|
|
4
4
|
/** Canonical stored protected field keys supported by AgentBrowse protected fill. */
|
|
5
5
|
export type ProtectedFieldKey = StoredSecretFieldKey;
|
|
6
6
|
export type ProtectedFieldPolicyMode = ProtectedFieldPolicy;
|
|
7
7
|
export type ProtectedFieldPolicies = StoredSecretFieldPolicies;
|
|
8
8
|
export type ProtectedFieldValueHint = ProtectedBindingValueHint;
|
|
9
9
|
export type ProtectedFillPresence = FillableFormPresence;
|
|
10
|
-
export type { ProtectedExactValueProfile } from
|
|
10
|
+
export type { ProtectedExactValueProfile } from './runtime-state.js';
|
|
11
11
|
/** Single field binding inside a protected fill form. */
|
|
12
12
|
export interface ProtectedFillFieldBinding extends FillableFormFieldBinding {
|
|
13
13
|
}
|
|
@@ -29,25 +29,25 @@ export interface ProtectedFilledField {
|
|
|
29
29
|
}
|
|
30
30
|
/** Field-level validation or application failure for protected fill. */
|
|
31
31
|
export interface ProtectedFieldError extends ProtectedFilledField {
|
|
32
|
-
reason:
|
|
32
|
+
reason: 'client_validation_rejected' | 'value_not_applied';
|
|
33
33
|
validationTextRedacted?: true;
|
|
34
34
|
}
|
|
35
35
|
export type ProtectedFillExecution = {
|
|
36
|
-
kind:
|
|
36
|
+
kind: 'success';
|
|
37
37
|
filledFields: ProtectedFilledField[];
|
|
38
38
|
} | {
|
|
39
|
-
kind:
|
|
39
|
+
kind: 'binding_stale';
|
|
40
40
|
targetRef: string;
|
|
41
41
|
fieldKeys: ProtectedFieldKey[];
|
|
42
|
-
reason:
|
|
42
|
+
reason: 'target_missing' | 'target_not_live' | 'page_signature_mismatch' | 'dom_signature_mismatch' | 'locator_resolution_failed' | 'target_blocked';
|
|
43
43
|
attempts: string[];
|
|
44
44
|
} | {
|
|
45
|
-
kind:
|
|
45
|
+
kind: 'validation_failed';
|
|
46
46
|
filledFields: ProtectedFilledField[];
|
|
47
47
|
fieldErrors: ProtectedFieldError[];
|
|
48
48
|
} | {
|
|
49
|
-
kind:
|
|
50
|
-
reason:
|
|
49
|
+
kind: 'unexpected_error';
|
|
50
|
+
reason: 'missing_protected_value' | 'unsupported_protected_field_group' | 'deterministic_only_resolution_failed' | 'assisted_value_resolution_failed' | 'action_failed';
|
|
51
51
|
};
|
|
52
52
|
export type FillProtectedFormResult = {
|
|
53
53
|
success: true;
|
|
@@ -57,7 +57,7 @@ export type FillProtectedFormResult = {
|
|
|
57
57
|
execution: ProtectedFillExecution;
|
|
58
58
|
} | {
|
|
59
59
|
success: false;
|
|
60
|
-
error:
|
|
60
|
+
error: 'browser_connection_failed' | 'page_resolution_failed';
|
|
61
61
|
message: string;
|
|
62
62
|
reason: string;
|
|
63
63
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"protected-fill.d.ts","sourceRoot":"","sources":["../src/protected-fill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAMtE,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAEpB,yBAAyB,EACzB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE,qFAAqF;AACrF,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC;AACrD,MAAM,MAAM,wBAAwB,GAAG,oBAAoB,CAAC;AAC5D,MAAM,MAAM,sBAAsB,GAAG,yBAAyB,CAAC;AAC/D,MAAM,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AAChE,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,CAAC;AACzD,YAAY,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE,yDAAyD;AACzD,MAAM,WAAW,yBAA0B,SAAQ,wBAAwB;CAAG;AAE9E,iGAAiG;AACjG,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wEAAwE;AACxE,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB;IAC/D,MAAM,EAAE,4BAA4B,GAAG,mBAAmB,CAAC;IAC3D,sBAAsB,CAAC,EAAE,IAAI,CAAC;CAC/B;AAED,MAAM,MAAM,sBAAsB,GAC9B;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,EAAE,oBAAoB,EAAE,CAAC;CACtC,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,MAAM,EACF,gBAAgB,GAChB,iBAAiB,GACjB,yBAAyB,GACzB,wBAAwB,GACxB,2BAA2B,GAC3B,gBAAgB,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,GACD;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,YAAY,EAAE,oBAAoB,EAAE,CAAC;IACrC,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC,GACD;IACE,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EACF,yBAAyB,GACzB,mCAAmC,GACnC,sCAAsC,GACtC,kCAAkC,GAClC,eAAe,CAAC;CACrB,CAAC;AAEN,MAAM,MAAM,uBAAuB,GAC/B;IACE,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,sBAAsB,CAAC;CACnC,GACD;IACE,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EAAE,2BAA2B,GAAG,wBAAwB,CAAC;IAC9D,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,aAAa,CAAC,EAAE,sBAAsB,CAAC;CACxC;AAED,MAAM,WAAW,oCAAoC;IACnD,OAAO,EAAE,mBAAmB,CAAC;IAC7B,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,aAAa,CAAC,EAAE,sBAAsB,CAAC;CACxC;AAED,MAAM,WAAW,0BAA0B;IACzC,iBAAiB,EAAE,0BAA0B,CAAC;CAC/C;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAOlC;AAED,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,oCAAoC,GAC3C,0BAA0B,
|
|
1
|
+
{"version":3,"file":"protected-fill.d.ts","sourceRoot":"","sources":["../src/protected-fill.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAMtE,OAAO,KAAK,EACV,wBAAwB,EACxB,oBAAoB,EAEpB,yBAAyB,EACzB,oBAAoB,EACpB,oBAAoB,EACpB,yBAAyB,EAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE,qFAAqF;AACrF,MAAM,MAAM,iBAAiB,GAAG,oBAAoB,CAAC;AACrD,MAAM,MAAM,wBAAwB,GAAG,oBAAoB,CAAC;AAC5D,MAAM,MAAM,sBAAsB,GAAG,yBAAyB,CAAC;AAC/D,MAAM,MAAM,uBAAuB,GAAG,yBAAyB,CAAC;AAChE,MAAM,MAAM,qBAAqB,GAAG,oBAAoB,CAAC;AACzD,YAAY,EAAE,0BAA0B,EAAE,MAAM,oBAAoB,CAAC;AAErE,yDAAyD;AACzD,MAAM,WAAW,yBAA0B,SAAQ,wBAAwB;CAAG;AAE9E,iGAAiG;AACjG,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,yBAAyB,EAAE,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,wEAAwE;AACxE,MAAM,WAAW,oBAAoB;IACnC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wEAAwE;AACxE,MAAM,WAAW,mBAAoB,SAAQ,oBAAoB;IAC/D,MAAM,EAAE,4BAA4B,GAAG,mBAAmB,CAAC;IAC3D,sBAAsB,CAAC,EAAE,IAAI,CAAC;CAC/B;AAED,MAAM,MAAM,sBAAsB,GAC9B;IACE,IAAI,EAAE,SAAS,CAAC;IAChB,YAAY,EAAE,oBAAoB,EAAE,CAAC;CACtC,GACD;IACE,IAAI,EAAE,eAAe,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,MAAM,EACF,gBAAgB,GAChB,iBAAiB,GACjB,yBAAyB,GACzB,wBAAwB,GACxB,2BAA2B,GAC3B,gBAAgB,CAAC;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB,GACD;IACE,IAAI,EAAE,mBAAmB,CAAC;IAC1B,YAAY,EAAE,oBAAoB,EAAE,CAAC;IACrC,WAAW,EAAE,mBAAmB,EAAE,CAAC;CACpC,GACD;IACE,IAAI,EAAE,kBAAkB,CAAC;IACzB,MAAM,EACF,yBAAyB,GACzB,mCAAmC,GACnC,sCAAsC,GACtC,kCAAkC,GAClC,eAAe,CAAC;CACrB,CAAC;AAEN,MAAM,MAAM,uBAAuB,GAC/B;IACE,OAAO,EAAE,IAAI,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,sBAAsB,CAAC;CACnC,GACD;IACE,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EAAE,2BAA2B,GAAG,wBAAwB,CAAC;IAC9D,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,mBAAmB,CAAC;IAC7B,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,aAAa,CAAC,EAAE,sBAAsB,CAAC;CACxC;AAED,MAAM,WAAW,oCAAoC;IACnD,OAAO,EAAE,mBAAmB,CAAC;IAC7B,YAAY,EAAE,iBAAiB,CAAC;IAChC,eAAe,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,aAAa,CAAC,EAAE,sBAAsB,CAAC;CACxC;AAED,MAAM,WAAW,0BAA0B;IACzC,iBAAiB,EAAE,0BAA0B,CAAC;CAC/C;AAED,wFAAwF;AACxF,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAOlC;AAED,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,oCAAoC,GAC3C,0BAA0B,CAW5B"}
|
package/dist/protected-fill.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { fillProtectedFormBrowser, } from
|
|
2
|
-
import { buildProtectedExactValueProfile as buildProtectedExactValueProfileInternal } from
|
|
1
|
+
import { fillProtectedFormBrowser, } from './protected-fill-browser.js';
|
|
2
|
+
import { buildProtectedExactValueProfile as buildProtectedExactValueProfileInternal } from './secrets/protected-exact-value-redaction.js';
|
|
3
3
|
/** Fills a previously observed protected form with caller-provided protected values. */
|
|
4
4
|
export async function fillProtectedForm(params) {
|
|
5
5
|
return (await fillProtectedFormBrowser({
|
|
@@ -12,7 +12,7 @@ export async function fillProtectedForm(params) {
|
|
|
12
12
|
export function buildProtectedExposureArtifacts(params) {
|
|
13
13
|
const filteredProtectedValues = Object.fromEntries(Object.entries(params.protectedValues).filter((entry) => {
|
|
14
14
|
const value = entry[1];
|
|
15
|
-
return typeof value ===
|
|
15
|
+
return typeof value === 'string' && value.length > 0;
|
|
16
16
|
}));
|
|
17
17
|
return {
|
|
18
18
|
exactValueProfile: buildProtectedExactValueProfileInternal(filteredProtectedValues),
|