@nuanu-ai/agentbrowse 0.2.47 → 0.2.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +81 -10
- package/dist/agentpay-gateway.d.ts +9 -0
- package/dist/agentpay-gateway.d.ts.map +1 -1
- package/dist/agentpay-gateway.js +31 -1
- package/dist/agentpay-stagehand-llm.d.ts.map +1 -1
- package/dist/agentpay-stagehand-llm.js +9 -1
- package/dist/command-api-tracing.d.ts +19 -0
- package/dist/command-api-tracing.d.ts.map +1 -0
- package/dist/command-api-tracing.js +137 -0
- package/dist/commands/act.d.ts.map +1 -1
- package/dist/commands/act.js +822 -670
- package/dist/commands/act.test-harness.d.ts +6 -0
- package/dist/commands/act.test-harness.d.ts.map +1 -1
- package/dist/commands/act.test-harness.js +44 -1
- package/dist/commands/action-acceptance.d.ts.map +1 -1
- package/dist/commands/action-acceptance.js +115 -0
- package/dist/commands/captcha-solve.d.ts.map +1 -1
- package/dist/commands/captcha-solve.js +83 -16
- package/dist/commands/click-action-executor.d.ts +0 -1
- package/dist/commands/click-action-executor.d.ts.map +1 -1
- package/dist/commands/click-action-executor.js +31 -77
- package/dist/commands/close.d.ts +3 -3
- package/dist/commands/close.d.ts.map +1 -1
- package/dist/commands/close.js +178 -0
- package/dist/commands/descriptor-validation.d.ts.map +1 -1
- package/dist/commands/descriptor-validation.js +75 -57
- package/dist/commands/end-session.d.ts +25 -0
- package/dist/commands/end-session.d.ts.map +1 -0
- package/dist/commands/end-session.js +161 -0
- package/dist/commands/extract-stagehand-executor.js +1 -1
- package/dist/commands/extract.d.ts.map +1 -1
- package/dist/commands/extract.js +339 -202
- package/dist/commands/fill-secret.d.ts +3 -3
- package/dist/commands/fill-secret.d.ts.map +1 -1
- package/dist/commands/fill-secret.js +419 -234
- package/dist/commands/get-secrets-catalog.d.ts.map +1 -1
- package/dist/commands/get-secrets-catalog.js +66 -5
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +6 -3
- package/dist/commands/interaction-kernel.d.ts +46 -0
- package/dist/commands/interaction-kernel.d.ts.map +1 -0
- package/dist/commands/interaction-kernel.js +215 -0
- package/dist/commands/launch.d.ts +1 -3
- package/dist/commands/launch.d.ts.map +1 -1
- package/dist/commands/launch.js +115 -27
- package/dist/commands/navigate.d.ts.map +1 -1
- package/dist/commands/navigate.js +188 -45
- package/dist/commands/observe-accessibility.d.ts.map +1 -1
- package/dist/commands/observe-accessibility.js +46 -39
- package/dist/commands/observe-dom-label-contract.d.ts.map +1 -1
- package/dist/commands/observe-dom-label-contract.js +5 -0
- package/dist/commands/observe-inventory.d.ts +13 -0
- package/dist/commands/observe-inventory.d.ts.map +1 -1
- package/dist/commands/observe-inventory.js +320 -65
- package/dist/commands/observe-persistence.d.ts.map +1 -1
- package/dist/commands/observe-persistence.js +3 -0
- package/dist/commands/observe-projection.d.ts +1 -0
- package/dist/commands/observe-projection.d.ts.map +1 -1
- package/dist/commands/observe-projection.js +7 -2
- package/dist/commands/observe-protected.d.ts +1 -0
- package/dist/commands/observe-protected.d.ts.map +1 -1
- package/dist/commands/observe-protected.js +9 -4
- package/dist/commands/observe-semantics.d.ts.map +1 -1
- package/dist/commands/observe-semantics.js +5 -2
- package/dist/commands/observe-stagehand.d.ts +1 -0
- package/dist/commands/observe-stagehand.d.ts.map +1 -1
- package/dist/commands/observe-stagehand.js +2 -0
- package/dist/commands/observe.d.ts +2 -0
- package/dist/commands/observe.d.ts.map +1 -1
- package/dist/commands/observe.js +387 -203
- package/dist/commands/observe.test-harness.d.ts +8 -0
- package/dist/commands/observe.test-harness.d.ts.map +1 -1
- package/dist/commands/observe.test-harness.js +48 -1
- package/dist/commands/poll-secret.d.ts +6 -0
- package/dist/commands/poll-secret.d.ts.map +1 -0
- package/dist/commands/poll-secret.js +159 -0
- package/dist/commands/request-secret.d.ts +6 -0
- package/dist/commands/request-secret.d.ts.map +1 -0
- package/dist/commands/request-secret.js +284 -0
- package/dist/commands/screenshot.d.ts.map +1 -1
- package/dist/commands/screenshot.js +172 -7
- package/dist/commands/select-action-executor.d.ts.map +1 -1
- package/dist/commands/semantic-observe.d.ts +4 -0
- package/dist/commands/semantic-observe.d.ts.map +1 -1
- package/dist/commands/semantic-observe.js +388 -17
- package/dist/commands/start-session.d.ts +31 -0
- package/dist/commands/start-session.d.ts.map +1 -0
- package/dist/commands/start-session.js +347 -0
- package/dist/commands/status.d.ts +2 -1
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +166 -144
- package/dist/control-semantics.d.ts +1 -0
- package/dist/control-semantics.d.ts.map +1 -1
- package/dist/control-semantics.js +51 -9
- package/dist/generated/build-config.d.ts +2 -0
- package/dist/generated/build-config.d.ts.map +1 -0
- package/dist/generated/build-config.js +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +163 -63
- package/dist/otel-exporter.d.ts +58 -0
- package/dist/otel-exporter.d.ts.map +1 -0
- package/dist/otel-exporter.js +263 -0
- package/dist/otel-projector.d.ts +75 -0
- package/dist/otel-projector.d.ts.map +1 -0
- package/dist/otel-projector.js +409 -0
- package/dist/owned-browser.d.ts +1 -1
- package/dist/owned-browser.d.ts.map +1 -1
- package/dist/owned-browser.js +13 -1
- package/dist/owned-process.d.ts +2 -0
- package/dist/owned-process.d.ts.map +1 -1
- package/dist/owned-process.js +7 -3
- package/dist/playwright-runtime.d.ts +1 -1
- package/dist/playwright-runtime.d.ts.map +1 -1
- package/dist/playwright-runtime.js +8 -8
- package/dist/run-observability.d.ts +25 -0
- package/dist/run-observability.d.ts.map +1 -0
- package/dist/run-observability.js +115 -0
- package/dist/run-store.d.ts +274 -0
- package/dist/run-store.d.ts.map +1 -0
- package/dist/run-store.js +631 -0
- package/dist/runtime-metrics.d.ts +27 -0
- package/dist/runtime-metrics.d.ts.map +1 -0
- package/dist/runtime-metrics.js +66 -0
- package/dist/runtime-page-state.d.ts +11 -0
- package/dist/runtime-page-state.d.ts.map +1 -0
- package/dist/runtime-page-state.js +62 -0
- package/dist/runtime-protected-state.d.ts +16 -0
- package/dist/runtime-protected-state.d.ts.map +1 -0
- package/dist/runtime-protected-state.js +157 -0
- package/dist/runtime-state.d.ts +10 -44
- package/dist/runtime-state.d.ts.map +1 -1
- package/dist/runtime-state.js +57 -222
- package/dist/secrets/backend.d.ts +65 -16
- package/dist/secrets/backend.d.ts.map +1 -1
- package/dist/secrets/backend.js +135 -95
- package/dist/secrets/catalog-sync.d.ts.map +1 -1
- package/dist/secrets/catalog-sync.js +4 -1
- package/dist/secrets/form-matcher.d.ts +5 -5
- package/dist/secrets/form-matcher.d.ts.map +1 -1
- package/dist/secrets/form-matcher.js +292 -164
- package/dist/secrets/intent-output.d.ts +6 -10
- package/dist/secrets/intent-output.d.ts.map +1 -1
- package/dist/secrets/intent-output.js +4 -58
- package/dist/secrets/mock-agentpay-cabinet.d.ts +38 -27
- package/dist/secrets/mock-agentpay-cabinet.d.ts.map +1 -1
- package/dist/secrets/mock-agentpay-cabinet.js +177 -111
- package/dist/secrets/protected-artifact-guard.d.ts +2 -2
- package/dist/secrets/protected-artifact-guard.d.ts.map +1 -1
- package/dist/secrets/protected-artifact-guard.js +2 -2
- package/dist/secrets/protected-bindings.d.ts +1 -1
- package/dist/secrets/protected-bindings.d.ts.map +1 -1
- package/dist/secrets/protected-bindings.js +6 -0
- package/dist/secrets/protected-field-semantics.d.ts +9 -0
- package/dist/secrets/protected-field-semantics.d.ts.map +1 -0
- package/dist/secrets/protected-field-semantics.js +154 -0
- package/dist/secrets/protected-field-values.d.ts.map +1 -1
- package/dist/secrets/protected-field-values.js +3 -3
- package/dist/secrets/protected-fill.d.ts +1 -1
- package/dist/secrets/protected-fill.d.ts.map +1 -1
- package/dist/secrets/protected-fill.js +45 -149
- package/dist/secrets/protected-value-adapters.d.ts +2 -1
- package/dist/secrets/protected-value-adapters.d.ts.map +1 -1
- package/dist/secrets/protected-value-adapters.js +80 -1
- package/dist/secrets/request-output.d.ts +11 -0
- package/dist/secrets/request-output.d.ts.map +1 -0
- package/dist/secrets/request-output.js +75 -0
- package/dist/secrets/types.d.ts +15 -9
- package/dist/secrets/types.d.ts.map +1 -1
- package/dist/secrets/types.js +3 -0
- package/dist/session-event-exporter.d.ts +36 -0
- package/dist/session-event-exporter.d.ts.map +1 -0
- package/dist/session-event-exporter.js +428 -0
- package/dist/session.d.ts +16 -7
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +150 -23
- package/dist/sessions-backend.d.ts +354 -0
- package/dist/sessions-backend.d.ts.map +1 -0
- package/dist/sessions-backend.js +126 -0
- package/dist/solver/browser-launcher.d.ts +1 -1
- package/dist/solver/browser-launcher.d.ts.map +1 -1
- package/dist/solver/browser-launcher.js +39 -13
- package/dist/solver/captcha-solver.d.ts.map +1 -1
- package/dist/solver/captcha-solver.js +8 -1
- package/dist/solver/config.d.ts +0 -1
- package/dist/solver/config.d.ts.map +1 -1
- package/dist/solver/config.js +0 -22
- package/dist/solver/types.d.ts +1 -0
- package/dist/solver/types.d.ts.map +1 -1
- package/dist/workflow-session-completion.d.ts +33 -0
- package/dist/workflow-session-completion.d.ts.map +1 -0
- package/dist/workflow-session-completion.js +156 -0
- package/package.json +11 -1
- package/dist/commands/create-intent.d.ts +0 -6
- package/dist/commands/create-intent.d.ts.map +0 -1
- package/dist/commands/create-intent.js +0 -75
- package/dist/commands/poll-intent.d.ts +0 -6
- package/dist/commands/poll-intent.d.ts.map +0 -1
- package/dist/commands/poll-intent.js +0 -57
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { z } from 'zod';
|
|
2
3
|
import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
|
|
3
4
|
import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
|
|
4
|
-
import { recordLlmUsage, recordPayloadBudget } from '../runtime-
|
|
5
|
+
import { recordLlmUsage, recordPayloadBudget } from '../runtime-metrics.js';
|
|
5
6
|
import { analyzeSemanticObserveText, buildSemanticObserveBm25CorpusStats, buildSemanticObserveLexicalDocument, normalizeSemanticObserveText, scoreSemanticObserveBm25, } from './semantic-observe-lexical.js';
|
|
6
7
|
const rerankSchema = z.object({
|
|
7
8
|
matches: z
|
|
@@ -55,6 +56,12 @@ const HIGH_SIGNAL_SCOPE_KINDS = new Set([
|
|
|
55
56
|
'card',
|
|
56
57
|
'form',
|
|
57
58
|
]);
|
|
59
|
+
const SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION = 1;
|
|
60
|
+
const SEMANTIC_OBSERVE_SNAPSHOT_CACHE_KEY_VERSION = 'semantic-observe-snapshot-v1';
|
|
61
|
+
const SEMANTIC_OBSERVE_GOAL_CACHE_KEY_VERSION = 'semantic-observe-goal-v1';
|
|
62
|
+
const SEMANTIC_OBSERVE_MAX_SNAPSHOT_CACHE_ENTRIES = 4;
|
|
63
|
+
const SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT = 6;
|
|
64
|
+
const semanticObserveSnapshotCacheBySession = new WeakMap();
|
|
58
65
|
function isFieldLikeTarget(target) {
|
|
59
66
|
const kind = (target.kind ?? '').trim().toLowerCase();
|
|
60
67
|
const role = (target.role ?? '').trim().toLowerCase();
|
|
@@ -135,12 +142,31 @@ function isPrimaryFormControlTarget(target) {
|
|
|
135
142
|
acceptancePolicy === 'disclosure' &&
|
|
136
143
|
(kind === 'button' || role === 'button'));
|
|
137
144
|
}
|
|
145
|
+
function isConcreteFormChoiceTarget(target) {
|
|
146
|
+
if (!Boolean(formBucketKey(target)) ||
|
|
147
|
+
isScopeLikeCandidate(target) ||
|
|
148
|
+
isFieldLikeTarget(target) ||
|
|
149
|
+
!isActionLikeTargetCandidate(target) ||
|
|
150
|
+
isPrimaryFormControlTarget(target)) {
|
|
151
|
+
return false;
|
|
152
|
+
}
|
|
153
|
+
const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
|
|
154
|
+
const controlFamily = (target.controlFamily ?? '').trim().toLowerCase();
|
|
155
|
+
const surfaceKind = (target.surfaceKind ?? '').trim().toLowerCase();
|
|
156
|
+
const hasGoalIdentity = Boolean(goalLabelOf(target));
|
|
157
|
+
if (!hasGoalIdentity) {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
return (controlFamily === 'trigger' ||
|
|
161
|
+
acceptancePolicy === 'selection' ||
|
|
162
|
+
['listbox', 'menu', 'dropdown', 'popover'].includes(surfaceKind));
|
|
163
|
+
}
|
|
138
164
|
function shouldPreserveStandaloneFormActionEntity(target) {
|
|
139
165
|
return (Boolean(formBucketKey(target)) &&
|
|
140
166
|
!isScopeLikeCandidate(target) &&
|
|
141
167
|
!isFieldLikeTarget(target) &&
|
|
142
168
|
isActionLikeTargetCandidate(target) &&
|
|
143
|
-
isPrimaryFormControlTarget(target));
|
|
169
|
+
(isPrimaryFormControlTarget(target) || isConcreteFormChoiceTarget(target)));
|
|
144
170
|
}
|
|
145
171
|
function primaryFormTargetPriority(target) {
|
|
146
172
|
const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
|
|
@@ -190,6 +216,322 @@ function contentItemBucketKey(target) {
|
|
|
190
216
|
}
|
|
191
217
|
return undefined;
|
|
192
218
|
}
|
|
219
|
+
function cachedContextNodeValue(node) {
|
|
220
|
+
if (!node) {
|
|
221
|
+
return undefined;
|
|
222
|
+
}
|
|
223
|
+
return {
|
|
224
|
+
...(node.kind ? { kind: node.kind } : {}),
|
|
225
|
+
...(node.label ? { label: node.label } : {}),
|
|
226
|
+
...(node.text ? { text: node.text } : {}),
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
function cachedStateEntries(states) {
|
|
230
|
+
if (!states) {
|
|
231
|
+
return undefined;
|
|
232
|
+
}
|
|
233
|
+
const entries = Object.entries(states).sort(([left], [right]) => left.localeCompare(right));
|
|
234
|
+
return entries.length > 0 ? entries : undefined;
|
|
235
|
+
}
|
|
236
|
+
function cachedStructureValue(structure) {
|
|
237
|
+
if (!structure) {
|
|
238
|
+
return undefined;
|
|
239
|
+
}
|
|
240
|
+
return {
|
|
241
|
+
...(structure.family ? { family: structure.family } : {}),
|
|
242
|
+
...(structure.variant ? { variant: structure.variant } : {}),
|
|
243
|
+
...(structure.row ? { row: structure.row } : {}),
|
|
244
|
+
...(structure.column ? { column: structure.column } : {}),
|
|
245
|
+
...(structure.zone ? { zone: structure.zone } : {}),
|
|
246
|
+
...(structure.cellLabel ? { cellLabel: structure.cellLabel } : {}),
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
function normalizeGoalIdentityValue(value, maxLength = 180) {
|
|
250
|
+
const normalized = value?.replace(/\s+/g, ' ').trim();
|
|
251
|
+
if (!normalized) {
|
|
252
|
+
return undefined;
|
|
253
|
+
}
|
|
254
|
+
return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 1)}…` : normalized;
|
|
255
|
+
}
|
|
256
|
+
function appendGoalIdentityAlias(values, value) {
|
|
257
|
+
const normalized = normalizeGoalIdentityValue(value);
|
|
258
|
+
if (!normalized) {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
const normalizedKey = normalized.toLowerCase();
|
|
262
|
+
if (values.some((existing) => existing.toLowerCase() === normalizedKey)) {
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
values.push(normalized);
|
|
266
|
+
}
|
|
267
|
+
function goalIdentityValuesOf(target) {
|
|
268
|
+
const values = [];
|
|
269
|
+
appendGoalIdentityAlias(values, target.goalLabel);
|
|
270
|
+
for (const value of target.goalAliases ?? []) {
|
|
271
|
+
appendGoalIdentityAlias(values, value);
|
|
272
|
+
}
|
|
273
|
+
if (values.length > 0) {
|
|
274
|
+
return values;
|
|
275
|
+
}
|
|
276
|
+
if (isFieldLikeTarget(target)) {
|
|
277
|
+
appendGoalIdentityAlias(values, target.label);
|
|
278
|
+
appendGoalIdentityAlias(values, target.displayLabel);
|
|
279
|
+
appendGoalIdentityAlias(values, target.placeholder);
|
|
280
|
+
appendGoalIdentityAlias(values, target.title);
|
|
281
|
+
return values;
|
|
282
|
+
}
|
|
283
|
+
appendGoalIdentityAlias(values, target.label);
|
|
284
|
+
appendGoalIdentityAlias(values, target.displayLabel);
|
|
285
|
+
appendGoalIdentityAlias(values, target.text);
|
|
286
|
+
appendGoalIdentityAlias(values, target.title);
|
|
287
|
+
return values;
|
|
288
|
+
}
|
|
289
|
+
function goalLabelOf(target) {
|
|
290
|
+
return goalIdentityValuesOf(target)[0];
|
|
291
|
+
}
|
|
292
|
+
function goalAliasesOf(target) {
|
|
293
|
+
return goalIdentityValuesOf(target).slice(1, 5);
|
|
294
|
+
}
|
|
295
|
+
function serializeObserveSnapshotTarget(target) {
|
|
296
|
+
const itemContext = cachedContextNodeValue(target.context?.item);
|
|
297
|
+
const groupContext = cachedContextNodeValue(target.context?.group);
|
|
298
|
+
const containerContext = cachedContextNodeValue(target.context?.container);
|
|
299
|
+
const landmarkContext = cachedContextNodeValue(target.context?.landmark);
|
|
300
|
+
const stateEntries = cachedStateEntries(target.states);
|
|
301
|
+
const structure = cachedStructureValue(target.structure);
|
|
302
|
+
const goalLabel = goalLabelOf(target);
|
|
303
|
+
const goalAliases = goalAliasesOf(target);
|
|
304
|
+
return {
|
|
305
|
+
goalInventoryType: target.goalInventoryType,
|
|
306
|
+
...(target.goalInventoryType === 'scope' && target.goalSurfaceId
|
|
307
|
+
? { goalSurfaceId: target.goalSurfaceId }
|
|
308
|
+
: {}),
|
|
309
|
+
...(goalLabel ? { goalLabel } : {}),
|
|
310
|
+
...(goalAliases.length ? { goalAliases } : {}),
|
|
311
|
+
...(target.kind ? { kind: target.kind } : {}),
|
|
312
|
+
...(target.label ? { label: target.label } : {}),
|
|
313
|
+
...(target.displayLabel ? { displayLabel: target.displayLabel } : {}),
|
|
314
|
+
...(target.role ? { role: target.role } : {}),
|
|
315
|
+
...(target.interactionHint ? { interactionHint: target.interactionHint } : {}),
|
|
316
|
+
...(target.text ? { text: target.text } : {}),
|
|
317
|
+
...(target.placeholder ? { placeholder: target.placeholder } : {}),
|
|
318
|
+
...(target.title ? { title: target.title } : {}),
|
|
319
|
+
...(target.capability ? { capability: target.capability } : {}),
|
|
320
|
+
...(target.allowedActions?.length
|
|
321
|
+
? { allowedActions: [...target.allowedActions].sort() }
|
|
322
|
+
: {}),
|
|
323
|
+
...(target.acceptancePolicy ? { acceptancePolicy: target.acceptancePolicy } : {}),
|
|
324
|
+
...(target.controlFamily ? { controlFamily: target.controlFamily } : {}),
|
|
325
|
+
...(target.surfaceRef ? { surfaceRef: target.surfaceRef } : {}),
|
|
326
|
+
...(target.surfaceKind ? { surfaceKind: target.surfaceKind } : {}),
|
|
327
|
+
...(target.surfaceLabel ? { surfaceLabel: target.surfaceLabel } : {}),
|
|
328
|
+
...(typeof target.surfacePriority === 'number' ? { surfacePriority: target.surfacePriority } : {}),
|
|
329
|
+
...(target.framePath?.length ? { framePath: [...target.framePath] } : {}),
|
|
330
|
+
...(target.frameUrl ? { frameUrl: target.frameUrl } : {}),
|
|
331
|
+
...(target.formSelector ? { formSelector: target.formSelector } : {}),
|
|
332
|
+
...(target.pageSignature ? { pageSignature: target.pageSignature } : {}),
|
|
333
|
+
...(target.controlsSurfaceSelector
|
|
334
|
+
? { controlsSurfaceSelector: target.controlsSurfaceSelector }
|
|
335
|
+
: {}),
|
|
336
|
+
...(stateEntries ? { states: stateEntries } : {}),
|
|
337
|
+
...(target.context
|
|
338
|
+
? {
|
|
339
|
+
context: {
|
|
340
|
+
...(itemContext ? { item: itemContext } : {}),
|
|
341
|
+
...(groupContext ? { group: groupContext } : {}),
|
|
342
|
+
...(containerContext ? { container: containerContext } : {}),
|
|
343
|
+
...(landmarkContext ? { landmark: landmarkContext } : {}),
|
|
344
|
+
...(target.context.hintText ? { hintText: target.context.hintText } : {}),
|
|
345
|
+
},
|
|
346
|
+
}
|
|
347
|
+
: {}),
|
|
348
|
+
...(structure ? { structure } : {}),
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
function semanticObserveSnapshotKey(targets) {
|
|
352
|
+
const payload = JSON.stringify(targets.map((target) => serializeObserveSnapshotTarget(target)));
|
|
353
|
+
return createHash('sha256')
|
|
354
|
+
.update(SEMANTIC_OBSERVE_SNAPSHOT_CACHE_KEY_VERSION)
|
|
355
|
+
.update(payload)
|
|
356
|
+
.digest('hex');
|
|
357
|
+
}
|
|
358
|
+
function semanticObserveGoalKey(goal) {
|
|
359
|
+
return createHash('sha256')
|
|
360
|
+
.update(SEMANTIC_OBSERVE_GOAL_CACHE_KEY_VERSION)
|
|
361
|
+
.update(normalizedGoalText(goal))
|
|
362
|
+
.digest('hex');
|
|
363
|
+
}
|
|
364
|
+
function serializeLexicalDocument(lexicalDocument) {
|
|
365
|
+
return {
|
|
366
|
+
analyzerKey: lexicalDocument.analyzerKey,
|
|
367
|
+
weightedLength: lexicalDocument.weightedLength,
|
|
368
|
+
weightedTermFrequencies: [...lexicalDocument.weightedTermFrequencies.entries()],
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
function deserializeLexicalDocument(lexicalDocument) {
|
|
372
|
+
return {
|
|
373
|
+
analyzerKey: lexicalDocument.analyzerKey,
|
|
374
|
+
weightedLength: lexicalDocument.weightedLength,
|
|
375
|
+
weightedTermFrequencies: new Map(lexicalDocument.weightedTermFrequencies),
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
function ensureSemanticObserveSnapshotCache(session) {
|
|
379
|
+
const existing = semanticObserveSnapshotCacheBySession.get(session);
|
|
380
|
+
if (existing &&
|
|
381
|
+
existing.version === SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION &&
|
|
382
|
+
Array.isArray(existing.order) &&
|
|
383
|
+
existing.snapshotsByKey &&
|
|
384
|
+
typeof existing.snapshotsByKey === 'object') {
|
|
385
|
+
return existing;
|
|
386
|
+
}
|
|
387
|
+
const nextCache = {
|
|
388
|
+
version: SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION,
|
|
389
|
+
order: [],
|
|
390
|
+
snapshotsByKey: {},
|
|
391
|
+
};
|
|
392
|
+
semanticObserveSnapshotCacheBySession.set(session, nextCache);
|
|
393
|
+
return nextCache;
|
|
394
|
+
}
|
|
395
|
+
function touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey) {
|
|
396
|
+
cache.order = [snapshotKey, ...cache.order.filter((key) => key !== snapshotKey)].slice(0, SEMANTIC_OBSERVE_MAX_SNAPSHOT_CACHE_ENTRIES);
|
|
397
|
+
for (const key of Object.keys(cache.snapshotsByKey)) {
|
|
398
|
+
if (!cache.order.includes(key)) {
|
|
399
|
+
delete cache.snapshotsByKey[key];
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
function materializeCachedRetrievalEntities(targets, cachedEntities) {
|
|
404
|
+
const materialized = [];
|
|
405
|
+
for (const cachedEntity of cachedEntities) {
|
|
406
|
+
const representative = targets[cachedEntity.representativeIndex];
|
|
407
|
+
if (!representative) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
if (cachedEntity.memberIndexes.some((index) => !targets[index])) {
|
|
411
|
+
return null;
|
|
412
|
+
}
|
|
413
|
+
materialized.push({
|
|
414
|
+
entityKind: cachedEntity.entityKind,
|
|
415
|
+
entityKey: cachedEntity.entityKey,
|
|
416
|
+
firstIndex: cachedEntity.firstIndex,
|
|
417
|
+
memberIndexes: [...cachedEntity.memberIndexes],
|
|
418
|
+
representative,
|
|
419
|
+
label: pickEntityLabel(cachedEntity.entityKind, representative),
|
|
420
|
+
kind: cachedEntity.entityKind === 'form'
|
|
421
|
+
? 'form'
|
|
422
|
+
: cachedEntity.entityKind === 'item'
|
|
423
|
+
? (representative.surfaceKind ?? representative.kind)
|
|
424
|
+
: representative.kind,
|
|
425
|
+
surfaceKind: representative.surfaceKind,
|
|
426
|
+
surfaceLabel: representative.surfaceLabel,
|
|
427
|
+
surfacePriority: representative.surfacePriority,
|
|
428
|
+
framePath: representative.framePath,
|
|
429
|
+
frameUrl: representative.frameUrl,
|
|
430
|
+
context: representative.context,
|
|
431
|
+
structure: representative.structure,
|
|
432
|
+
representativeLabels: [],
|
|
433
|
+
analyzerKey: cachedEntity.analyzerKey,
|
|
434
|
+
lexicalDocument: deserializeLexicalDocument(cachedEntity.lexicalDocument),
|
|
435
|
+
});
|
|
436
|
+
}
|
|
437
|
+
return materialized;
|
|
438
|
+
}
|
|
439
|
+
function loadCachedRetrievalEntities(session, snapshotKey, targets) {
|
|
440
|
+
if (!session) {
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
const cache = ensureSemanticObserveSnapshotCache(session);
|
|
444
|
+
const entry = cache.snapshotsByKey[snapshotKey];
|
|
445
|
+
if (!entry || entry.targetCount !== targets.length || !entry.retrievalEntities) {
|
|
446
|
+
return null;
|
|
447
|
+
}
|
|
448
|
+
const materialized = materializeCachedRetrievalEntities(targets, entry.retrievalEntities);
|
|
449
|
+
if (!materialized) {
|
|
450
|
+
delete cache.snapshotsByKey[snapshotKey];
|
|
451
|
+
cache.order = cache.order.filter((key) => key !== snapshotKey);
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
|
|
455
|
+
return materialized;
|
|
456
|
+
}
|
|
457
|
+
function saveCachedRetrievalEntities(session, snapshotKey, targets, entities) {
|
|
458
|
+
if (!session) {
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
const cache = ensureSemanticObserveSnapshotCache(session);
|
|
462
|
+
const indexByTarget = new Map(targets.map((target, index) => [target, index]));
|
|
463
|
+
const existingEntry = cache.snapshotsByKey[snapshotKey];
|
|
464
|
+
cache.snapshotsByKey[snapshotKey] = {
|
|
465
|
+
snapshotKey,
|
|
466
|
+
targetCount: targets.length,
|
|
467
|
+
cachedAt: new Date().toISOString(),
|
|
468
|
+
retrievalEntities: entities.map((entity) => ({
|
|
469
|
+
entityKind: entity.entityKind,
|
|
470
|
+
entityKey: entity.entityKey,
|
|
471
|
+
firstIndex: entity.firstIndex,
|
|
472
|
+
representativeIndex: indexByTarget.get(entity.representative) ?? entity.firstIndex,
|
|
473
|
+
memberIndexes: [...entity.memberIndexes],
|
|
474
|
+
analyzerKey: entity.analyzerKey,
|
|
475
|
+
lexicalDocument: serializeLexicalDocument(entity.lexicalDocument),
|
|
476
|
+
})),
|
|
477
|
+
goalShortlists: existingEntry?.goalShortlists ?? {},
|
|
478
|
+
goalOrder: existingEntry?.goalOrder ?? [],
|
|
479
|
+
};
|
|
480
|
+
touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
|
|
481
|
+
}
|
|
482
|
+
function loadCachedGoalShortlist(session, snapshotKey, goalKey, targets) {
|
|
483
|
+
if (!session || !goalKey) {
|
|
484
|
+
return null;
|
|
485
|
+
}
|
|
486
|
+
const cache = ensureSemanticObserveSnapshotCache(session);
|
|
487
|
+
const entry = cache.snapshotsByKey[snapshotKey];
|
|
488
|
+
const shortlist = entry?.goalShortlists?.[goalKey];
|
|
489
|
+
if (!entry || entry.targetCount !== targets.length || !shortlist) {
|
|
490
|
+
return null;
|
|
491
|
+
}
|
|
492
|
+
const candidates = shortlist.candidateIndexes.map((index) => targets[index]).filter(Boolean);
|
|
493
|
+
if (candidates.length !== shortlist.candidateIndexes.length) {
|
|
494
|
+
delete entry.goalShortlists?.[goalKey];
|
|
495
|
+
entry.goalOrder = (entry.goalOrder ?? []).filter((key) => key !== goalKey);
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
entry.goalOrder = [goalKey, ...(entry.goalOrder ?? []).filter((key) => key !== goalKey)].slice(0, SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT);
|
|
499
|
+
touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
|
|
500
|
+
return candidates;
|
|
501
|
+
}
|
|
502
|
+
function saveCachedGoalShortlist(session, snapshotKey, goalKey, targets, shortlist) {
|
|
503
|
+
if (!session || !goalKey) {
|
|
504
|
+
return;
|
|
505
|
+
}
|
|
506
|
+
const cache = ensureSemanticObserveSnapshotCache(session);
|
|
507
|
+
const existingEntry = cache.snapshotsByKey[snapshotKey];
|
|
508
|
+
const entry = existingEntry ?? {
|
|
509
|
+
snapshotKey,
|
|
510
|
+
targetCount: targets.length,
|
|
511
|
+
cachedAt: new Date().toISOString(),
|
|
512
|
+
goalShortlists: {},
|
|
513
|
+
goalOrder: [],
|
|
514
|
+
};
|
|
515
|
+
const indexByTarget = new Map(targets.map((target, index) => [target, index]));
|
|
516
|
+
entry.targetCount = targets.length;
|
|
517
|
+
entry.cachedAt = new Date().toISOString();
|
|
518
|
+
entry.goalShortlists ??= {};
|
|
519
|
+
entry.goalOrder = [goalKey, ...(entry.goalOrder ?? []).filter((key) => key !== goalKey)].slice(0, SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT);
|
|
520
|
+
entry.goalShortlists[goalKey] = {
|
|
521
|
+
goalKey,
|
|
522
|
+
candidateIndexes: shortlist
|
|
523
|
+
.map((candidate) => indexByTarget.get(candidate))
|
|
524
|
+
.filter((index) => index !== undefined),
|
|
525
|
+
cachedAt: entry.cachedAt,
|
|
526
|
+
};
|
|
527
|
+
for (const cachedGoalKey of Object.keys(entry.goalShortlists)) {
|
|
528
|
+
if (!entry.goalOrder.includes(cachedGoalKey)) {
|
|
529
|
+
delete entry.goalShortlists[cachedGoalKey];
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
cache.snapshotsByKey[snapshotKey] = entry;
|
|
533
|
+
touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
|
|
534
|
+
}
|
|
193
535
|
function scopeCandidatePriority(target) {
|
|
194
536
|
const kind = (target.kind ?? '').trim().toLowerCase();
|
|
195
537
|
if (kind === 'dialog' || kind === 'listbox' || kind === 'menu') {
|
|
@@ -276,7 +618,7 @@ function pickEntityLabel(entityKind, representative) {
|
|
|
276
618
|
representative.surfaceLabel ??
|
|
277
619
|
representative.label);
|
|
278
620
|
}
|
|
279
|
-
return (representative
|
|
621
|
+
return (goalLabelOf(representative) ??
|
|
280
622
|
representative.context?.container?.label ??
|
|
281
623
|
representative.context?.item?.label ??
|
|
282
624
|
representative.surfaceLabel ??
|
|
@@ -286,13 +628,14 @@ function pickEntityLabel(entityKind, representative) {
|
|
|
286
628
|
function collectRepresentativeLabels(targets) {
|
|
287
629
|
const labels = [];
|
|
288
630
|
for (const target of targets) {
|
|
289
|
-
const label
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
631
|
+
for (const label of [goalLabelOf(target), ...goalAliasesOf(target)]) {
|
|
632
|
+
if (!label || label === 'Button' || label === 'Link' || labels.includes(label)) {
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
635
|
+
labels.push(label);
|
|
636
|
+
if (labels.length >= 4) {
|
|
637
|
+
return labels;
|
|
638
|
+
}
|
|
296
639
|
}
|
|
297
640
|
}
|
|
298
641
|
return labels;
|
|
@@ -368,9 +711,10 @@ function latentActionHintText(target) {
|
|
|
368
711
|
return hints.size > 0 ? [...hints].join(' ') : undefined;
|
|
369
712
|
}
|
|
370
713
|
function buildEntityLexicalFields(entityLabel, representative, representativeLabels) {
|
|
714
|
+
const representativeGoalLabel = goalLabelOf(representative);
|
|
371
715
|
return [
|
|
372
716
|
{ value: entityLabel, weight: RETRIEVAL_FIELD_WEIGHTS.entityLabel },
|
|
373
|
-
{ value:
|
|
717
|
+
{ value: representativeGoalLabel, weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabel },
|
|
374
718
|
...representativeLabels.map((value) => ({
|
|
375
719
|
value,
|
|
376
720
|
weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabels,
|
|
@@ -857,6 +1201,8 @@ function buildSurfaceSummaries(targets) {
|
|
|
857
1201
|
function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
|
|
858
1202
|
const parts = [`id=c${index + 1}`];
|
|
859
1203
|
const compactContext = compactCandidateContext(target);
|
|
1204
|
+
const goalLabel = goalLabelOf(target);
|
|
1205
|
+
const goalAliases = goalAliasesOf(target).map((value) => compactContextValue(value, 120) ?? value);
|
|
860
1206
|
const surfaceSummaryId = surfaceIdentityOf(target)
|
|
861
1207
|
? surfaceSummaryIdBySurfaceKey?.get(surfaceIdentityOf(target))
|
|
862
1208
|
: undefined;
|
|
@@ -864,8 +1210,10 @@ function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKe
|
|
|
864
1210
|
parts.push(`kind=${JSON.stringify(target.kind)}`);
|
|
865
1211
|
if (target.role)
|
|
866
1212
|
parts.push(`role=${JSON.stringify(target.role)}`);
|
|
867
|
-
if (
|
|
868
|
-
parts.push(`label=${JSON.stringify(
|
|
1213
|
+
if (goalLabel)
|
|
1214
|
+
parts.push(`label=${JSON.stringify(goalLabel)}`);
|
|
1215
|
+
if (goalAliases.length)
|
|
1216
|
+
parts.push(`aliases=${JSON.stringify(goalAliases)}`);
|
|
869
1217
|
if (target.allowedActions?.length) {
|
|
870
1218
|
parts.push(`actions=${JSON.stringify(target.allowedActions)}`);
|
|
871
1219
|
}
|
|
@@ -910,11 +1258,34 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
|
|
|
910
1258
|
});
|
|
911
1259
|
const gateway = resolveAgentpayGatewayConfig();
|
|
912
1260
|
const client = new AgentpayStagehandLlmClient(gateway);
|
|
913
|
-
const
|
|
914
|
-
const
|
|
915
|
-
|
|
916
|
-
|
|
1261
|
+
const snapshotKey = options.session ? semanticObserveSnapshotKey(relevantTargets) : undefined;
|
|
1262
|
+
const goalKey = snapshotKey ? semanticObserveGoalKey(instruction) : undefined;
|
|
1263
|
+
const cachedCandidates = snapshotKey && goalKey
|
|
1264
|
+
? loadCachedGoalShortlist(options.session, snapshotKey, goalKey, relevantTargets)
|
|
1265
|
+
: null;
|
|
1266
|
+
let retrievedCandidates;
|
|
1267
|
+
if (cachedCandidates) {
|
|
1268
|
+
retrievedCandidates = cachedCandidates;
|
|
1269
|
+
}
|
|
1270
|
+
else if (relevantTargets.length > RERANK_CANDIDATE_LIMIT) {
|
|
1271
|
+
let retrievalEntities = snapshotKey
|
|
1272
|
+
? loadCachedRetrievalEntities(options.session, snapshotKey, relevantTargets)
|
|
1273
|
+
: null;
|
|
1274
|
+
if (!retrievalEntities) {
|
|
1275
|
+
retrievalEntities = buildGoalRetrievalEntities(relevantTargets);
|
|
1276
|
+
if (snapshotKey) {
|
|
1277
|
+
saveCachedRetrievalEntities(options.session, snapshotKey, relevantTargets, retrievalEntities);
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1280
|
+
retrievedCandidates = expandRetrievalEntitiesToCandidates(relevantTargets, preselectRetrievalEntities(instruction, retrievalEntities));
|
|
1281
|
+
}
|
|
1282
|
+
else {
|
|
1283
|
+
retrievedCandidates = [...relevantTargets];
|
|
1284
|
+
}
|
|
917
1285
|
const candidates = diversifyCandidates(retrievedCandidates.length > 0 ? retrievedCandidates : [...relevantTargets]);
|
|
1286
|
+
if (!cachedCandidates && snapshotKey && goalKey) {
|
|
1287
|
+
saveCachedGoalShortlist(options.session, snapshotKey, goalKey, relevantTargets, candidates);
|
|
1288
|
+
}
|
|
918
1289
|
const { summaries: surfaceSummaries, summaryIdBySurfaceKey } = buildSurfaceSummaries(candidates);
|
|
919
1290
|
options.session &&
|
|
920
1291
|
recordPayloadBudget(options.session, {
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* browse start-session — Start the workflow/run context on top of an existing browser session.
|
|
3
|
+
*/
|
|
4
|
+
export type StartSessionSuccessResult = {
|
|
5
|
+
success: true;
|
|
6
|
+
runId: string;
|
|
7
|
+
startedAt: string;
|
|
8
|
+
browserSessionId: string;
|
|
9
|
+
intentSessionId: string;
|
|
10
|
+
lastEventSeq: number;
|
|
11
|
+
status: string;
|
|
12
|
+
currentRequestId: string | null;
|
|
13
|
+
replacedPreviousRun: boolean;
|
|
14
|
+
profile?: string;
|
|
15
|
+
pageRef?: string;
|
|
16
|
+
url?: string;
|
|
17
|
+
title?: string;
|
|
18
|
+
};
|
|
19
|
+
export type StartSessionFailureResult = {
|
|
20
|
+
success: false;
|
|
21
|
+
error: 'browser_session_unavailable' | 'backend_session_start_failed' | 'workflow_session_unavailable' | 'workflow_run_unavailable' | 'workflow_step_unavailable' | 'workflow_session_complete_failed';
|
|
22
|
+
outcomeType: 'blocked' | 'failed';
|
|
23
|
+
message: 'Session start failed.';
|
|
24
|
+
reason: string;
|
|
25
|
+
};
|
|
26
|
+
export type StartSessionResult = StartSessionSuccessResult | StartSessionFailureResult;
|
|
27
|
+
export interface StartSessionOptions {
|
|
28
|
+
merchantName?: string;
|
|
29
|
+
}
|
|
30
|
+
export declare function startSession(requestedName?: string, options?: StartSessionOptions): Promise<StartSessionResult>;
|
|
31
|
+
//# sourceMappingURL=start-session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"start-session.d.ts","sourceRoot":"","sources":["../../src/commands/start-session.ts"],"names":[],"mappings":"AAAA;;GAEG;AAyBH,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EACD,6BAA6B,GAC7B,8BAA8B,GAC9B,8BAA8B,GAC9B,0BAA0B,GAC1B,2BAA2B,GAC3B,kCAAkC,CAAC;IACvC,WAAW,EAAE,SAAS,GAAG,QAAQ,CAAC;IAClC,OAAO,EAAE,uBAAuB,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,yBAAyB,GAAG,yBAAyB,CAAC;AAIvF,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAgLD,wBAAsB,YAAY,CAChC,aAAa,CAAC,EAAE,MAAM,EACtB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA4N7B"}
|