@nuanu-ai/agentbrowse 0.2.47 → 0.2.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/README.md +81 -10
  2. package/dist/agentpay-gateway.d.ts +9 -0
  3. package/dist/agentpay-gateway.d.ts.map +1 -1
  4. package/dist/agentpay-gateway.js +31 -1
  5. package/dist/agentpay-stagehand-llm.d.ts.map +1 -1
  6. package/dist/agentpay-stagehand-llm.js +9 -1
  7. package/dist/command-api-tracing.d.ts +19 -0
  8. package/dist/command-api-tracing.d.ts.map +1 -0
  9. package/dist/command-api-tracing.js +137 -0
  10. package/dist/commands/act.d.ts.map +1 -1
  11. package/dist/commands/act.js +822 -670
  12. package/dist/commands/act.test-harness.d.ts +6 -0
  13. package/dist/commands/act.test-harness.d.ts.map +1 -1
  14. package/dist/commands/act.test-harness.js +44 -1
  15. package/dist/commands/action-acceptance.d.ts.map +1 -1
  16. package/dist/commands/action-acceptance.js +115 -0
  17. package/dist/commands/captcha-solve.d.ts.map +1 -1
  18. package/dist/commands/captcha-solve.js +83 -16
  19. package/dist/commands/click-action-executor.d.ts +0 -1
  20. package/dist/commands/click-action-executor.d.ts.map +1 -1
  21. package/dist/commands/click-action-executor.js +31 -77
  22. package/dist/commands/close.d.ts +3 -3
  23. package/dist/commands/close.d.ts.map +1 -1
  24. package/dist/commands/close.js +178 -0
  25. package/dist/commands/descriptor-validation.d.ts.map +1 -1
  26. package/dist/commands/descriptor-validation.js +75 -57
  27. package/dist/commands/end-session.d.ts +25 -0
  28. package/dist/commands/end-session.d.ts.map +1 -0
  29. package/dist/commands/end-session.js +161 -0
  30. package/dist/commands/extract-stagehand-executor.js +1 -1
  31. package/dist/commands/extract.d.ts.map +1 -1
  32. package/dist/commands/extract.js +339 -202
  33. package/dist/commands/fill-secret.d.ts +3 -3
  34. package/dist/commands/fill-secret.d.ts.map +1 -1
  35. package/dist/commands/fill-secret.js +419 -234
  36. package/dist/commands/get-secrets-catalog.d.ts.map +1 -1
  37. package/dist/commands/get-secrets-catalog.js +66 -5
  38. package/dist/commands/init.d.ts.map +1 -1
  39. package/dist/commands/init.js +6 -3
  40. package/dist/commands/interaction-kernel.d.ts +46 -0
  41. package/dist/commands/interaction-kernel.d.ts.map +1 -0
  42. package/dist/commands/interaction-kernel.js +215 -0
  43. package/dist/commands/launch.d.ts +1 -3
  44. package/dist/commands/launch.d.ts.map +1 -1
  45. package/dist/commands/launch.js +115 -27
  46. package/dist/commands/navigate.d.ts.map +1 -1
  47. package/dist/commands/navigate.js +188 -45
  48. package/dist/commands/observe-accessibility.d.ts.map +1 -1
  49. package/dist/commands/observe-accessibility.js +46 -39
  50. package/dist/commands/observe-dom-label-contract.d.ts.map +1 -1
  51. package/dist/commands/observe-dom-label-contract.js +5 -0
  52. package/dist/commands/observe-inventory.d.ts +13 -0
  53. package/dist/commands/observe-inventory.d.ts.map +1 -1
  54. package/dist/commands/observe-inventory.js +320 -65
  55. package/dist/commands/observe-persistence.d.ts.map +1 -1
  56. package/dist/commands/observe-persistence.js +3 -0
  57. package/dist/commands/observe-projection.d.ts +1 -0
  58. package/dist/commands/observe-projection.d.ts.map +1 -1
  59. package/dist/commands/observe-projection.js +7 -2
  60. package/dist/commands/observe-protected.d.ts +1 -0
  61. package/dist/commands/observe-protected.d.ts.map +1 -1
  62. package/dist/commands/observe-protected.js +9 -4
  63. package/dist/commands/observe-semantics.d.ts.map +1 -1
  64. package/dist/commands/observe-semantics.js +5 -2
  65. package/dist/commands/observe-stagehand.d.ts +1 -0
  66. package/dist/commands/observe-stagehand.d.ts.map +1 -1
  67. package/dist/commands/observe-stagehand.js +2 -0
  68. package/dist/commands/observe.d.ts +2 -0
  69. package/dist/commands/observe.d.ts.map +1 -1
  70. package/dist/commands/observe.js +387 -203
  71. package/dist/commands/observe.test-harness.d.ts +8 -0
  72. package/dist/commands/observe.test-harness.d.ts.map +1 -1
  73. package/dist/commands/observe.test-harness.js +48 -1
  74. package/dist/commands/poll-secret.d.ts +6 -0
  75. package/dist/commands/poll-secret.d.ts.map +1 -0
  76. package/dist/commands/poll-secret.js +159 -0
  77. package/dist/commands/request-secret.d.ts +6 -0
  78. package/dist/commands/request-secret.d.ts.map +1 -0
  79. package/dist/commands/request-secret.js +284 -0
  80. package/dist/commands/screenshot.d.ts.map +1 -1
  81. package/dist/commands/screenshot.js +172 -7
  82. package/dist/commands/select-action-executor.d.ts.map +1 -1
  83. package/dist/commands/semantic-observe.d.ts +4 -0
  84. package/dist/commands/semantic-observe.d.ts.map +1 -1
  85. package/dist/commands/semantic-observe.js +388 -17
  86. package/dist/commands/start-session.d.ts +31 -0
  87. package/dist/commands/start-session.d.ts.map +1 -0
  88. package/dist/commands/start-session.js +347 -0
  89. package/dist/commands/status.d.ts +2 -1
  90. package/dist/commands/status.d.ts.map +1 -1
  91. package/dist/commands/status.js +166 -144
  92. package/dist/control-semantics.d.ts +1 -0
  93. package/dist/control-semantics.d.ts.map +1 -1
  94. package/dist/control-semantics.js +51 -9
  95. package/dist/generated/build-config.d.ts +2 -0
  96. package/dist/generated/build-config.d.ts.map +1 -0
  97. package/dist/generated/build-config.js +2 -0
  98. package/dist/index.d.ts.map +1 -1
  99. package/dist/index.js +163 -63
  100. package/dist/otel-exporter.d.ts +58 -0
  101. package/dist/otel-exporter.d.ts.map +1 -0
  102. package/dist/otel-exporter.js +263 -0
  103. package/dist/otel-projector.d.ts +75 -0
  104. package/dist/otel-projector.d.ts.map +1 -0
  105. package/dist/otel-projector.js +409 -0
  106. package/dist/owned-browser.d.ts +1 -1
  107. package/dist/owned-browser.d.ts.map +1 -1
  108. package/dist/owned-browser.js +13 -1
  109. package/dist/owned-process.d.ts +2 -0
  110. package/dist/owned-process.d.ts.map +1 -1
  111. package/dist/owned-process.js +7 -3
  112. package/dist/playwright-runtime.d.ts +1 -1
  113. package/dist/playwright-runtime.d.ts.map +1 -1
  114. package/dist/playwright-runtime.js +8 -8
  115. package/dist/run-observability.d.ts +25 -0
  116. package/dist/run-observability.d.ts.map +1 -0
  117. package/dist/run-observability.js +115 -0
  118. package/dist/run-store.d.ts +274 -0
  119. package/dist/run-store.d.ts.map +1 -0
  120. package/dist/run-store.js +631 -0
  121. package/dist/runtime-metrics.d.ts +27 -0
  122. package/dist/runtime-metrics.d.ts.map +1 -0
  123. package/dist/runtime-metrics.js +66 -0
  124. package/dist/runtime-page-state.d.ts +11 -0
  125. package/dist/runtime-page-state.d.ts.map +1 -0
  126. package/dist/runtime-page-state.js +62 -0
  127. package/dist/runtime-protected-state.d.ts +16 -0
  128. package/dist/runtime-protected-state.d.ts.map +1 -0
  129. package/dist/runtime-protected-state.js +157 -0
  130. package/dist/runtime-state.d.ts +10 -44
  131. package/dist/runtime-state.d.ts.map +1 -1
  132. package/dist/runtime-state.js +57 -222
  133. package/dist/secrets/backend.d.ts +65 -16
  134. package/dist/secrets/backend.d.ts.map +1 -1
  135. package/dist/secrets/backend.js +135 -95
  136. package/dist/secrets/catalog-sync.d.ts.map +1 -1
  137. package/dist/secrets/catalog-sync.js +4 -1
  138. package/dist/secrets/form-matcher.d.ts +5 -5
  139. package/dist/secrets/form-matcher.d.ts.map +1 -1
  140. package/dist/secrets/form-matcher.js +292 -164
  141. package/dist/secrets/intent-output.d.ts +6 -10
  142. package/dist/secrets/intent-output.d.ts.map +1 -1
  143. package/dist/secrets/intent-output.js +4 -58
  144. package/dist/secrets/mock-agentpay-cabinet.d.ts +38 -27
  145. package/dist/secrets/mock-agentpay-cabinet.d.ts.map +1 -1
  146. package/dist/secrets/mock-agentpay-cabinet.js +177 -111
  147. package/dist/secrets/protected-artifact-guard.d.ts +2 -2
  148. package/dist/secrets/protected-artifact-guard.d.ts.map +1 -1
  149. package/dist/secrets/protected-artifact-guard.js +2 -2
  150. package/dist/secrets/protected-bindings.d.ts +1 -1
  151. package/dist/secrets/protected-bindings.d.ts.map +1 -1
  152. package/dist/secrets/protected-bindings.js +6 -0
  153. package/dist/secrets/protected-field-semantics.d.ts +9 -0
  154. package/dist/secrets/protected-field-semantics.d.ts.map +1 -0
  155. package/dist/secrets/protected-field-semantics.js +154 -0
  156. package/dist/secrets/protected-field-values.d.ts.map +1 -1
  157. package/dist/secrets/protected-field-values.js +3 -3
  158. package/dist/secrets/protected-fill.d.ts +1 -1
  159. package/dist/secrets/protected-fill.d.ts.map +1 -1
  160. package/dist/secrets/protected-fill.js +45 -149
  161. package/dist/secrets/protected-value-adapters.d.ts +2 -1
  162. package/dist/secrets/protected-value-adapters.d.ts.map +1 -1
  163. package/dist/secrets/protected-value-adapters.js +80 -1
  164. package/dist/secrets/request-output.d.ts +11 -0
  165. package/dist/secrets/request-output.d.ts.map +1 -0
  166. package/dist/secrets/request-output.js +75 -0
  167. package/dist/secrets/types.d.ts +15 -9
  168. package/dist/secrets/types.d.ts.map +1 -1
  169. package/dist/secrets/types.js +3 -0
  170. package/dist/session-event-exporter.d.ts +36 -0
  171. package/dist/session-event-exporter.d.ts.map +1 -0
  172. package/dist/session-event-exporter.js +428 -0
  173. package/dist/session.d.ts +16 -7
  174. package/dist/session.d.ts.map +1 -1
  175. package/dist/session.js +150 -23
  176. package/dist/sessions-backend.d.ts +354 -0
  177. package/dist/sessions-backend.d.ts.map +1 -0
  178. package/dist/sessions-backend.js +126 -0
  179. package/dist/solver/browser-launcher.d.ts +1 -1
  180. package/dist/solver/browser-launcher.d.ts.map +1 -1
  181. package/dist/solver/browser-launcher.js +39 -13
  182. package/dist/solver/captcha-solver.d.ts.map +1 -1
  183. package/dist/solver/captcha-solver.js +8 -1
  184. package/dist/solver/config.d.ts +0 -1
  185. package/dist/solver/config.d.ts.map +1 -1
  186. package/dist/solver/config.js +0 -22
  187. package/dist/solver/types.d.ts +1 -0
  188. package/dist/solver/types.d.ts.map +1 -1
  189. package/dist/workflow-session-completion.d.ts +33 -0
  190. package/dist/workflow-session-completion.d.ts.map +1 -0
  191. package/dist/workflow-session-completion.js +156 -0
  192. package/package.json +11 -1
  193. package/dist/commands/create-intent.d.ts +0 -6
  194. package/dist/commands/create-intent.d.ts.map +0 -1
  195. package/dist/commands/create-intent.js +0 -75
  196. package/dist/commands/poll-intent.d.ts +0 -6
  197. package/dist/commands/poll-intent.d.ts.map +0 -1
  198. package/dist/commands/poll-intent.js +0 -57
@@ -1,7 +1,8 @@
1
+ import { createHash } from 'node:crypto';
1
2
  import { z } from 'zod';
2
3
  import { AgentpayStagehandLlmClient } from '../agentpay-stagehand-llm.js';
3
4
  import { resolveAgentpayGatewayConfig } from '../agentpay-gateway.js';
4
- import { recordLlmUsage, recordPayloadBudget } from '../runtime-state.js';
5
+ import { recordLlmUsage, recordPayloadBudget } from '../runtime-metrics.js';
5
6
  import { analyzeSemanticObserveText, buildSemanticObserveBm25CorpusStats, buildSemanticObserveLexicalDocument, normalizeSemanticObserveText, scoreSemanticObserveBm25, } from './semantic-observe-lexical.js';
6
7
  const rerankSchema = z.object({
7
8
  matches: z
@@ -55,6 +56,12 @@ const HIGH_SIGNAL_SCOPE_KINDS = new Set([
55
56
  'card',
56
57
  'form',
57
58
  ]);
59
+ const SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION = 1;
60
+ const SEMANTIC_OBSERVE_SNAPSHOT_CACHE_KEY_VERSION = 'semantic-observe-snapshot-v1';
61
+ const SEMANTIC_OBSERVE_GOAL_CACHE_KEY_VERSION = 'semantic-observe-goal-v1';
62
+ const SEMANTIC_OBSERVE_MAX_SNAPSHOT_CACHE_ENTRIES = 4;
63
+ const SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT = 6;
64
+ const semanticObserveSnapshotCacheBySession = new WeakMap();
58
65
  function isFieldLikeTarget(target) {
59
66
  const kind = (target.kind ?? '').trim().toLowerCase();
60
67
  const role = (target.role ?? '').trim().toLowerCase();
@@ -135,12 +142,31 @@ function isPrimaryFormControlTarget(target) {
135
142
  acceptancePolicy === 'disclosure' &&
136
143
  (kind === 'button' || role === 'button'));
137
144
  }
145
+ function isConcreteFormChoiceTarget(target) {
146
+ if (!Boolean(formBucketKey(target)) ||
147
+ isScopeLikeCandidate(target) ||
148
+ isFieldLikeTarget(target) ||
149
+ !isActionLikeTargetCandidate(target) ||
150
+ isPrimaryFormControlTarget(target)) {
151
+ return false;
152
+ }
153
+ const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
154
+ const controlFamily = (target.controlFamily ?? '').trim().toLowerCase();
155
+ const surfaceKind = (target.surfaceKind ?? '').trim().toLowerCase();
156
+ const hasGoalIdentity = Boolean(goalLabelOf(target));
157
+ if (!hasGoalIdentity) {
158
+ return false;
159
+ }
160
+ return (controlFamily === 'trigger' ||
161
+ acceptancePolicy === 'selection' ||
162
+ ['listbox', 'menu', 'dropdown', 'popover'].includes(surfaceKind));
163
+ }
138
164
  function shouldPreserveStandaloneFormActionEntity(target) {
139
165
  return (Boolean(formBucketKey(target)) &&
140
166
  !isScopeLikeCandidate(target) &&
141
167
  !isFieldLikeTarget(target) &&
142
168
  isActionLikeTargetCandidate(target) &&
143
- isPrimaryFormControlTarget(target));
169
+ (isPrimaryFormControlTarget(target) || isConcreteFormChoiceTarget(target)));
144
170
  }
145
171
  function primaryFormTargetPriority(target) {
146
172
  const acceptancePolicy = (target.acceptancePolicy ?? '').trim().toLowerCase();
@@ -190,6 +216,322 @@ function contentItemBucketKey(target) {
190
216
  }
191
217
  return undefined;
192
218
  }
219
+ function cachedContextNodeValue(node) {
220
+ if (!node) {
221
+ return undefined;
222
+ }
223
+ return {
224
+ ...(node.kind ? { kind: node.kind } : {}),
225
+ ...(node.label ? { label: node.label } : {}),
226
+ ...(node.text ? { text: node.text } : {}),
227
+ };
228
+ }
229
+ function cachedStateEntries(states) {
230
+ if (!states) {
231
+ return undefined;
232
+ }
233
+ const entries = Object.entries(states).sort(([left], [right]) => left.localeCompare(right));
234
+ return entries.length > 0 ? entries : undefined;
235
+ }
236
+ function cachedStructureValue(structure) {
237
+ if (!structure) {
238
+ return undefined;
239
+ }
240
+ return {
241
+ ...(structure.family ? { family: structure.family } : {}),
242
+ ...(structure.variant ? { variant: structure.variant } : {}),
243
+ ...(structure.row ? { row: structure.row } : {}),
244
+ ...(structure.column ? { column: structure.column } : {}),
245
+ ...(structure.zone ? { zone: structure.zone } : {}),
246
+ ...(structure.cellLabel ? { cellLabel: structure.cellLabel } : {}),
247
+ };
248
+ }
249
+ function normalizeGoalIdentityValue(value, maxLength = 180) {
250
+ const normalized = value?.replace(/\s+/g, ' ').trim();
251
+ if (!normalized) {
252
+ return undefined;
253
+ }
254
+ return normalized.length > maxLength ? `${normalized.slice(0, maxLength - 1)}…` : normalized;
255
+ }
256
+ function appendGoalIdentityAlias(values, value) {
257
+ const normalized = normalizeGoalIdentityValue(value);
258
+ if (!normalized) {
259
+ return;
260
+ }
261
+ const normalizedKey = normalized.toLowerCase();
262
+ if (values.some((existing) => existing.toLowerCase() === normalizedKey)) {
263
+ return;
264
+ }
265
+ values.push(normalized);
266
+ }
267
+ function goalIdentityValuesOf(target) {
268
+ const values = [];
269
+ appendGoalIdentityAlias(values, target.goalLabel);
270
+ for (const value of target.goalAliases ?? []) {
271
+ appendGoalIdentityAlias(values, value);
272
+ }
273
+ if (values.length > 0) {
274
+ return values;
275
+ }
276
+ if (isFieldLikeTarget(target)) {
277
+ appendGoalIdentityAlias(values, target.label);
278
+ appendGoalIdentityAlias(values, target.displayLabel);
279
+ appendGoalIdentityAlias(values, target.placeholder);
280
+ appendGoalIdentityAlias(values, target.title);
281
+ return values;
282
+ }
283
+ appendGoalIdentityAlias(values, target.label);
284
+ appendGoalIdentityAlias(values, target.displayLabel);
285
+ appendGoalIdentityAlias(values, target.text);
286
+ appendGoalIdentityAlias(values, target.title);
287
+ return values;
288
+ }
289
+ function goalLabelOf(target) {
290
+ return goalIdentityValuesOf(target)[0];
291
+ }
292
+ function goalAliasesOf(target) {
293
+ return goalIdentityValuesOf(target).slice(1, 5);
294
+ }
295
+ function serializeObserveSnapshotTarget(target) {
296
+ const itemContext = cachedContextNodeValue(target.context?.item);
297
+ const groupContext = cachedContextNodeValue(target.context?.group);
298
+ const containerContext = cachedContextNodeValue(target.context?.container);
299
+ const landmarkContext = cachedContextNodeValue(target.context?.landmark);
300
+ const stateEntries = cachedStateEntries(target.states);
301
+ const structure = cachedStructureValue(target.structure);
302
+ const goalLabel = goalLabelOf(target);
303
+ const goalAliases = goalAliasesOf(target);
304
+ return {
305
+ goalInventoryType: target.goalInventoryType,
306
+ ...(target.goalInventoryType === 'scope' && target.goalSurfaceId
307
+ ? { goalSurfaceId: target.goalSurfaceId }
308
+ : {}),
309
+ ...(goalLabel ? { goalLabel } : {}),
310
+ ...(goalAliases.length ? { goalAliases } : {}),
311
+ ...(target.kind ? { kind: target.kind } : {}),
312
+ ...(target.label ? { label: target.label } : {}),
313
+ ...(target.displayLabel ? { displayLabel: target.displayLabel } : {}),
314
+ ...(target.role ? { role: target.role } : {}),
315
+ ...(target.interactionHint ? { interactionHint: target.interactionHint } : {}),
316
+ ...(target.text ? { text: target.text } : {}),
317
+ ...(target.placeholder ? { placeholder: target.placeholder } : {}),
318
+ ...(target.title ? { title: target.title } : {}),
319
+ ...(target.capability ? { capability: target.capability } : {}),
320
+ ...(target.allowedActions?.length
321
+ ? { allowedActions: [...target.allowedActions].sort() }
322
+ : {}),
323
+ ...(target.acceptancePolicy ? { acceptancePolicy: target.acceptancePolicy } : {}),
324
+ ...(target.controlFamily ? { controlFamily: target.controlFamily } : {}),
325
+ ...(target.surfaceRef ? { surfaceRef: target.surfaceRef } : {}),
326
+ ...(target.surfaceKind ? { surfaceKind: target.surfaceKind } : {}),
327
+ ...(target.surfaceLabel ? { surfaceLabel: target.surfaceLabel } : {}),
328
+ ...(typeof target.surfacePriority === 'number' ? { surfacePriority: target.surfacePriority } : {}),
329
+ ...(target.framePath?.length ? { framePath: [...target.framePath] } : {}),
330
+ ...(target.frameUrl ? { frameUrl: target.frameUrl } : {}),
331
+ ...(target.formSelector ? { formSelector: target.formSelector } : {}),
332
+ ...(target.pageSignature ? { pageSignature: target.pageSignature } : {}),
333
+ ...(target.controlsSurfaceSelector
334
+ ? { controlsSurfaceSelector: target.controlsSurfaceSelector }
335
+ : {}),
336
+ ...(stateEntries ? { states: stateEntries } : {}),
337
+ ...(target.context
338
+ ? {
339
+ context: {
340
+ ...(itemContext ? { item: itemContext } : {}),
341
+ ...(groupContext ? { group: groupContext } : {}),
342
+ ...(containerContext ? { container: containerContext } : {}),
343
+ ...(landmarkContext ? { landmark: landmarkContext } : {}),
344
+ ...(target.context.hintText ? { hintText: target.context.hintText } : {}),
345
+ },
346
+ }
347
+ : {}),
348
+ ...(structure ? { structure } : {}),
349
+ };
350
+ }
351
+ function semanticObserveSnapshotKey(targets) {
352
+ const payload = JSON.stringify(targets.map((target) => serializeObserveSnapshotTarget(target)));
353
+ return createHash('sha256')
354
+ .update(SEMANTIC_OBSERVE_SNAPSHOT_CACHE_KEY_VERSION)
355
+ .update(payload)
356
+ .digest('hex');
357
+ }
358
+ function semanticObserveGoalKey(goal) {
359
+ return createHash('sha256')
360
+ .update(SEMANTIC_OBSERVE_GOAL_CACHE_KEY_VERSION)
361
+ .update(normalizedGoalText(goal))
362
+ .digest('hex');
363
+ }
364
+ function serializeLexicalDocument(lexicalDocument) {
365
+ return {
366
+ analyzerKey: lexicalDocument.analyzerKey,
367
+ weightedLength: lexicalDocument.weightedLength,
368
+ weightedTermFrequencies: [...lexicalDocument.weightedTermFrequencies.entries()],
369
+ };
370
+ }
371
+ function deserializeLexicalDocument(lexicalDocument) {
372
+ return {
373
+ analyzerKey: lexicalDocument.analyzerKey,
374
+ weightedLength: lexicalDocument.weightedLength,
375
+ weightedTermFrequencies: new Map(lexicalDocument.weightedTermFrequencies),
376
+ };
377
+ }
378
+ function ensureSemanticObserveSnapshotCache(session) {
379
+ const existing = semanticObserveSnapshotCacheBySession.get(session);
380
+ if (existing &&
381
+ existing.version === SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION &&
382
+ Array.isArray(existing.order) &&
383
+ existing.snapshotsByKey &&
384
+ typeof existing.snapshotsByKey === 'object') {
385
+ return existing;
386
+ }
387
+ const nextCache = {
388
+ version: SEMANTIC_OBSERVE_SNAPSHOT_CACHE_VERSION,
389
+ order: [],
390
+ snapshotsByKey: {},
391
+ };
392
+ semanticObserveSnapshotCacheBySession.set(session, nextCache);
393
+ return nextCache;
394
+ }
395
+ function touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey) {
396
+ cache.order = [snapshotKey, ...cache.order.filter((key) => key !== snapshotKey)].slice(0, SEMANTIC_OBSERVE_MAX_SNAPSHOT_CACHE_ENTRIES);
397
+ for (const key of Object.keys(cache.snapshotsByKey)) {
398
+ if (!cache.order.includes(key)) {
399
+ delete cache.snapshotsByKey[key];
400
+ }
401
+ }
402
+ }
403
+ function materializeCachedRetrievalEntities(targets, cachedEntities) {
404
+ const materialized = [];
405
+ for (const cachedEntity of cachedEntities) {
406
+ const representative = targets[cachedEntity.representativeIndex];
407
+ if (!representative) {
408
+ return null;
409
+ }
410
+ if (cachedEntity.memberIndexes.some((index) => !targets[index])) {
411
+ return null;
412
+ }
413
+ materialized.push({
414
+ entityKind: cachedEntity.entityKind,
415
+ entityKey: cachedEntity.entityKey,
416
+ firstIndex: cachedEntity.firstIndex,
417
+ memberIndexes: [...cachedEntity.memberIndexes],
418
+ representative,
419
+ label: pickEntityLabel(cachedEntity.entityKind, representative),
420
+ kind: cachedEntity.entityKind === 'form'
421
+ ? 'form'
422
+ : cachedEntity.entityKind === 'item'
423
+ ? (representative.surfaceKind ?? representative.kind)
424
+ : representative.kind,
425
+ surfaceKind: representative.surfaceKind,
426
+ surfaceLabel: representative.surfaceLabel,
427
+ surfacePriority: representative.surfacePriority,
428
+ framePath: representative.framePath,
429
+ frameUrl: representative.frameUrl,
430
+ context: representative.context,
431
+ structure: representative.structure,
432
+ representativeLabels: [],
433
+ analyzerKey: cachedEntity.analyzerKey,
434
+ lexicalDocument: deserializeLexicalDocument(cachedEntity.lexicalDocument),
435
+ });
436
+ }
437
+ return materialized;
438
+ }
439
+ function loadCachedRetrievalEntities(session, snapshotKey, targets) {
440
+ if (!session) {
441
+ return null;
442
+ }
443
+ const cache = ensureSemanticObserveSnapshotCache(session);
444
+ const entry = cache.snapshotsByKey[snapshotKey];
445
+ if (!entry || entry.targetCount !== targets.length || !entry.retrievalEntities) {
446
+ return null;
447
+ }
448
+ const materialized = materializeCachedRetrievalEntities(targets, entry.retrievalEntities);
449
+ if (!materialized) {
450
+ delete cache.snapshotsByKey[snapshotKey];
451
+ cache.order = cache.order.filter((key) => key !== snapshotKey);
452
+ return null;
453
+ }
454
+ touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
455
+ return materialized;
456
+ }
457
+ function saveCachedRetrievalEntities(session, snapshotKey, targets, entities) {
458
+ if (!session) {
459
+ return;
460
+ }
461
+ const cache = ensureSemanticObserveSnapshotCache(session);
462
+ const indexByTarget = new Map(targets.map((target, index) => [target, index]));
463
+ const existingEntry = cache.snapshotsByKey[snapshotKey];
464
+ cache.snapshotsByKey[snapshotKey] = {
465
+ snapshotKey,
466
+ targetCount: targets.length,
467
+ cachedAt: new Date().toISOString(),
468
+ retrievalEntities: entities.map((entity) => ({
469
+ entityKind: entity.entityKind,
470
+ entityKey: entity.entityKey,
471
+ firstIndex: entity.firstIndex,
472
+ representativeIndex: indexByTarget.get(entity.representative) ?? entity.firstIndex,
473
+ memberIndexes: [...entity.memberIndexes],
474
+ analyzerKey: entity.analyzerKey,
475
+ lexicalDocument: serializeLexicalDocument(entity.lexicalDocument),
476
+ })),
477
+ goalShortlists: existingEntry?.goalShortlists ?? {},
478
+ goalOrder: existingEntry?.goalOrder ?? [],
479
+ };
480
+ touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
481
+ }
482
+ function loadCachedGoalShortlist(session, snapshotKey, goalKey, targets) {
483
+ if (!session || !goalKey) {
484
+ return null;
485
+ }
486
+ const cache = ensureSemanticObserveSnapshotCache(session);
487
+ const entry = cache.snapshotsByKey[snapshotKey];
488
+ const shortlist = entry?.goalShortlists?.[goalKey];
489
+ if (!entry || entry.targetCount !== targets.length || !shortlist) {
490
+ return null;
491
+ }
492
+ const candidates = shortlist.candidateIndexes.map((index) => targets[index]).filter(Boolean);
493
+ if (candidates.length !== shortlist.candidateIndexes.length) {
494
+ delete entry.goalShortlists?.[goalKey];
495
+ entry.goalOrder = (entry.goalOrder ?? []).filter((key) => key !== goalKey);
496
+ return null;
497
+ }
498
+ entry.goalOrder = [goalKey, ...(entry.goalOrder ?? []).filter((key) => key !== goalKey)].slice(0, SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT);
499
+ touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
500
+ return candidates;
501
+ }
502
+ function saveCachedGoalShortlist(session, snapshotKey, goalKey, targets, shortlist) {
503
+ if (!session || !goalKey) {
504
+ return;
505
+ }
506
+ const cache = ensureSemanticObserveSnapshotCache(session);
507
+ const existingEntry = cache.snapshotsByKey[snapshotKey];
508
+ const entry = existingEntry ?? {
509
+ snapshotKey,
510
+ targetCount: targets.length,
511
+ cachedAt: new Date().toISOString(),
512
+ goalShortlists: {},
513
+ goalOrder: [],
514
+ };
515
+ const indexByTarget = new Map(targets.map((target, index) => [target, index]));
516
+ entry.targetCount = targets.length;
517
+ entry.cachedAt = new Date().toISOString();
518
+ entry.goalShortlists ??= {};
519
+ entry.goalOrder = [goalKey, ...(entry.goalOrder ?? []).filter((key) => key !== goalKey)].slice(0, SEMANTIC_OBSERVE_MAX_GOAL_SHORTLISTS_PER_SNAPSHOT);
520
+ entry.goalShortlists[goalKey] = {
521
+ goalKey,
522
+ candidateIndexes: shortlist
523
+ .map((candidate) => indexByTarget.get(candidate))
524
+ .filter((index) => index !== undefined),
525
+ cachedAt: entry.cachedAt,
526
+ };
527
+ for (const cachedGoalKey of Object.keys(entry.goalShortlists)) {
528
+ if (!entry.goalOrder.includes(cachedGoalKey)) {
529
+ delete entry.goalShortlists[cachedGoalKey];
530
+ }
531
+ }
532
+ cache.snapshotsByKey[snapshotKey] = entry;
533
+ touchSemanticObserveSnapshotCacheEntry(cache, snapshotKey);
534
+ }
193
535
  function scopeCandidatePriority(target) {
194
536
  const kind = (target.kind ?? '').trim().toLowerCase();
195
537
  if (kind === 'dialog' || kind === 'listbox' || kind === 'menu') {
@@ -276,7 +618,7 @@ function pickEntityLabel(entityKind, representative) {
276
618
  representative.surfaceLabel ??
277
619
  representative.label);
278
620
  }
279
- return (representative.label ??
621
+ return (goalLabelOf(representative) ??
280
622
  representative.context?.container?.label ??
281
623
  representative.context?.item?.label ??
282
624
  representative.surfaceLabel ??
@@ -286,13 +628,14 @@ function pickEntityLabel(entityKind, representative) {
286
628
  function collectRepresentativeLabels(targets) {
287
629
  const labels = [];
288
630
  for (const target of targets) {
289
- const label = target.label?.replace(/\s+/g, ' ').trim();
290
- if (!label || label === 'Button' || label === 'Link' || labels.includes(label)) {
291
- continue;
292
- }
293
- labels.push(label);
294
- if (labels.length >= 4) {
295
- break;
631
+ for (const label of [goalLabelOf(target), ...goalAliasesOf(target)]) {
632
+ if (!label || label === 'Button' || label === 'Link' || labels.includes(label)) {
633
+ continue;
634
+ }
635
+ labels.push(label);
636
+ if (labels.length >= 4) {
637
+ return labels;
638
+ }
296
639
  }
297
640
  }
298
641
  return labels;
@@ -368,9 +711,10 @@ function latentActionHintText(target) {
368
711
  return hints.size > 0 ? [...hints].join(' ') : undefined;
369
712
  }
370
713
  function buildEntityLexicalFields(entityLabel, representative, representativeLabels) {
714
+ const representativeGoalLabel = goalLabelOf(representative);
371
715
  return [
372
716
  { value: entityLabel, weight: RETRIEVAL_FIELD_WEIGHTS.entityLabel },
373
- { value: representative.label, weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabel },
717
+ { value: representativeGoalLabel, weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabel },
374
718
  ...representativeLabels.map((value) => ({
375
719
  value,
376
720
  weight: RETRIEVAL_FIELD_WEIGHTS.representativeLabels,
@@ -857,6 +1201,8 @@ function buildSurfaceSummaries(targets) {
857
1201
  function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKey) {
858
1202
  const parts = [`id=c${index + 1}`];
859
1203
  const compactContext = compactCandidateContext(target);
1204
+ const goalLabel = goalLabelOf(target);
1205
+ const goalAliases = goalAliasesOf(target).map((value) => compactContextValue(value, 120) ?? value);
860
1206
  const surfaceSummaryId = surfaceIdentityOf(target)
861
1207
  ? surfaceSummaryIdBySurfaceKey?.get(surfaceIdentityOf(target))
862
1208
  : undefined;
@@ -864,8 +1210,10 @@ function buildCompactCandidateSummary(target, index, surfaceSummaryIdBySurfaceKe
864
1210
  parts.push(`kind=${JSON.stringify(target.kind)}`);
865
1211
  if (target.role)
866
1212
  parts.push(`role=${JSON.stringify(target.role)}`);
867
- if (target.label)
868
- parts.push(`label=${JSON.stringify(target.label)}`);
1213
+ if (goalLabel)
1214
+ parts.push(`label=${JSON.stringify(goalLabel)}`);
1215
+ if (goalAliases.length)
1216
+ parts.push(`aliases=${JSON.stringify(goalAliases)}`);
869
1217
  if (target.allowedActions?.length) {
870
1218
  parts.push(`actions=${JSON.stringify(target.allowedActions)}`);
871
1219
  }
@@ -910,11 +1258,34 @@ export async function rerankDomTargetsForGoal(instruction, targets, options = {}
910
1258
  });
911
1259
  const gateway = resolveAgentpayGatewayConfig();
912
1260
  const client = new AgentpayStagehandLlmClient(gateway);
913
- const retrievalEntities = buildGoalRetrievalEntities(relevantTargets);
914
- const retrievedCandidates = relevantTargets.length > RERANK_CANDIDATE_LIMIT
915
- ? expandRetrievalEntitiesToCandidates(relevantTargets, preselectRetrievalEntities(instruction, retrievalEntities))
916
- : [...relevantTargets];
1261
+ const snapshotKey = options.session ? semanticObserveSnapshotKey(relevantTargets) : undefined;
1262
+ const goalKey = snapshotKey ? semanticObserveGoalKey(instruction) : undefined;
1263
+ const cachedCandidates = snapshotKey && goalKey
1264
+ ? loadCachedGoalShortlist(options.session, snapshotKey, goalKey, relevantTargets)
1265
+ : null;
1266
+ let retrievedCandidates;
1267
+ if (cachedCandidates) {
1268
+ retrievedCandidates = cachedCandidates;
1269
+ }
1270
+ else if (relevantTargets.length > RERANK_CANDIDATE_LIMIT) {
1271
+ let retrievalEntities = snapshotKey
1272
+ ? loadCachedRetrievalEntities(options.session, snapshotKey, relevantTargets)
1273
+ : null;
1274
+ if (!retrievalEntities) {
1275
+ retrievalEntities = buildGoalRetrievalEntities(relevantTargets);
1276
+ if (snapshotKey) {
1277
+ saveCachedRetrievalEntities(options.session, snapshotKey, relevantTargets, retrievalEntities);
1278
+ }
1279
+ }
1280
+ retrievedCandidates = expandRetrievalEntitiesToCandidates(relevantTargets, preselectRetrievalEntities(instruction, retrievalEntities));
1281
+ }
1282
+ else {
1283
+ retrievedCandidates = [...relevantTargets];
1284
+ }
917
1285
  const candidates = diversifyCandidates(retrievedCandidates.length > 0 ? retrievedCandidates : [...relevantTargets]);
1286
+ if (!cachedCandidates && snapshotKey && goalKey) {
1287
+ saveCachedGoalShortlist(options.session, snapshotKey, goalKey, relevantTargets, candidates);
1288
+ }
918
1289
  const { summaries: surfaceSummaries, summaryIdBySurfaceKey } = buildSurfaceSummaries(candidates);
919
1290
  options.session &&
920
1291
  recordPayloadBudget(options.session, {
@@ -0,0 +1,31 @@
1
+ /**
2
+ * browse start-session — Start the workflow/run context on top of an existing browser session.
3
+ */
4
+ export type StartSessionSuccessResult = {
5
+ success: true;
6
+ runId: string;
7
+ startedAt: string;
8
+ browserSessionId: string;
9
+ intentSessionId: string;
10
+ lastEventSeq: number;
11
+ status: string;
12
+ currentRequestId: string | null;
13
+ replacedPreviousRun: boolean;
14
+ profile?: string;
15
+ pageRef?: string;
16
+ url?: string;
17
+ title?: string;
18
+ };
19
+ export type StartSessionFailureResult = {
20
+ success: false;
21
+ error: 'browser_session_unavailable' | 'backend_session_start_failed' | 'workflow_session_unavailable' | 'workflow_run_unavailable' | 'workflow_step_unavailable' | 'workflow_session_complete_failed';
22
+ outcomeType: 'blocked' | 'failed';
23
+ message: 'Session start failed.';
24
+ reason: string;
25
+ };
26
+ export type StartSessionResult = StartSessionSuccessResult | StartSessionFailureResult;
27
+ export interface StartSessionOptions {
28
+ merchantName?: string;
29
+ }
30
+ export declare function startSession(requestedName?: string, options?: StartSessionOptions): Promise<StartSessionResult>;
31
+ //# sourceMappingURL=start-session.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"start-session.d.ts","sourceRoot":"","sources":["../../src/commands/start-session.ts"],"names":[],"mappings":"AAAA;;GAEG;AAyBH,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,IAAI,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,yBAAyB,GAAG;IACtC,OAAO,EAAE,KAAK,CAAC;IACf,KAAK,EACD,6BAA6B,GAC7B,8BAA8B,GAC9B,8BAA8B,GAC9B,0BAA0B,GAC1B,2BAA2B,GAC3B,kCAAkC,CAAC;IACvC,WAAW,EAAE,SAAS,GAAG,QAAQ,CAAC;IAClC,OAAO,EAAE,uBAAuB,CAAC;IACjC,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,yBAAyB,GAAG,yBAAyB,CAAC;AAIvF,MAAM,WAAW,mBAAmB;IAClC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAgLD,wBAAsB,YAAY,CAChC,aAAa,CAAC,EAAE,MAAM,EACtB,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,kBAAkB,CAAC,CA4N7B"}