@predicatelabs/sdk 0.99.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +252 -0
- package/dist/actions.d.ts +185 -0
- package/dist/actions.d.ts.map +1 -0
- package/dist/actions.js +1120 -0
- package/dist/actions.js.map +1 -0
- package/dist/agent-runtime.d.ts +352 -0
- package/dist/agent-runtime.d.ts.map +1 -0
- package/dist/agent-runtime.js +1170 -0
- package/dist/agent-runtime.js.map +1 -0
- package/dist/agent.d.ts +164 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +408 -0
- package/dist/agent.js.map +1 -0
- package/dist/asserts/expect.d.ts +159 -0
- package/dist/asserts/expect.d.ts.map +1 -0
- package/dist/asserts/expect.js +547 -0
- package/dist/asserts/expect.js.map +1 -0
- package/dist/asserts/index.d.ts +58 -0
- package/dist/asserts/index.d.ts.map +1 -0
- package/dist/asserts/index.js +70 -0
- package/dist/asserts/index.js.map +1 -0
- package/dist/asserts/query.d.ts +199 -0
- package/dist/asserts/query.d.ts.map +1 -0
- package/dist/asserts/query.js +288 -0
- package/dist/asserts/query.js.map +1 -0
- package/dist/backends/actions.d.ts +119 -0
- package/dist/backends/actions.d.ts.map +1 -0
- package/dist/backends/actions.js +291 -0
- package/dist/backends/actions.js.map +1 -0
- package/dist/backends/browser-use-adapter.d.ts +131 -0
- package/dist/backends/browser-use-adapter.d.ts.map +1 -0
- package/dist/backends/browser-use-adapter.js +219 -0
- package/dist/backends/browser-use-adapter.js.map +1 -0
- package/dist/backends/cdp-backend.d.ts +66 -0
- package/dist/backends/cdp-backend.d.ts.map +1 -0
- package/dist/backends/cdp-backend.js +273 -0
- package/dist/backends/cdp-backend.js.map +1 -0
- package/dist/backends/index.d.ts +80 -0
- package/dist/backends/index.d.ts.map +1 -0
- package/dist/backends/index.js +101 -0
- package/dist/backends/index.js.map +1 -0
- package/dist/backends/protocol.d.ts +156 -0
- package/dist/backends/protocol.d.ts.map +1 -0
- package/dist/backends/protocol.js +16 -0
- package/dist/backends/protocol.js.map +1 -0
- package/dist/backends/sentience-context.d.ts +143 -0
- package/dist/backends/sentience-context.d.ts.map +1 -0
- package/dist/backends/sentience-context.js +359 -0
- package/dist/backends/sentience-context.js.map +1 -0
- package/dist/backends/snapshot.d.ts +188 -0
- package/dist/backends/snapshot.d.ts.map +1 -0
- package/dist/backends/snapshot.js +360 -0
- package/dist/backends/snapshot.js.map +1 -0
- package/dist/browser.d.ts +154 -0
- package/dist/browser.d.ts.map +1 -0
- package/dist/browser.js +920 -0
- package/dist/browser.js.map +1 -0
- package/dist/canonicalization.d.ts +126 -0
- package/dist/canonicalization.d.ts.map +1 -0
- package/dist/canonicalization.js +161 -0
- package/dist/canonicalization.js.map +1 -0
- package/dist/captcha/strategies.d.ts +12 -0
- package/dist/captcha/strategies.d.ts.map +1 -0
- package/dist/captcha/strategies.js +43 -0
- package/dist/captcha/strategies.js.map +1 -0
- package/dist/captcha/types.d.ts +45 -0
- package/dist/captcha/types.d.ts.map +1 -0
- package/dist/captcha/types.js +12 -0
- package/dist/captcha/types.js.map +1 -0
- package/dist/cli.d.ts +5 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +422 -0
- package/dist/cli.js.map +1 -0
- package/dist/conversational-agent.d.ts +123 -0
- package/dist/conversational-agent.d.ts.map +1 -0
- package/dist/conversational-agent.js +341 -0
- package/dist/conversational-agent.js.map +1 -0
- package/dist/cursor-policy.d.ts +41 -0
- package/dist/cursor-policy.d.ts.map +1 -0
- package/dist/cursor-policy.js +81 -0
- package/dist/cursor-policy.js.map +1 -0
- package/dist/debugger.d.ts +28 -0
- package/dist/debugger.d.ts.map +1 -0
- package/dist/debugger.js +107 -0
- package/dist/debugger.js.map +1 -0
- package/dist/expect.d.ts +16 -0
- package/dist/expect.d.ts.map +1 -0
- package/dist/expect.js +67 -0
- package/dist/expect.js.map +1 -0
- package/dist/failure-artifacts.d.ts +95 -0
- package/dist/failure-artifacts.d.ts.map +1 -0
- package/dist/failure-artifacts.js +805 -0
- package/dist/failure-artifacts.js.map +1 -0
- package/dist/generator.d.ts +16 -0
- package/dist/generator.d.ts.map +1 -0
- package/dist/generator.js +205 -0
- package/dist/generator.js.map +1 -0
- package/dist/index.d.ts +37 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +160 -0
- package/dist/index.js.map +1 -0
- package/dist/inspector.d.ts +13 -0
- package/dist/inspector.d.ts.map +1 -0
- package/dist/inspector.js +153 -0
- package/dist/inspector.js.map +1 -0
- package/dist/llm-provider.d.ts +144 -0
- package/dist/llm-provider.d.ts.map +1 -0
- package/dist/llm-provider.js +460 -0
- package/dist/llm-provider.js.map +1 -0
- package/dist/ordinal.d.ts +90 -0
- package/dist/ordinal.d.ts.map +1 -0
- package/dist/ordinal.js +249 -0
- package/dist/ordinal.js.map +1 -0
- package/dist/overlay.d.ts +63 -0
- package/dist/overlay.d.ts.map +1 -0
- package/dist/overlay.js +102 -0
- package/dist/overlay.js.map +1 -0
- package/dist/protocols/browser-protocol.d.ts +79 -0
- package/dist/protocols/browser-protocol.d.ts.map +1 -0
- package/dist/protocols/browser-protocol.js +9 -0
- package/dist/protocols/browser-protocol.js.map +1 -0
- package/dist/query.d.ts +66 -0
- package/dist/query.d.ts.map +1 -0
- package/dist/query.js +482 -0
- package/dist/query.js.map +1 -0
- package/dist/read.d.ts +47 -0
- package/dist/read.d.ts.map +1 -0
- package/dist/read.js +128 -0
- package/dist/read.js.map +1 -0
- package/dist/recorder.d.ts +44 -0
- package/dist/recorder.d.ts.map +1 -0
- package/dist/recorder.js +262 -0
- package/dist/recorder.js.map +1 -0
- package/dist/runtime-agent.d.ts +72 -0
- package/dist/runtime-agent.d.ts.map +1 -0
- package/dist/runtime-agent.js +357 -0
- package/dist/runtime-agent.js.map +1 -0
- package/dist/screenshot.d.ts +17 -0
- package/dist/screenshot.d.ts.map +1 -0
- package/dist/screenshot.js +40 -0
- package/dist/screenshot.js.map +1 -0
- package/dist/snapshot-diff.d.ts +23 -0
- package/dist/snapshot-diff.d.ts.map +1 -0
- package/dist/snapshot-diff.js +119 -0
- package/dist/snapshot-diff.js.map +1 -0
- package/dist/snapshot.d.ts +47 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +358 -0
- package/dist/snapshot.js.map +1 -0
- package/dist/textSearch.d.ts +64 -0
- package/dist/textSearch.d.ts.map +1 -0
- package/dist/textSearch.js +113 -0
- package/dist/textSearch.js.map +1 -0
- package/dist/tools/context.d.ts +18 -0
- package/dist/tools/context.d.ts.map +1 -0
- package/dist/tools/context.js +40 -0
- package/dist/tools/context.js.map +1 -0
- package/dist/tools/defaults.d.ts +5 -0
- package/dist/tools/defaults.d.ts.map +1 -0
- package/dist/tools/defaults.js +368 -0
- package/dist/tools/defaults.js.map +1 -0
- package/dist/tools/filesystem.d.ts +12 -0
- package/dist/tools/filesystem.d.ts.map +1 -0
- package/dist/tools/filesystem.js +137 -0
- package/dist/tools/filesystem.js.map +1 -0
- package/dist/tools/index.d.ts +5 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +15 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/registry.d.ts +38 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +100 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/tracing/cloud-sink.d.ts +189 -0
- package/dist/tracing/cloud-sink.d.ts.map +1 -0
- package/dist/tracing/cloud-sink.js +1067 -0
- package/dist/tracing/cloud-sink.js.map +1 -0
- package/dist/tracing/index-schema.d.ts +231 -0
- package/dist/tracing/index-schema.d.ts.map +1 -0
- package/dist/tracing/index-schema.js +235 -0
- package/dist/tracing/index-schema.js.map +1 -0
- package/dist/tracing/index.d.ts +12 -0
- package/dist/tracing/index.d.ts.map +1 -0
- package/dist/tracing/index.js +28 -0
- package/dist/tracing/index.js.map +1 -0
- package/dist/tracing/indexer.d.ts +20 -0
- package/dist/tracing/indexer.d.ts.map +1 -0
- package/dist/tracing/indexer.js +347 -0
- package/dist/tracing/indexer.js.map +1 -0
- package/dist/tracing/jsonl-sink.d.ts +51 -0
- package/dist/tracing/jsonl-sink.d.ts.map +1 -0
- package/dist/tracing/jsonl-sink.js +329 -0
- package/dist/tracing/jsonl-sink.js.map +1 -0
- package/dist/tracing/sink.d.ts +25 -0
- package/dist/tracing/sink.d.ts.map +1 -0
- package/dist/tracing/sink.js +15 -0
- package/dist/tracing/sink.js.map +1 -0
- package/dist/tracing/tracer-factory.d.ts +102 -0
- package/dist/tracing/tracer-factory.d.ts.map +1 -0
- package/dist/tracing/tracer-factory.js +375 -0
- package/dist/tracing/tracer-factory.js.map +1 -0
- package/dist/tracing/tracer.d.ts +140 -0
- package/dist/tracing/tracer.d.ts.map +1 -0
- package/dist/tracing/tracer.js +336 -0
- package/dist/tracing/tracer.js.map +1 -0
- package/dist/tracing/types.d.ts +203 -0
- package/dist/tracing/types.d.ts.map +1 -0
- package/dist/tracing/types.js +8 -0
- package/dist/tracing/types.js.map +1 -0
- package/dist/types.d.ts +422 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +6 -0
- package/dist/types.js.map +1 -0
- package/dist/utils/action-executor.d.ts +25 -0
- package/dist/utils/action-executor.d.ts.map +1 -0
- package/dist/utils/action-executor.js +121 -0
- package/dist/utils/action-executor.js.map +1 -0
- package/dist/utils/browser-evaluator.d.ts +76 -0
- package/dist/utils/browser-evaluator.d.ts.map +1 -0
- package/dist/utils/browser-evaluator.js +130 -0
- package/dist/utils/browser-evaluator.js.map +1 -0
- package/dist/utils/browser.d.ts +30 -0
- package/dist/utils/browser.d.ts.map +1 -0
- package/dist/utils/browser.js +75 -0
- package/dist/utils/browser.js.map +1 -0
- package/dist/utils/element-filter.d.ts +76 -0
- package/dist/utils/element-filter.d.ts.map +1 -0
- package/dist/utils/element-filter.js +195 -0
- package/dist/utils/element-filter.js.map +1 -0
- package/dist/utils/grid-utils.d.ts +37 -0
- package/dist/utils/grid-utils.d.ts.map +1 -0
- package/dist/utils/grid-utils.js +283 -0
- package/dist/utils/grid-utils.js.map +1 -0
- package/dist/utils/llm-interaction-handler.d.ts +41 -0
- package/dist/utils/llm-interaction-handler.d.ts.map +1 -0
- package/dist/utils/llm-interaction-handler.js +171 -0
- package/dist/utils/llm-interaction-handler.js.map +1 -0
- package/dist/utils/llm-response-builder.d.ts +56 -0
- package/dist/utils/llm-response-builder.d.ts.map +1 -0
- package/dist/utils/llm-response-builder.js +130 -0
- package/dist/utils/llm-response-builder.js.map +1 -0
- package/dist/utils/selector-utils.d.ts +12 -0
- package/dist/utils/selector-utils.d.ts.map +1 -0
- package/dist/utils/selector-utils.js +32 -0
- package/dist/utils/selector-utils.js.map +1 -0
- package/dist/utils/snapshot-event-builder.d.ts +28 -0
- package/dist/utils/snapshot-event-builder.d.ts.map +1 -0
- package/dist/utils/snapshot-event-builder.js +88 -0
- package/dist/utils/snapshot-event-builder.js.map +1 -0
- package/dist/utils/snapshot-processor.d.ts +27 -0
- package/dist/utils/snapshot-processor.d.ts.map +1 -0
- package/dist/utils/snapshot-processor.js +47 -0
- package/dist/utils/snapshot-processor.js.map +1 -0
- package/dist/utils/trace-event-builder.d.ts +122 -0
- package/dist/utils/trace-event-builder.d.ts.map +1 -0
- package/dist/utils/trace-event-builder.js +365 -0
- package/dist/utils/trace-event-builder.js.map +1 -0
- package/dist/utils/trace-file-manager.d.ts +70 -0
- package/dist/utils/trace-file-manager.d.ts.map +1 -0
- package/dist/utils/trace-file-manager.js +194 -0
- package/dist/utils/trace-file-manager.js.map +1 -0
- package/dist/utils/zod.d.ts +5 -0
- package/dist/utils/zod.d.ts.map +1 -0
- package/dist/utils/zod.js +80 -0
- package/dist/utils/zod.js.map +1 -0
- package/dist/utils.d.ts +8 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +13 -0
- package/dist/utils.js.map +1 -0
- package/dist/verification.d.ts +194 -0
- package/dist/verification.d.ts.map +1 -0
- package/dist/verification.js +530 -0
- package/dist/verification.js.map +1 -0
- package/dist/vision-executor.d.ts +18 -0
- package/dist/vision-executor.d.ts.map +1 -0
- package/dist/vision-executor.js +60 -0
- package/dist/vision-executor.js.map +1 -0
- package/dist/visual-agent.d.ts +120 -0
- package/dist/visual-agent.d.ts.map +1 -0
- package/dist/visual-agent.js +796 -0
- package/dist/visual-agent.js.map +1 -0
- package/dist/wait.d.ts +35 -0
- package/dist/wait.d.ts.map +1 -0
- package/dist/wait.js +76 -0
- package/dist/wait.js.map +1 -0
- package/package.json +94 -0
- package/spec/README.md +72 -0
- package/spec/SNAPSHOT_V1.md +208 -0
- package/spec/sdk-types.md +259 -0
- package/spec/snapshot.schema.json +148 -0
- package/src/extension/background.js +104 -0
- package/src/extension/content.js +162 -0
- package/src/extension/injected_api.js +1399 -0
- package/src/extension/manifest.json +36 -0
- package/src/extension/pkg/README.md +1340 -0
- package/src/extension/pkg/package.json +15 -0
- package/src/extension/pkg/sentience_core.d.ts +51 -0
- package/src/extension/pkg/sentience_core.js +371 -0
- package/src/extension/pkg/sentience_core_bg.wasm +0 -0
- package/src/extension/pkg/sentience_core_bg.wasm.d.ts +10 -0
- package/src/extension/release.json +116 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* v0 BrowserBackend Protocol - Minimal interface for browser-use integration.
|
|
3
|
+
*
|
|
4
|
+
* This protocol defines the minimal interface required to:
|
|
5
|
+
* - Take Sentience snapshots (DOM/geometry via extension)
|
|
6
|
+
* - Compute viewport-coord clicks
|
|
7
|
+
* - Scroll + re-snapshot + click
|
|
8
|
+
* - Stabilize after action
|
|
9
|
+
*
|
|
10
|
+
* No navigation API required (browser-use already handles navigation).
|
|
11
|
+
*
|
|
12
|
+
* Design principle: Keep it so small that nothing can break.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Viewport and scroll position information.
|
|
16
|
+
*/
|
|
17
|
+
export interface ViewportInfo {
|
|
18
|
+
width: number;
|
|
19
|
+
height: number;
|
|
20
|
+
scrollX: number;
|
|
21
|
+
scrollY: number;
|
|
22
|
+
contentWidth?: number;
|
|
23
|
+
contentHeight?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Page layout metrics from CDP Page.getLayoutMetrics.
|
|
27
|
+
*/
|
|
28
|
+
export interface LayoutMetrics {
|
|
29
|
+
viewportX: number;
|
|
30
|
+
viewportY: number;
|
|
31
|
+
viewportWidth: number;
|
|
32
|
+
viewportHeight: number;
|
|
33
|
+
contentWidth: number;
|
|
34
|
+
contentHeight: number;
|
|
35
|
+
deviceScaleFactor: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Mouse button type for click operations.
|
|
39
|
+
*/
|
|
40
|
+
export type MouseButton = 'left' | 'right' | 'middle';
|
|
41
|
+
/**
|
|
42
|
+
* Document ready state for wait operations.
|
|
43
|
+
*/
|
|
44
|
+
export type ReadyState = 'interactive' | 'complete';
|
|
45
|
+
/**
|
|
46
|
+
* Minimal backend protocol for v0 proof-of-concept.
|
|
47
|
+
*
|
|
48
|
+
* This is enough to:
|
|
49
|
+
* - Take Sentience snapshots (DOM/geometry via extension)
|
|
50
|
+
* - Execute JavaScript for element interaction
|
|
51
|
+
* - Perform mouse operations (move, click, scroll)
|
|
52
|
+
* - Wait for page stability
|
|
53
|
+
*
|
|
54
|
+
* Implementers:
|
|
55
|
+
* - CDPBackend: For browser-use integration via CDP
|
|
56
|
+
* - PlaywrightBackend: Wrapper around existing SentienceBrowser (future)
|
|
57
|
+
*/
|
|
58
|
+
export interface BrowserBackend {
|
|
59
|
+
/**
|
|
60
|
+
* Cache viewport + scroll offsets + url; cheap & safe to call often.
|
|
61
|
+
*
|
|
62
|
+
* @returns ViewportInfo with current viewport state
|
|
63
|
+
*/
|
|
64
|
+
refreshPageInfo(): Promise<ViewportInfo>;
|
|
65
|
+
/**
|
|
66
|
+
* Evaluate JavaScript expression in page context.
|
|
67
|
+
*
|
|
68
|
+
* Uses CDP Runtime.evaluate with returnByValue=True.
|
|
69
|
+
*
|
|
70
|
+
* @param expression - JavaScript expression to evaluate
|
|
71
|
+
* @returns Result value (JSON-serializable)
|
|
72
|
+
*/
|
|
73
|
+
eval(expression: string): Promise<unknown>;
|
|
74
|
+
/**
|
|
75
|
+
* Call a JavaScript function with arguments.
|
|
76
|
+
*
|
|
77
|
+
* Uses CDP Runtime.callFunctionOn for safe argument passing.
|
|
78
|
+
* Safer than eval() for passing complex arguments.
|
|
79
|
+
*
|
|
80
|
+
* @param functionDeclaration - JavaScript function body, e.g., "(x, y) => x + y"
|
|
81
|
+
* @param args - Arguments to pass to the function
|
|
82
|
+
* @returns Result value (JSON-serializable)
|
|
83
|
+
*/
|
|
84
|
+
call(functionDeclaration: string, args?: unknown[]): Promise<unknown>;
|
|
85
|
+
/**
|
|
86
|
+
* Get page layout metrics.
|
|
87
|
+
*
|
|
88
|
+
* Uses CDP Page.getLayoutMetrics to get viewport and content dimensions.
|
|
89
|
+
*
|
|
90
|
+
* @returns LayoutMetrics with viewport and content size info
|
|
91
|
+
*/
|
|
92
|
+
getLayoutMetrics(): Promise<LayoutMetrics>;
|
|
93
|
+
/**
|
|
94
|
+
* Capture viewport screenshot as PNG bytes.
|
|
95
|
+
*
|
|
96
|
+
* Uses CDP Page.captureScreenshot.
|
|
97
|
+
*
|
|
98
|
+
* @returns PNG image as base64 string
|
|
99
|
+
*/
|
|
100
|
+
screenshotPng(): Promise<string>;
|
|
101
|
+
/**
|
|
102
|
+
* Move mouse to viewport coordinates.
|
|
103
|
+
*
|
|
104
|
+
* Uses CDP Input.dispatchMouseEvent with type="mouseMoved".
|
|
105
|
+
*
|
|
106
|
+
* @param x - X coordinate in viewport
|
|
107
|
+
* @param y - Y coordinate in viewport
|
|
108
|
+
*/
|
|
109
|
+
mouseMove(x: number, y: number): Promise<void>;
|
|
110
|
+
/**
|
|
111
|
+
* Click at viewport coordinates.
|
|
112
|
+
*
|
|
113
|
+
* Uses CDP Input.dispatchMouseEvent with mousePressed + mouseReleased.
|
|
114
|
+
*
|
|
115
|
+
* @param x - X coordinate in viewport
|
|
116
|
+
* @param y - Y coordinate in viewport
|
|
117
|
+
* @param button - Mouse button to click (default: 'left')
|
|
118
|
+
* @param clickCount - Number of clicks (1 for single, 2 for double)
|
|
119
|
+
*/
|
|
120
|
+
mouseClick(x: number, y: number, button?: MouseButton, clickCount?: number): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Scroll using mouse wheel.
|
|
123
|
+
*
|
|
124
|
+
* Uses CDP Input.dispatchMouseEvent with type="mouseWheel".
|
|
125
|
+
*
|
|
126
|
+
* @param deltaY - Scroll amount (positive = down, negative = up)
|
|
127
|
+
* @param x - X coordinate for scroll (default: viewport center)
|
|
128
|
+
* @param y - Y coordinate for scroll (default: viewport center)
|
|
129
|
+
*/
|
|
130
|
+
wheel(deltaY: number, x?: number, y?: number): Promise<void>;
|
|
131
|
+
/**
|
|
132
|
+
* Type text using keyboard input.
|
|
133
|
+
*
|
|
134
|
+
* Uses CDP Input.dispatchKeyEvent for each character.
|
|
135
|
+
*
|
|
136
|
+
* @param text - Text to type
|
|
137
|
+
*/
|
|
138
|
+
typeText(text: string): Promise<void>;
|
|
139
|
+
/**
|
|
140
|
+
* Wait for document.readyState to reach target state.
|
|
141
|
+
*
|
|
142
|
+
* Uses polling instead of CDP events (no leak from unregistered listeners).
|
|
143
|
+
*
|
|
144
|
+
* @param state - Target state ("interactive" or "complete")
|
|
145
|
+
* @param timeoutMs - Maximum time to wait in milliseconds
|
|
146
|
+
* @throws TimeoutError if state not reached within timeout
|
|
147
|
+
*/
|
|
148
|
+
waitReadyState(state?: ReadyState, timeoutMs?: number): Promise<void>;
|
|
149
|
+
/**
|
|
150
|
+
* Get current page URL.
|
|
151
|
+
*
|
|
152
|
+
* @returns Current page URL (window.location.href)
|
|
153
|
+
*/
|
|
154
|
+
getUrl(): Promise<string>;
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=protocol.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.d.ts","sourceRoot":"","sources":["../../src/backends/protocol.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAE5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IAGvB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IAGtB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC;AAEtD;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,UAAU,CAAC;AAEpD;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;IAEzC;;;;;;;OAOG;IACH,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAE3C;;;;;;;;;OASG;IACH,IAAI,CAAC,mBAAmB,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAEtE;;;;;;OAMG;IACH,gBAAgB,IAAI,OAAO,CAAC,aAAa,CAAC,CAAC;IAE3C;;;;;;OAMG;IACH,aAAa,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;IAEjC;;;;;;;OAOG;IACH,SAAS,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE/C;;;;;;;;;OASG;IACH,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3F;;;;;;;;OAQG;IACH,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE7D;;;;;;OAMG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtC;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEtE;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CAC3B"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* v0 BrowserBackend Protocol - Minimal interface for browser-use integration.
|
|
4
|
+
*
|
|
5
|
+
* This protocol defines the minimal interface required to:
|
|
6
|
+
* - Take Sentience snapshots (DOM/geometry via extension)
|
|
7
|
+
* - Compute viewport-coord clicks
|
|
8
|
+
* - Scroll + re-snapshot + click
|
|
9
|
+
* - Stabilize after action
|
|
10
|
+
*
|
|
11
|
+
* No navigation API required (browser-use already handles navigation).
|
|
12
|
+
*
|
|
13
|
+
* Design principle: Keep it so small that nothing can break.
|
|
14
|
+
*/
|
|
15
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
16
|
+
//# sourceMappingURL=protocol.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"protocol.js","sourceRoot":"","sources":["../../src/backends/protocol.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SentienceContext: Token-Slasher Context Middleware for browser-use.
|
|
3
|
+
*
|
|
4
|
+
* This module provides a compact, ranked DOM context block for browser-use agents,
|
|
5
|
+
* reducing tokens and improving reliability by using Sentience snapshots.
|
|
6
|
+
*
|
|
7
|
+
* Example usage:
|
|
8
|
+
* import { SentienceContext } from 'sentience/backends';
|
|
9
|
+
*
|
|
10
|
+
* const ctx = new SentienceContext({ showOverlay: true });
|
|
11
|
+
* const state = await ctx.build(browserSession, { goal: "Click the first Show HN post" });
|
|
12
|
+
* if (state) {
|
|
13
|
+
* agent.addContext(state.promptBlock); // or however browser-use injects state
|
|
14
|
+
* }
|
|
15
|
+
*/
|
|
16
|
+
import type { Snapshot } from '../types';
|
|
17
|
+
/**
|
|
18
|
+
* Configuration for element selection strategy.
|
|
19
|
+
*
|
|
20
|
+
* The selector uses a 3-way merge to pick elements for the LLM context:
|
|
21
|
+
* 1. Top N by importance score (most actionable elements)
|
|
22
|
+
* 2. Top N from dominant group (for ordinal tasks like "click 3rd item")
|
|
23
|
+
* 3. Top N by position (elements at top of page, lowest doc_y)
|
|
24
|
+
*
|
|
25
|
+
* Elements are deduplicated across all three sources.
|
|
26
|
+
*/
|
|
27
|
+
export interface TopElementSelector {
|
|
28
|
+
/** Number of top elements to select by importance score (descending). Default: 60 */
|
|
29
|
+
byImportance?: number;
|
|
30
|
+
/** Number of top elements to select from the dominant group (for ordinal tasks). Default: 15 */
|
|
31
|
+
fromDominantGroup?: number;
|
|
32
|
+
/** Number of top elements to select by position (lowest doc_y = top of page). Default: 10 */
|
|
33
|
+
byPosition?: number;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Sentience context state with snapshot and formatted prompt block.
|
|
37
|
+
*/
|
|
38
|
+
export interface SentienceContextState {
|
|
39
|
+
url: string;
|
|
40
|
+
snapshot: Snapshot;
|
|
41
|
+
promptBlock: string;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Options for SentienceContext initialization.
|
|
45
|
+
*/
|
|
46
|
+
export interface SentienceContextOptions {
|
|
47
|
+
/** Canonical API key for gateway mode */
|
|
48
|
+
predicateApiKey?: string;
|
|
49
|
+
/** Backward-compatible API key alias */
|
|
50
|
+
sentienceApiKey?: string;
|
|
51
|
+
/** Force API vs extension mode (auto-detected if undefined) */
|
|
52
|
+
useApi?: boolean;
|
|
53
|
+
/** Maximum elements to fetch from snapshot. Default: 60 */
|
|
54
|
+
maxElements?: number;
|
|
55
|
+
/** Show visual overlay highlighting elements in browser. Default: false */
|
|
56
|
+
showOverlay?: boolean;
|
|
57
|
+
/** Configuration for element selection strategy */
|
|
58
|
+
topElementSelector?: TopElementSelector;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Options for the build() method.
|
|
62
|
+
*/
|
|
63
|
+
export interface BuildOptions {
|
|
64
|
+
/** Optional goal/task description (passed to gateway for reranking) */
|
|
65
|
+
goal?: string;
|
|
66
|
+
/** Maximum time to wait for extension injection in milliseconds. Default: 5000 */
|
|
67
|
+
waitForExtensionMs?: number;
|
|
68
|
+
/** Number of retry attempts on snapshot failure. Default: 2 */
|
|
69
|
+
retries?: number;
|
|
70
|
+
/** Delay between retries in milliseconds. Default: 1000 */
|
|
71
|
+
retryDelayMs?: number;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Token-Slasher Context Middleware for browser-use.
|
|
75
|
+
*
|
|
76
|
+
* Creates a compact, ranked DOM context block using Sentience snapshots,
|
|
77
|
+
* reducing tokens and improving reliability for LLM-based browser agents.
|
|
78
|
+
*
|
|
79
|
+
* Example:
|
|
80
|
+
* import { SentienceContext } from 'sentience/backends';
|
|
81
|
+
*
|
|
82
|
+
* const ctx = new SentienceContext({ showOverlay: true });
|
|
83
|
+
* const state = await ctx.build(browserSession, { goal: "Click the first Show HN post" });
|
|
84
|
+
* if (state) {
|
|
85
|
+
* agent.addContext(state.promptBlock);
|
|
86
|
+
* }
|
|
87
|
+
*/
|
|
88
|
+
export declare class SentienceContext {
|
|
89
|
+
private _apiKey;
|
|
90
|
+
private _useApi;
|
|
91
|
+
private _maxElements;
|
|
92
|
+
private _showOverlay;
|
|
93
|
+
private _selector;
|
|
94
|
+
constructor(options?: SentienceContextOptions);
|
|
95
|
+
/**
|
|
96
|
+
* Build context state from browser session.
|
|
97
|
+
*
|
|
98
|
+
* Takes a snapshot using the Sentience extension and formats it for LLM consumption.
|
|
99
|
+
* Returns null if snapshot fails (extension not loaded, timeout, etc.).
|
|
100
|
+
*
|
|
101
|
+
* @param browserSession - Browser-use BrowserSession instance (or any object with getOrCreateCdpSession)
|
|
102
|
+
* @param options - Build options
|
|
103
|
+
* @returns SentienceContextState with snapshot and formatted prompt, or null if failed
|
|
104
|
+
*/
|
|
105
|
+
build(browserSession: unknown, options?: BuildOptions): Promise<SentienceContextState | null>;
|
|
106
|
+
/**
|
|
107
|
+
* Format Sentience snapshot for LLM consumption.
|
|
108
|
+
*
|
|
109
|
+
* Creates an ultra-compact inventory of interactive elements optimized
|
|
110
|
+
* for minimal token usage. Uses 3-way selection: by importance,
|
|
111
|
+
* from dominant group, and by position.
|
|
112
|
+
*
|
|
113
|
+
* @param snap - Sentience Snapshot object
|
|
114
|
+
* @returns Formatted string with format: ID|role|text|imp|is_primary|docYq|ord|DG|href
|
|
115
|
+
*/
|
|
116
|
+
private _formatSnapshotForLLM;
|
|
117
|
+
/**
|
|
118
|
+
* Wait for Sentience extension to be ready in the browser.
|
|
119
|
+
*
|
|
120
|
+
* Polls window.sentience until it's defined or timeout is reached.
|
|
121
|
+
*
|
|
122
|
+
* @param backend - Browser backend with eval() method
|
|
123
|
+
* @param timeoutMs - Maximum time to wait in milliseconds
|
|
124
|
+
* @param pollIntervalMs - Interval between polls in milliseconds
|
|
125
|
+
* @returns true if extension is ready, false if timeout
|
|
126
|
+
*/
|
|
127
|
+
private _waitForExtension;
|
|
128
|
+
/**
|
|
129
|
+
* Compress href into a short token for minimal tokens.
|
|
130
|
+
*
|
|
131
|
+
* @param href - Full URL or undefined
|
|
132
|
+
* @returns Short token (domain second-level or last path segment)
|
|
133
|
+
*/
|
|
134
|
+
private _compressHref;
|
|
135
|
+
private _sleep;
|
|
136
|
+
get selector(): Required<TopElementSelector>;
|
|
137
|
+
}
|
|
138
|
+
/**
|
|
139
|
+
* Predicate rebrand alias for SentienceContext.
|
|
140
|
+
* Kept as a runtime alias to avoid breaking existing integrations.
|
|
141
|
+
*/
|
|
142
|
+
export declare const PredicateContext: typeof SentienceContext;
|
|
143
|
+
//# sourceMappingURL=sentience-context.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentience-context.d.ts","sourceRoot":"","sources":["../../src/backends/sentience-context.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAW,QAAQ,EAAE,MAAM,UAAU,CAAC;AAKlD;;;;;;;;;GASG;AACH,MAAM,WAAW,kBAAkB;IACjC,qFAAqF;IACrF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,gGAAgG;IAChG,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,6FAA6F;IAC7F,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,yCAAyC;IACzC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,wCAAwC;IACxC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2DAA2D;IAC3D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,2EAA2E;IAC3E,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mDAAmD;IACnD,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,uEAAuE;IACvE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,+DAA+D;IAC/D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAuBD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,OAAO,CAAqB;IACpC,OAAO,CAAC,OAAO,CAAsB;IACrC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,YAAY,CAAU;IAC9B,OAAO,CAAC,SAAS,CAA+B;gBAEpC,OAAO,GAAE,uBAA4B;IAYjD;;;;;;;;;OASG;IACG,KAAK,CACT,cAAc,EAAE,OAAO,EACvB,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;IAgFxC;;;;;;;;;OASG;IACH,OAAO,CAAC,qBAAqB;IA6J7B;;;;;;;;;OASG;YACW,iBAAiB;IA0B/B;;;;;OAKG;IACH,OAAO,CAAC,aAAa;IA+BrB,OAAO,CAAC,MAAM;IAKd,IAAI,QAAQ,IAAI,QAAQ,CAAC,kBAAkB,CAAC,CAE3C;CACF;AAED;;;GAGG;AACH,eAAO,MAAM,gBAAgB,yBAAmB,CAAC"}
|
|
@@ -0,0 +1,359 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* SentienceContext: Token-Slasher Context Middleware for browser-use.
|
|
4
|
+
*
|
|
5
|
+
* This module provides a compact, ranked DOM context block for browser-use agents,
|
|
6
|
+
* reducing tokens and improving reliability by using Sentience snapshots.
|
|
7
|
+
*
|
|
8
|
+
* Example usage:
|
|
9
|
+
* import { SentienceContext } from 'sentience/backends';
|
|
10
|
+
*
|
|
11
|
+
* const ctx = new SentienceContext({ showOverlay: true });
|
|
12
|
+
* const state = await ctx.build(browserSession, { goal: "Click the first Show HN post" });
|
|
13
|
+
* if (state) {
|
|
14
|
+
* agent.addContext(state.promptBlock); // or however browser-use injects state
|
|
15
|
+
* }
|
|
16
|
+
*/
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.PredicateContext = exports.SentienceContext = void 0;
|
|
19
|
+
const browser_use_adapter_1 = require("./browser-use-adapter");
|
|
20
|
+
const snapshot_1 = require("./snapshot");
|
|
21
|
+
/** Interactive roles that should be included in the context */
|
|
22
|
+
const INTERACTIVE_ROLES = new Set([
|
|
23
|
+
'button',
|
|
24
|
+
'link',
|
|
25
|
+
'textbox',
|
|
26
|
+
'searchbox',
|
|
27
|
+
'combobox',
|
|
28
|
+
'checkbox',
|
|
29
|
+
'radio',
|
|
30
|
+
'slider',
|
|
31
|
+
'tab',
|
|
32
|
+
'menuitem',
|
|
33
|
+
'option',
|
|
34
|
+
'switch',
|
|
35
|
+
'cell',
|
|
36
|
+
'a',
|
|
37
|
+
'input',
|
|
38
|
+
'select',
|
|
39
|
+
'textarea',
|
|
40
|
+
]);
|
|
41
|
+
/**
|
|
42
|
+
* Token-Slasher Context Middleware for browser-use.
|
|
43
|
+
*
|
|
44
|
+
* Creates a compact, ranked DOM context block using Sentience snapshots,
|
|
45
|
+
* reducing tokens and improving reliability for LLM-based browser agents.
|
|
46
|
+
*
|
|
47
|
+
* Example:
|
|
48
|
+
* import { SentienceContext } from 'sentience/backends';
|
|
49
|
+
*
|
|
50
|
+
* const ctx = new SentienceContext({ showOverlay: true });
|
|
51
|
+
* const state = await ctx.build(browserSession, { goal: "Click the first Show HN post" });
|
|
52
|
+
* if (state) {
|
|
53
|
+
* agent.addContext(state.promptBlock);
|
|
54
|
+
* }
|
|
55
|
+
*/
|
|
56
|
+
class SentienceContext {
|
|
57
|
+
constructor(options = {}) {
|
|
58
|
+
this._apiKey = options.predicateApiKey ?? options.sentienceApiKey;
|
|
59
|
+
this._useApi = options.useApi;
|
|
60
|
+
this._maxElements = options.maxElements ?? 60;
|
|
61
|
+
this._showOverlay = options.showOverlay ?? false;
|
|
62
|
+
this._selector = {
|
|
63
|
+
byImportance: options.topElementSelector?.byImportance ?? 60,
|
|
64
|
+
fromDominantGroup: options.topElementSelector?.fromDominantGroup ?? 15,
|
|
65
|
+
byPosition: options.topElementSelector?.byPosition ?? 10,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Build context state from browser session.
|
|
70
|
+
*
|
|
71
|
+
* Takes a snapshot using the Sentience extension and formats it for LLM consumption.
|
|
72
|
+
* Returns null if snapshot fails (extension not loaded, timeout, etc.).
|
|
73
|
+
*
|
|
74
|
+
* @param browserSession - Browser-use BrowserSession instance (or any object with getOrCreateCdpSession)
|
|
75
|
+
* @param options - Build options
|
|
76
|
+
* @returns SentienceContextState with snapshot and formatted prompt, or null if failed
|
|
77
|
+
*/
|
|
78
|
+
async build(browserSession, options = {}) {
|
|
79
|
+
const { goal, waitForExtensionMs = 5000, retries = 2, retryDelayMs = 1000 } = options;
|
|
80
|
+
try {
|
|
81
|
+
// Create adapter and backend
|
|
82
|
+
const adapter = new browser_use_adapter_1.BrowserUseAdapter(browserSession);
|
|
83
|
+
const backend = await adapter.createBackend();
|
|
84
|
+
// Wait for extension to inject (poll until ready or timeout)
|
|
85
|
+
await this._waitForExtension(backend, waitForExtensionMs);
|
|
86
|
+
// Build snapshot options
|
|
87
|
+
const snapshotOptions = {
|
|
88
|
+
limit: this._maxElements,
|
|
89
|
+
showOverlay: this._showOverlay,
|
|
90
|
+
goal,
|
|
91
|
+
};
|
|
92
|
+
// Set API options
|
|
93
|
+
if (this._apiKey) {
|
|
94
|
+
snapshotOptions.predicateApiKey = this._apiKey;
|
|
95
|
+
}
|
|
96
|
+
if (this._useApi !== undefined) {
|
|
97
|
+
snapshotOptions.useApi = this._useApi;
|
|
98
|
+
}
|
|
99
|
+
else if (this._apiKey) {
|
|
100
|
+
snapshotOptions.useApi = true;
|
|
101
|
+
}
|
|
102
|
+
// Take snapshot with retry logic
|
|
103
|
+
let snap = null;
|
|
104
|
+
let lastError = null;
|
|
105
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
106
|
+
try {
|
|
107
|
+
snap = await (0, snapshot_1.snapshot)(backend, snapshotOptions);
|
|
108
|
+
break; // Success
|
|
109
|
+
}
|
|
110
|
+
catch (e) {
|
|
111
|
+
lastError = e instanceof Error ? e : new Error(String(e));
|
|
112
|
+
if (attempt < retries - 1) {
|
|
113
|
+
console.debug(`Sentience snapshot attempt ${attempt + 1} failed: ${lastError.message}, retrying...`);
|
|
114
|
+
await this._sleep(retryDelayMs);
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
console.warn(`Sentience snapshot failed after ${retries} attempts: ${lastError.message}`);
|
|
118
|
+
return null;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
if (!snap) {
|
|
123
|
+
console.warn('Sentience snapshot returned null');
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
// Get URL from snapshot
|
|
127
|
+
const url = snap.url || '';
|
|
128
|
+
// Format for LLM
|
|
129
|
+
const formatted = this._formatSnapshotForLLM(snap);
|
|
130
|
+
// Build prompt block
|
|
131
|
+
const promptBlock = 'Elements: ID|role|text|imp|is_primary|docYq|ord|DG|href\n' +
|
|
132
|
+
'Rules: ordinal→DG=1 then ord asc; otherwise imp desc. ' +
|
|
133
|
+
'Use click(ID)/input_text(ID,...).\n' +
|
|
134
|
+
formatted;
|
|
135
|
+
console.info(`SentienceContext snapshot: ${snap.elements.length} elements URL=${url}`);
|
|
136
|
+
return { url, snapshot: snap, promptBlock };
|
|
137
|
+
}
|
|
138
|
+
catch (e) {
|
|
139
|
+
const error = e instanceof Error ? e : new Error(String(e));
|
|
140
|
+
console.warn(`Sentience snapshot skipped: ${error.message}`);
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Format Sentience snapshot for LLM consumption.
|
|
146
|
+
*
|
|
147
|
+
* Creates an ultra-compact inventory of interactive elements optimized
|
|
148
|
+
* for minimal token usage. Uses 3-way selection: by importance,
|
|
149
|
+
* from dominant group, and by position.
|
|
150
|
+
*
|
|
151
|
+
* @param snap - Sentience Snapshot object
|
|
152
|
+
* @returns Formatted string with format: ID|role|text|imp|is_primary|docYq|ord|DG|href
|
|
153
|
+
*/
|
|
154
|
+
_formatSnapshotForLLM(snap) {
|
|
155
|
+
// Filter to interactive elements only
|
|
156
|
+
const interactiveElements = snap.elements.filter(el => {
|
|
157
|
+
const role = (el.role || '').toLowerCase();
|
|
158
|
+
return INTERACTIVE_ROLES.has(role);
|
|
159
|
+
});
|
|
160
|
+
// Sort by importance (descending) for importance-based selection
|
|
161
|
+
interactiveElements.sort((a, b) => (b.importance || 0) - (a.importance || 0));
|
|
162
|
+
// Get top N by importance (track by ID for deduplication)
|
|
163
|
+
const selectedIds = new Set();
|
|
164
|
+
const selectedElements = [];
|
|
165
|
+
for (const el of interactiveElements.slice(0, this._selector.byImportance)) {
|
|
166
|
+
if (!selectedIds.has(el.id)) {
|
|
167
|
+
selectedIds.add(el.id);
|
|
168
|
+
selectedElements.push(el);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
// Get top elements from dominant group (for ordinal tasks)
|
|
172
|
+
// Prefer in_dominant_group field (uses fuzzy matching from gateway)
|
|
173
|
+
let dominantGroupElements = interactiveElements.filter(el => el.in_dominant_group === true);
|
|
174
|
+
// Fallback to exact group_key match if in_dominant_group not populated
|
|
175
|
+
if (dominantGroupElements.length === 0 && snap.dominant_group_key) {
|
|
176
|
+
dominantGroupElements = interactiveElements.filter(el => el.group_key === snap.dominant_group_key);
|
|
177
|
+
}
|
|
178
|
+
// Sort by group_index for ordinal ordering
|
|
179
|
+
dominantGroupElements.sort((a, b) => (a.group_index ?? 999) - (b.group_index ?? 999));
|
|
180
|
+
for (const el of dominantGroupElements.slice(0, this._selector.fromDominantGroup)) {
|
|
181
|
+
if (!selectedIds.has(el.id)) {
|
|
182
|
+
selectedIds.add(el.id);
|
|
183
|
+
selectedElements.push(el);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
// Get top elements by position (lowest doc_y = top of page)
|
|
187
|
+
const getYPosition = (el) => {
|
|
188
|
+
if (el.doc_y !== undefined)
|
|
189
|
+
return el.doc_y;
|
|
190
|
+
if (el.bbox)
|
|
191
|
+
return el.bbox.y;
|
|
192
|
+
return Infinity;
|
|
193
|
+
};
|
|
194
|
+
const elementsByPosition = [...interactiveElements].sort((a, b) => {
|
|
195
|
+
const yDiff = getYPosition(a) - getYPosition(b);
|
|
196
|
+
if (yDiff !== 0)
|
|
197
|
+
return yDiff;
|
|
198
|
+
// Tie-breaker: higher importance first
|
|
199
|
+
return (b.importance || 0) - (a.importance || 0);
|
|
200
|
+
});
|
|
201
|
+
for (const el of elementsByPosition.slice(0, this._selector.byPosition)) {
|
|
202
|
+
if (!selectedIds.has(el.id)) {
|
|
203
|
+
selectedIds.add(el.id);
|
|
204
|
+
selectedElements.push(el);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
// Compute local rank_in_group for dominant group elements
|
|
208
|
+
const rankInGroupMap = new Map();
|
|
209
|
+
// Get all dominant group elements for rank computation
|
|
210
|
+
let dgElementsForRank = interactiveElements.filter(el => el.in_dominant_group === true);
|
|
211
|
+
if (dgElementsForRank.length === 0 && snap.dominant_group_key) {
|
|
212
|
+
dgElementsForRank = interactiveElements.filter(el => el.group_key === snap.dominant_group_key);
|
|
213
|
+
}
|
|
214
|
+
// Sort by (doc_y, bbox.y, bbox.x, -importance)
|
|
215
|
+
dgElementsForRank.sort((a, b) => {
|
|
216
|
+
const docYA = a.doc_y ?? Infinity;
|
|
217
|
+
const docYB = b.doc_y ?? Infinity;
|
|
218
|
+
if (docYA !== docYB)
|
|
219
|
+
return docYA - docYB;
|
|
220
|
+
const bboxYA = a.bbox?.y ?? Infinity;
|
|
221
|
+
const bboxYB = b.bbox?.y ?? Infinity;
|
|
222
|
+
if (bboxYA !== bboxYB)
|
|
223
|
+
return bboxYA - bboxYB;
|
|
224
|
+
const bboxXA = a.bbox?.x ?? Infinity;
|
|
225
|
+
const bboxXB = b.bbox?.x ?? Infinity;
|
|
226
|
+
if (bboxXA !== bboxXB)
|
|
227
|
+
return bboxXA - bboxXB;
|
|
228
|
+
return (b.importance || 0) - (a.importance || 0);
|
|
229
|
+
});
|
|
230
|
+
dgElementsForRank.forEach((el, rank) => {
|
|
231
|
+
rankInGroupMap.set(el.id, rank);
|
|
232
|
+
});
|
|
233
|
+
// Format lines
|
|
234
|
+
const lines = [];
|
|
235
|
+
for (const el of selectedElements) {
|
|
236
|
+
// Get role (override to "link" if element has href)
|
|
237
|
+
let role = el.role || '';
|
|
238
|
+
if (el.href) {
|
|
239
|
+
role = 'link';
|
|
240
|
+
}
|
|
241
|
+
else if (!role) {
|
|
242
|
+
// Generic fallback for interactive elements without explicit role
|
|
243
|
+
role = 'element';
|
|
244
|
+
}
|
|
245
|
+
// Get name/text (truncate aggressively, normalize whitespace)
|
|
246
|
+
let name = el.text || '';
|
|
247
|
+
// Remove newlines and normalize whitespace
|
|
248
|
+
name = name.replace(/\s+/g, ' ').trim();
|
|
249
|
+
if (name.length > 30) {
|
|
250
|
+
name = name.slice(0, 27) + '...';
|
|
251
|
+
}
|
|
252
|
+
// Extract fields
|
|
253
|
+
const importance = el.importance || 0;
|
|
254
|
+
const docY = el.doc_y || 0;
|
|
255
|
+
// is_primary: from visual_cues.is_primary (boolean)
|
|
256
|
+
const isPrimary = el.visual_cues?.is_primary || false;
|
|
257
|
+
const isPrimaryFlag = isPrimary ? '1' : '0';
|
|
258
|
+
// docYq: bucketed doc_y (round to nearest 200 for smaller numbers)
|
|
259
|
+
const docYq = docY ? Math.round(docY / 200) : 0;
|
|
260
|
+
// Determine if in dominant group
|
|
261
|
+
let inDg = el.in_dominant_group;
|
|
262
|
+
if (inDg === undefined && snap.dominant_group_key) {
|
|
263
|
+
// Fallback for older gateway versions
|
|
264
|
+
inDg = el.group_key === snap.dominant_group_key;
|
|
265
|
+
}
|
|
266
|
+
// ord_val: rank_in_group if in dominant group
|
|
267
|
+
let ordVal = '-';
|
|
268
|
+
if (inDg && rankInGroupMap.has(el.id)) {
|
|
269
|
+
ordVal = rankInGroupMap.get(el.id);
|
|
270
|
+
}
|
|
271
|
+
// DG: 1 if dominant group, else 0
|
|
272
|
+
const dgFlag = inDg ? '1' : '0';
|
|
273
|
+
// href: short token (domain or last path segment, or blank)
|
|
274
|
+
const href = this._compressHref(el.href);
|
|
275
|
+
// Ultra-compact format: ID|role|text|imp|is_primary|docYq|ord|DG|href
|
|
276
|
+
const line = `${el.id}|${role}|${name}|${importance}|${isPrimaryFlag}|${docYq}|${ordVal}|${dgFlag}|${href}`;
|
|
277
|
+
lines.push(line);
|
|
278
|
+
}
|
|
279
|
+
console.debug(`Formatted ${lines.length} elements (top ${this._selector.byImportance} by importance + top ${this._selector.fromDominantGroup} from dominant group + top ${this._selector.byPosition} by position)`);
|
|
280
|
+
return lines.join('\n');
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Wait for Sentience extension to be ready in the browser.
|
|
284
|
+
*
|
|
285
|
+
* Polls window.sentience until it's defined or timeout is reached.
|
|
286
|
+
*
|
|
287
|
+
* @param backend - Browser backend with eval() method
|
|
288
|
+
* @param timeoutMs - Maximum time to wait in milliseconds
|
|
289
|
+
* @param pollIntervalMs - Interval between polls in milliseconds
|
|
290
|
+
* @returns true if extension is ready, false if timeout
|
|
291
|
+
*/
|
|
292
|
+
async _waitForExtension(backend, timeoutMs = 5000, pollIntervalMs = 100) {
|
|
293
|
+
let elapsedMs = 0;
|
|
294
|
+
while (elapsedMs < timeoutMs) {
|
|
295
|
+
try {
|
|
296
|
+
const result = await backend.eval("typeof window.sentience !== 'undefined'");
|
|
297
|
+
if (result === true) {
|
|
298
|
+
console.debug(`Sentience extension ready after ${elapsedMs}ms`);
|
|
299
|
+
return true;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
303
|
+
// Extension not ready yet, continue polling
|
|
304
|
+
}
|
|
305
|
+
await this._sleep(pollIntervalMs);
|
|
306
|
+
elapsedMs += pollIntervalMs;
|
|
307
|
+
}
|
|
308
|
+
console.warn(`Sentience extension not ready after ${timeoutMs}ms timeout`);
|
|
309
|
+
return false;
|
|
310
|
+
}
|
|
311
|
+
/**
|
|
312
|
+
* Compress href into a short token for minimal tokens.
|
|
313
|
+
*
|
|
314
|
+
* @param href - Full URL or undefined
|
|
315
|
+
* @returns Short token (domain second-level or last path segment)
|
|
316
|
+
*/
|
|
317
|
+
_compressHref(href) {
|
|
318
|
+
if (!href) {
|
|
319
|
+
return '';
|
|
320
|
+
}
|
|
321
|
+
try {
|
|
322
|
+
// Check if it's a full URL
|
|
323
|
+
if (href.startsWith('http://') || href.startsWith('https://')) {
|
|
324
|
+
const url = new URL(href);
|
|
325
|
+
if (url.hostname) {
|
|
326
|
+
// Extract second-level domain (e.g., "github" from "github.com")
|
|
327
|
+
const parts = url.hostname.split('.');
|
|
328
|
+
if (parts.length >= 2) {
|
|
329
|
+
return parts[parts.length - 2].slice(0, 10);
|
|
330
|
+
}
|
|
331
|
+
return url.hostname.slice(0, 10);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
// Handle relative URLs - use last path segment
|
|
335
|
+
const segments = href.split('/').filter(s => s);
|
|
336
|
+
if (segments.length > 0) {
|
|
337
|
+
return segments[segments.length - 1].slice(0, 10);
|
|
338
|
+
}
|
|
339
|
+
return 'item';
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return 'item';
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
_sleep(ms) {
|
|
346
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
347
|
+
}
|
|
348
|
+
// Expose selector for testing
|
|
349
|
+
get selector() {
|
|
350
|
+
return this._selector;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
exports.SentienceContext = SentienceContext;
|
|
354
|
+
/**
|
|
355
|
+
* Predicate rebrand alias for SentienceContext.
|
|
356
|
+
* Kept as a runtime alias to avoid breaking existing integrations.
|
|
357
|
+
*/
|
|
358
|
+
exports.PredicateContext = SentienceContext;
|
|
359
|
+
//# sourceMappingURL=sentience-context.js.map
|