agent-web-interface 4.2.0 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/browser/connection-utils.d.ts +48 -0
- package/dist/src/browser/connection-utils.d.ts.map +1 -0
- package/dist/src/browser/connection-utils.js +129 -0
- package/dist/src/browser/connection-utils.js.map +1 -0
- package/dist/src/browser/index.d.ts +3 -1
- package/dist/src/browser/index.d.ts.map +1 -1
- package/dist/src/browser/index.js +2 -1
- package/dist/src/browser/index.js.map +1 -1
- package/dist/src/browser/session-manager.d.ts +1 -89
- package/dist/src/browser/session-manager.d.ts.map +1 -1
- package/dist/src/browser/session-manager.js +1 -116
- package/dist/src/browser/session-manager.js.map +1 -1
- package/dist/src/browser/session-manager.types.d.ts +90 -0
- package/dist/src/browser/session-manager.types.d.ts.map +1 -0
- package/dist/src/browser/session-manager.types.js +7 -0
- package/dist/src/browser/session-manager.types.js.map +1 -0
- package/dist/src/form/constraint-extraction.d.ts +31 -0
- package/dist/src/form/constraint-extraction.d.ts.map +1 -0
- package/dist/src/form/constraint-extraction.js +110 -0
- package/dist/src/form/constraint-extraction.js.map +1 -0
- package/dist/src/form/field-extractor.d.ts.map +1 -1
- package/dist/src/form/field-extractor.js +3 -444
- package/dist/src/form/field-extractor.js.map +1 -1
- package/dist/src/form/field-state-extractor.d.ts +22 -0
- package/dist/src/form/field-state-extractor.d.ts.map +1 -0
- package/dist/src/form/field-state-extractor.js +55 -0
- package/dist/src/form/field-state-extractor.js.map +1 -0
- package/dist/src/form/form-actions.d.ts +45 -0
- package/dist/src/form/form-actions.d.ts.map +1 -0
- package/dist/src/form/form-actions.js +108 -0
- package/dist/src/form/form-actions.js.map +1 -0
- package/dist/src/form/form-detector.d.ts +0 -36
- package/dist/src/form/form-detector.d.ts.map +1 -1
- package/dist/src/form/form-detector.js +11 -376
- package/dist/src/form/form-detector.js.map +1 -1
- package/dist/src/form/input-clustering.d.ts +15 -0
- package/dist/src/form/input-clustering.d.ts.map +1 -0
- package/dist/src/form/input-clustering.js +61 -0
- package/dist/src/form/input-clustering.js.map +1 -0
- package/dist/src/form/intent-inference.d.ts +28 -0
- package/dist/src/form/intent-inference.d.ts.map +1 -0
- package/dist/src/form/intent-inference.js +137 -0
- package/dist/src/form/intent-inference.js.map +1 -0
- package/dist/src/form/purpose-inference.d.ts +50 -0
- package/dist/src/form/purpose-inference.d.ts.map +1 -0
- package/dist/src/form/purpose-inference.js +313 -0
- package/dist/src/form/purpose-inference.js.map +1 -0
- package/dist/src/form/submit-detection.d.ts +36 -0
- package/dist/src/form/submit-detection.d.ts.map +1 -0
- package/dist/src/form/submit-detection.js +101 -0
- package/dist/src/form/submit-detection.js.map +1 -0
- package/dist/src/form/types.d.ts +2 -2
- package/dist/src/index.js +65 -48
- package/dist/src/index.js.map +1 -1
- package/dist/src/observation/observation-accumulator.d.ts +1 -1
- package/dist/src/observation/observation-accumulator.js +1 -1
- package/dist/src/observation/observer-script.d.ts +1 -1
- package/dist/src/observation/observer-script.d.ts.map +1 -1
- package/dist/src/observation/observer-script.js +129 -7
- package/dist/src/observation/observer-script.js.map +1 -1
- package/dist/src/query/disambiguation.d.ts +18 -0
- package/dist/src/query/disambiguation.d.ts.map +1 -0
- package/dist/src/query/disambiguation.js +123 -0
- package/dist/src/query/disambiguation.js.map +1 -0
- package/dist/src/query/fuzzy-match.d.ts +17 -0
- package/dist/src/query/fuzzy-match.d.ts.map +1 -0
- package/dist/src/query/fuzzy-match.js +34 -0
- package/dist/src/query/fuzzy-match.js.map +1 -0
- package/dist/src/query/index.d.ts +3 -0
- package/dist/src/query/index.d.ts.map +1 -1
- package/dist/src/query/index.js +6 -0
- package/dist/src/query/index.js.map +1 -1
- package/dist/src/query/query-engine.d.ts +0 -35
- package/dist/src/query/query-engine.d.ts.map +1 -1
- package/dist/src/query/query-engine.js +9 -309
- package/dist/src/query/query-engine.js.map +1 -1
- package/dist/src/query/scoring.d.ts +52 -0
- package/dist/src/query/scoring.d.ts.map +1 -0
- package/dist/src/query/scoring.js +162 -0
- package/dist/src/query/scoring.js.map +1 -0
- package/dist/src/server/mcp-server.d.ts.map +1 -1
- package/dist/src/server/mcp-server.js +29 -1
- package/dist/src/server/mcp-server.js.map +1 -1
- package/dist/src/snapshot/element-resolver.d.ts +50 -18
- package/dist/src/snapshot/element-resolver.d.ts.map +1 -1
- package/dist/src/snapshot/element-resolver.js +180 -101
- package/dist/src/snapshot/element-resolver.js.map +1 -1
- package/dist/src/snapshot/extractors/ax-extractor.d.ts +1 -1
- package/dist/src/snapshot/extractors/ax-extractor.d.ts.map +1 -1
- package/dist/src/snapshot/extractors/ax-extractor.js +4 -1
- package/dist/src/snapshot/extractors/ax-extractor.js.map +1 -1
- package/dist/src/snapshot/extractors/index.d.ts +1 -1
- package/dist/src/snapshot/extractors/index.d.ts.map +1 -1
- package/dist/src/snapshot/extractors/index.js +1 -1
- package/dist/src/snapshot/extractors/index.js.map +1 -1
- package/dist/src/snapshot/extractors/region-resolver.d.ts.map +1 -1
- package/dist/src/snapshot/extractors/region-resolver.js +8 -0
- package/dist/src/snapshot/extractors/region-resolver.js.map +1 -1
- package/dist/src/snapshot/extractors/types.d.ts +8 -0
- package/dist/src/snapshot/extractors/types.d.ts.map +1 -1
- package/dist/src/snapshot/extractors/types.js +16 -0
- package/dist/src/snapshot/extractors/types.js.map +1 -1
- package/dist/src/snapshot/frame-context.d.ts +68 -0
- package/dist/src/snapshot/frame-context.d.ts.map +1 -0
- package/dist/src/snapshot/frame-context.js +131 -0
- package/dist/src/snapshot/frame-context.js.map +1 -0
- package/dist/src/snapshot/heading-index.d.ts +28 -0
- package/dist/src/snapshot/heading-index.d.ts.map +1 -0
- package/dist/src/snapshot/heading-index.js +108 -0
- package/dist/src/snapshot/heading-index.js.map +1 -0
- package/dist/src/snapshot/index.d.ts +5 -3
- package/dist/src/snapshot/index.d.ts.map +1 -1
- package/dist/src/snapshot/index.js +3 -2
- package/dist/src/snapshot/index.js.map +1 -1
- package/dist/src/snapshot/kind-mapping.d.ts +30 -0
- package/dist/src/snapshot/kind-mapping.d.ts.map +1 -0
- package/dist/src/snapshot/kind-mapping.js +114 -0
- package/dist/src/snapshot/kind-mapping.js.map +1 -0
- package/dist/src/snapshot/node-filter.d.ts +31 -0
- package/dist/src/snapshot/node-filter.d.ts.map +1 -0
- package/dist/src/snapshot/node-filter.js +137 -0
- package/dist/src/snapshot/node-filter.js.map +1 -0
- package/dist/src/snapshot/node-synthesizer.d.ts +62 -0
- package/dist/src/snapshot/node-synthesizer.d.ts.map +1 -0
- package/dist/src/snapshot/node-synthesizer.js +185 -0
- package/dist/src/snapshot/node-synthesizer.js.map +1 -0
- package/dist/src/snapshot/snapshot-compiler.d.ts +2 -36
- package/dist/src/snapshot/snapshot-compiler.d.ts.map +1 -1
- package/dist/src/snapshot/snapshot-compiler.js +28 -520
- package/dist/src/snapshot/snapshot-compiler.js.map +1 -1
- package/dist/src/snapshot/snapshot.types.d.ts +7 -2
- package/dist/src/snapshot/snapshot.types.d.ts.map +1 -1
- package/dist/src/snapshot/snapshot.types.js +9 -0
- package/dist/src/snapshot/snapshot.types.js.map +1 -1
- package/dist/src/state/actionables-filter.d.ts +5 -0
- package/dist/src/state/actionables-filter.d.ts.map +1 -1
- package/dist/src/state/actionables-filter.js +22 -3
- package/dist/src/state/actionables-filter.js.map +1 -1
- package/dist/src/state/diff-engine.js +3 -3
- package/dist/src/state/diff-engine.js.map +1 -1
- package/dist/src/state/element-registry.d.ts.map +1 -1
- package/dist/src/state/element-registry.js +6 -4
- package/dist/src/state/element-registry.js.map +1 -1
- package/dist/src/state/hash-utils.d.ts +24 -0
- package/dist/src/state/hash-utils.d.ts.map +1 -0
- package/dist/src/state/hash-utils.js +41 -0
- package/dist/src/state/hash-utils.js.map +1 -0
- package/dist/src/state/layer-detector.d.ts.map +1 -1
- package/dist/src/state/layer-detector.js +15 -286
- package/dist/src/state/layer-detector.js.map +1 -1
- package/dist/src/state/layer-detectors/drawer-detector.d.ts +32 -0
- package/dist/src/state/layer-detectors/drawer-detector.d.ts.map +1 -0
- package/dist/src/state/layer-detectors/drawer-detector.js +96 -0
- package/dist/src/state/layer-detectors/drawer-detector.js.map +1 -0
- package/dist/src/state/layer-detectors/index.d.ts +10 -0
- package/dist/src/state/layer-detectors/index.d.ts.map +1 -0
- package/dist/src/state/layer-detectors/index.js +10 -0
- package/dist/src/state/layer-detectors/index.js.map +1 -0
- package/dist/src/state/layer-detectors/modal-detector.d.ts +30 -0
- package/dist/src/state/layer-detectors/modal-detector.d.ts.map +1 -0
- package/dist/src/state/layer-detectors/modal-detector.js +127 -0
- package/dist/src/state/layer-detectors/modal-detector.js.map +1 -0
- package/dist/src/state/layer-detectors/popover-detector.d.ts +20 -0
- package/dist/src/state/layer-detectors/popover-detector.d.ts.map +1 -0
- package/dist/src/state/layer-detectors/popover-detector.js +76 -0
- package/dist/src/state/layer-detectors/popover-detector.js.map +1 -0
- package/dist/src/state/layer-detectors/toast-detector.d.ts +24 -0
- package/dist/src/state/layer-detectors/toast-detector.d.ts.map +1 -0
- package/dist/src/state/layer-detectors/toast-detector.js +48 -0
- package/dist/src/state/layer-detectors/toast-detector.js.map +1 -0
- package/dist/src/state/region-mapping.d.ts +13 -0
- package/dist/src/state/region-mapping.d.ts.map +1 -0
- package/dist/src/state/region-mapping.js +25 -0
- package/dist/src/state/region-mapping.js.map +1 -0
- package/dist/src/state/state-manager.d.ts.map +1 -1
- package/dist/src/state/state-manager.js +8 -192
- package/dist/src/state/state-manager.js.map +1 -1
- package/dist/src/state/state-renderer.d.ts.map +1 -1
- package/dist/src/state/state-renderer.js +16 -2
- package/dist/src/state/state-renderer.js.map +1 -1
- package/dist/src/state/types.d.ts +8 -4
- package/dist/src/state/types.d.ts.map +1 -1
- package/dist/src/state/url-sanitization.d.ts +22 -0
- package/dist/src/state/url-sanitization.d.ts.map +1 -0
- package/dist/src/state/url-sanitization.js +60 -0
- package/dist/src/state/url-sanitization.js.map +1 -0
- package/dist/src/state/value-masking.d.ts +36 -0
- package/dist/src/state/value-masking.d.ts.map +1 -0
- package/dist/src/state/value-masking.js +86 -0
- package/dist/src/state/value-masking.js.map +1 -0
- package/dist/src/tools/action-context.d.ts +60 -0
- package/dist/src/tools/action-context.d.ts.map +1 -0
- package/dist/src/tools/action-context.js +78 -0
- package/dist/src/tools/action-context.js.map +1 -0
- package/dist/src/tools/action-stabilization.d.ts +48 -0
- package/dist/src/tools/action-stabilization.d.ts.map +1 -0
- package/dist/src/tools/action-stabilization.js +87 -0
- package/dist/src/tools/action-stabilization.js.map +1 -0
- package/dist/src/tools/browser-tools.d.ts +8 -146
- package/dist/src/tools/browser-tools.d.ts.map +1 -1
- package/dist/src/tools/browser-tools.js +13 -689
- package/dist/src/tools/browser-tools.js.map +1 -1
- package/dist/src/tools/canvas-tools.d.ts +32 -0
- package/dist/src/tools/canvas-tools.d.ts.map +1 -0
- package/dist/src/tools/canvas-tools.js +370 -0
- package/dist/src/tools/canvas-tools.js.map +1 -0
- package/dist/src/tools/effect-tracker.d.ts +25 -0
- package/dist/src/tools/effect-tracker.d.ts.map +1 -0
- package/dist/src/tools/effect-tracker.js +69 -0
- package/dist/src/tools/effect-tracker.js.map +1 -0
- package/dist/src/tools/execute-action.d.ts +1 -31
- package/dist/src/tools/execute-action.d.ts.map +1 -1
- package/dist/src/tools/execute-action.js +7 -276
- package/dist/src/tools/execute-action.js.map +1 -1
- package/dist/src/tools/form-tools.d.ts +4 -6
- package/dist/src/tools/form-tools.d.ts.map +1 -1
- package/dist/src/tools/form-tools.js +10 -42
- package/dist/src/tools/form-tools.js.map +1 -1
- package/dist/src/tools/index.d.ts +6 -4
- package/dist/src/tools/index.d.ts.map +1 -1
- package/dist/src/tools/index.js +21 -10
- package/dist/src/tools/index.js.map +1 -1
- package/dist/src/tools/interaction-tools.d.ts +46 -0
- package/dist/src/tools/interaction-tools.d.ts.map +1 -0
- package/dist/src/tools/interaction-tools.js +138 -0
- package/dist/src/tools/interaction-tools.js.map +1 -0
- package/dist/src/tools/navigation-detection.d.ts +31 -0
- package/dist/src/tools/navigation-detection.d.ts.map +1 -0
- package/dist/src/tools/navigation-detection.js +46 -0
- package/dist/src/tools/navigation-detection.js.map +1 -0
- package/dist/src/tools/navigation-tools.d.ts +57 -0
- package/dist/src/tools/navigation-tools.d.ts.map +1 -0
- package/dist/src/tools/navigation-tools.js +178 -0
- package/dist/src/tools/navigation-tools.js.map +1 -0
- package/dist/src/tools/observation-tools.d.ts +53 -0
- package/dist/src/tools/observation-tools.d.ts.map +1 -0
- package/dist/src/tools/observation-tools.js +247 -0
- package/dist/src/tools/observation-tools.js.map +1 -0
- package/dist/src/tools/response-builder.js +2 -2
- package/dist/src/tools/response-builder.js.map +1 -1
- package/dist/src/tools/stale-element-retry.d.ts +37 -0
- package/dist/src/tools/stale-element-retry.d.ts.map +1 -0
- package/dist/src/tools/stale-element-retry.js +68 -0
- package/dist/src/tools/stale-element-retry.js.map +1 -0
- package/dist/src/tools/state-manager-registry.d.ts +26 -0
- package/dist/src/tools/state-manager-registry.d.ts.map +1 -0
- package/dist/src/tools/state-manager-registry.js +39 -0
- package/dist/src/tools/state-manager-registry.js.map +1 -0
- package/dist/src/tools/tool-context.d.ts +53 -0
- package/dist/src/tools/tool-context.d.ts.map +1 -0
- package/dist/src/tools/tool-context.js +119 -0
- package/dist/src/tools/tool-context.js.map +1 -0
- package/dist/src/tools/tool-result.types.d.ts +16 -1
- package/dist/src/tools/tool-result.types.d.ts.map +1 -1
- package/dist/src/tools/tool-result.types.js +11 -0
- package/dist/src/tools/tool-result.types.js.map +1 -1
- package/dist/src/tools/tool-schemas.d.ts +358 -146
- package/dist/src/tools/tool-schemas.d.ts.map +1 -1
- package/dist/src/tools/tool-schemas.js +142 -19
- package/dist/src/tools/tool-schemas.js.map +1 -1
- package/dist/src/tools/viewport-tools.d.ts +36 -0
- package/dist/src/tools/viewport-tools.d.ts.map +1 -0
- package/dist/src/tools/viewport-tools.js +105 -0
- package/dist/src/tools/viewport-tools.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Form Actions
|
|
3
|
+
*
|
|
4
|
+
* Extracts action buttons (submit, cancel, reset, etc.) associated
|
|
5
|
+
* with form regions using spatial proximity and keyword matching.
|
|
6
|
+
*
|
|
7
|
+
* @module form/form-actions
|
|
8
|
+
*/
|
|
9
|
+
import { isSubmitButton } from './submit-detection.js';
|
|
10
|
+
/**
|
|
11
|
+
* Interactive input kinds
|
|
12
|
+
*/
|
|
13
|
+
export const INPUT_KINDS = new Set([
|
|
14
|
+
'input',
|
|
15
|
+
'textarea',
|
|
16
|
+
'select',
|
|
17
|
+
'combobox',
|
|
18
|
+
'checkbox',
|
|
19
|
+
'radio',
|
|
20
|
+
'switch',
|
|
21
|
+
'slider',
|
|
22
|
+
]);
|
|
23
|
+
/**
|
|
24
|
+
* Button kinds that could be submit buttons
|
|
25
|
+
*/
|
|
26
|
+
export const BUTTON_KINDS = new Set(['button']);
|
|
27
|
+
/**
|
|
28
|
+
* Signal weights for form detection
|
|
29
|
+
*/
|
|
30
|
+
export const SIGNAL_WEIGHTS = {
|
|
31
|
+
form_tag: 0.5,
|
|
32
|
+
role_form: 0.45,
|
|
33
|
+
role_search: 0.4,
|
|
34
|
+
fieldset: 0.3,
|
|
35
|
+
input_cluster: 0.25,
|
|
36
|
+
label_input_pairs: 0.2,
|
|
37
|
+
submit_button: 0.3,
|
|
38
|
+
form_keywords: 0.15,
|
|
39
|
+
naming_pattern: 0.1,
|
|
40
|
+
};
|
|
41
|
+
/**
|
|
42
|
+
* Extract form action buttons.
|
|
43
|
+
*/
|
|
44
|
+
export function extractFormActions(snapshot, candidate, computeClusterBbox) {
|
|
45
|
+
const actions = [];
|
|
46
|
+
const buttons = snapshot.nodes.filter((n) => BUTTON_KINDS.has(n.kind));
|
|
47
|
+
for (const button of buttons) {
|
|
48
|
+
// Skip disabled buttons for now but still include them
|
|
49
|
+
const isSubmit = isSubmitButton(button);
|
|
50
|
+
const isNearForm = candidate.bbox
|
|
51
|
+
? isButtonNearBbox(button, candidate.bbox)
|
|
52
|
+
: candidate.field_eids.length === 0 ||
|
|
53
|
+
isButtonNearFields(button, snapshot, candidate.field_eids, computeClusterBbox);
|
|
54
|
+
if (!isNearForm)
|
|
55
|
+
continue;
|
|
56
|
+
// Determine action type
|
|
57
|
+
let type = 'action';
|
|
58
|
+
const label = button.label.toLowerCase();
|
|
59
|
+
if (isSubmit) {
|
|
60
|
+
type = 'submit';
|
|
61
|
+
}
|
|
62
|
+
else if (label.includes('cancel') || label.includes('close')) {
|
|
63
|
+
type = 'cancel';
|
|
64
|
+
}
|
|
65
|
+
else if (label.includes('back') || label.includes('previous')) {
|
|
66
|
+
type = 'back';
|
|
67
|
+
}
|
|
68
|
+
else if (label.includes('next') || label.includes('continue')) {
|
|
69
|
+
type = 'next';
|
|
70
|
+
}
|
|
71
|
+
else if (label.includes('reset') || label.includes('clear')) {
|
|
72
|
+
type = 'reset';
|
|
73
|
+
}
|
|
74
|
+
actions.push({
|
|
75
|
+
eid: button.node_id,
|
|
76
|
+
backend_node_id: button.backend_node_id,
|
|
77
|
+
label: button.label,
|
|
78
|
+
type,
|
|
79
|
+
enabled: button.state?.enabled ?? true,
|
|
80
|
+
is_primary: isSubmit,
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
return actions;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Check if a button is near a bounding box.
|
|
87
|
+
*/
|
|
88
|
+
export function isButtonNearBbox(button, bbox) {
|
|
89
|
+
if (!button.layout?.bbox)
|
|
90
|
+
return false;
|
|
91
|
+
const btnBbox = button.layout.bbox;
|
|
92
|
+
const isNearX = btnBbox.x >= bbox.x - 100 && btnBbox.x <= bbox.x + bbox.width + 100;
|
|
93
|
+
const isNearY = btnBbox.y >= bbox.y - 50 && btnBbox.y <= bbox.y + bbox.height + 150;
|
|
94
|
+
return isNearX && isNearY;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Check if a button is near a set of fields.
|
|
98
|
+
*/
|
|
99
|
+
export function isButtonNearFields(button, snapshot, fieldEids, computeClusterBbox) {
|
|
100
|
+
const fieldNodes = fieldEids
|
|
101
|
+
.map((eid) => snapshot.nodes.find((n) => n.node_id === eid))
|
|
102
|
+
.filter((n) => n !== undefined);
|
|
103
|
+
const clusterBbox = computeClusterBbox(fieldNodes);
|
|
104
|
+
if (!clusterBbox)
|
|
105
|
+
return false;
|
|
106
|
+
return isButtonNearBbox(button, clusterBbox);
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=form-actions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"form-actions.js","sourceRoot":"","sources":["../../../src/form/form-actions.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAIH,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEvD;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAW;IAC3C,OAAO;IACP,UAAU;IACV,QAAQ;IACR,UAAU;IACV,UAAU;IACV,OAAO;IACP,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC;AAEH;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;AAE1D;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAuC;IAChE,QAAQ,EAAE,GAAG;IACb,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,GAAG;IAChB,QAAQ,EAAE,GAAG;IACb,aAAa,EAAE,IAAI;IACnB,iBAAiB,EAAE,GAAG;IACtB,aAAa,EAAE,GAAG;IAClB,aAAa,EAAE,IAAI;IACnB,cAAc,EAAE,GAAG;CACpB,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,QAAsB,EACtB,SAAwB,EACxB,kBAEwE;IAExE,MAAM,OAAO,GAA0B,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAEvE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,uDAAuD;QACvD,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI;YAC/B,CAAC,CAAC,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC;YAC1C,CAAC,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;gBACjC,kBAAkB,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;QAEnF,IAAI,CAAC,UAAU;YAAE,SAAS;QAE1B,wBAAwB;QACxB,IAAI,IAAI,GAAqC,QAAQ,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC;QAEzC,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,QAAQ,CAAC;QAClB,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/D,IAAI,GAAG,QAAQ,CAAC;QAClB,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAChE,IAAI,GAAG,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;YAChE,IAAI,GAAG,MAAM,CAAC;QAChB,CAAC;aAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9D,IAAI,GAAG,OAAO,CAAC;QACjB,CAAC;QAED,OAAO,CAAC,IAAI,CAAC;YACX,GAAG,EAAE,MAAM,CAAC,OAAO;YACnB,eAAe,EAAE,MAAM,CAAC,eAAe;YACvC,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI;YACJ,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI;YACtC,UAAU,EAAE,QAAQ;SACrB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAC9B,MAAoB,EACpB,IAAwC;IAExC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;IAEnC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;IACpF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;IAEpF,OAAO,OAAO,IAAI,OAAO,CAAC;AAC5B,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAChC,MAAoB,EACpB,QAAsB,EACtB,SAAmB,EACnB,kBAEwE;IAExE,MAAM,UAAU,GAAG,SAAS;SACzB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAC,CAAC;SAC3D,MAAM,CAAC,CAAC,CAAC,EAAqB,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC;IAErD,MAAM,WAAW,GAAG,kBAAkB,CAAC,UAAU,CAAC,CAAC;IACnD,IAAI,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IAE/B,OAAO,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;AAC/C,CAAC"}
|
|
@@ -35,38 +35,14 @@ export declare class FormDetector {
|
|
|
35
35
|
* Detect implicit forms (input clusters without form tag)
|
|
36
36
|
*/
|
|
37
37
|
private detectImplicitForms;
|
|
38
|
-
/**
|
|
39
|
-
* Cluster input nodes by proximity and structural context.
|
|
40
|
-
*/
|
|
41
|
-
private clusterInputs;
|
|
42
38
|
/**
|
|
43
39
|
* Check if a node is likely within a form's scope.
|
|
44
40
|
*/
|
|
45
41
|
private isNodeWithinForm;
|
|
46
|
-
/**
|
|
47
|
-
* Find a submit button associated with a form.
|
|
48
|
-
*/
|
|
49
|
-
private findSubmitButton;
|
|
50
|
-
/**
|
|
51
|
-
* Find a submit button near a cluster of inputs.
|
|
52
|
-
*/
|
|
53
|
-
private findSubmitButtonNearCluster;
|
|
54
|
-
/**
|
|
55
|
-
* Check if a button looks like a submit button.
|
|
56
|
-
*/
|
|
57
|
-
private isSubmitButton;
|
|
58
42
|
/**
|
|
59
43
|
* Compute confidence score from signals.
|
|
60
44
|
*/
|
|
61
45
|
private computeConfidence;
|
|
62
|
-
/**
|
|
63
|
-
* Infer the intent of a form.
|
|
64
|
-
*/
|
|
65
|
-
private inferIntent;
|
|
66
|
-
/**
|
|
67
|
-
* Check if text contains any of the given keywords.
|
|
68
|
-
*/
|
|
69
|
-
private hasIntentKeywords;
|
|
70
46
|
/**
|
|
71
47
|
* Compute bounding box for a cluster of nodes.
|
|
72
48
|
*/
|
|
@@ -79,18 +55,6 @@ export declare class FormDetector {
|
|
|
79
55
|
* Generate a unique form ID.
|
|
80
56
|
*/
|
|
81
57
|
private generateFormId;
|
|
82
|
-
/**
|
|
83
|
-
* Extract form action buttons.
|
|
84
|
-
*/
|
|
85
|
-
private extractFormActions;
|
|
86
|
-
/**
|
|
87
|
-
* Check if a button is near a bounding box.
|
|
88
|
-
*/
|
|
89
|
-
private isButtonNearBbox;
|
|
90
|
-
/**
|
|
91
|
-
* Check if a button is near a set of fields.
|
|
92
|
-
*/
|
|
93
|
-
private isButtonNearFields;
|
|
94
58
|
/**
|
|
95
59
|
* Infer form pattern (single page, multi-step, etc.)
|
|
96
60
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"form-detector.d.ts","sourceRoot":"","sources":["../../../src/form/form-detector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,
|
|
1
|
+
{"version":3,"file":"form-detector.d.ts","sourceRoot":"","sources":["../../../src/form/form-detector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAgB,MAAM,+BAA+B,CAAC;AAChF,OAAO,KAAK,EACV,UAAU,EAIV,mBAAmB,EACpB,MAAM,YAAY,CAAC;AAUpB;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;gBAEjC,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC;IAIjD;;;;;OAKG;IACH,MAAM,CAAC,QAAQ,EAAE,YAAY,GAAG,UAAU,EAAE;IA8B5C;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA+F3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAsF3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAyCxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAYzB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA6B1B;;OAEG;IACH,OAAO,CAAC,eAAe;IAuCvB;;OAEG;IACH,OAAO,CAAC,cAAc;IAUtB;;OAEG;IACH,OAAO,CAAC,YAAY;CAKrB;AAED;;GAEG;AACH,wBAAgB,WAAW,CACzB,QAAQ,EAAE,YAAY,EACtB,MAAM,CAAC,EAAE,OAAO,CAAC,mBAAmB,CAAC,GACpC,UAAU,EAAE,CAGd"}
|
|
@@ -16,136 +16,10 @@ import { DEFAULT_FORM_DETECTION_CONFIG } from './types.js';
|
|
|
16
16
|
import { extractFields } from './field-extractor.js';
|
|
17
17
|
import { computeFormState } from './form-state.js';
|
|
18
18
|
import { createHash } from 'crypto';
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
'input',
|
|
24
|
-
'textarea',
|
|
25
|
-
'select',
|
|
26
|
-
'combobox',
|
|
27
|
-
'checkbox',
|
|
28
|
-
'radio',
|
|
29
|
-
'switch',
|
|
30
|
-
'slider',
|
|
31
|
-
]);
|
|
32
|
-
/**
|
|
33
|
-
* Button kinds that could be submit buttons
|
|
34
|
-
*/
|
|
35
|
-
const BUTTON_KINDS = new Set(['button']);
|
|
36
|
-
/**
|
|
37
|
-
* Signal weights for form detection
|
|
38
|
-
*/
|
|
39
|
-
const SIGNAL_WEIGHTS = {
|
|
40
|
-
form_tag: 0.5,
|
|
41
|
-
role_form: 0.45,
|
|
42
|
-
role_search: 0.4,
|
|
43
|
-
fieldset: 0.3,
|
|
44
|
-
input_cluster: 0.25,
|
|
45
|
-
label_input_pairs: 0.2,
|
|
46
|
-
submit_button: 0.3,
|
|
47
|
-
form_keywords: 0.15,
|
|
48
|
-
naming_pattern: 0.1,
|
|
49
|
-
};
|
|
50
|
-
/**
|
|
51
|
-
* Weighted keywords that indicate form intent.
|
|
52
|
-
* Higher weight = more explicit signal (e.g., "create account" is more explicitly signup than "email")
|
|
53
|
-
* Lower weight = ambiguous signal that appears in multiple form types
|
|
54
|
-
*/
|
|
55
|
-
const INTENT_KEYWORDS = {
|
|
56
|
-
login: [
|
|
57
|
-
{ keyword: 'log in', weight: 3 },
|
|
58
|
-
{ keyword: 'login', weight: 3 },
|
|
59
|
-
{ keyword: 'sign in', weight: 3 },
|
|
60
|
-
{ keyword: 'signin', weight: 3 },
|
|
61
|
-
// These are ambiguous - they appear in both login and signup forms
|
|
62
|
-
{ keyword: 'email', weight: 0.5 },
|
|
63
|
-
{ keyword: 'password', weight: 0.5 },
|
|
64
|
-
{ keyword: 'username', weight: 0.5 },
|
|
65
|
-
],
|
|
66
|
-
signup: [
|
|
67
|
-
{ keyword: 'sign up', weight: 3 },
|
|
68
|
-
{ keyword: 'signup', weight: 3 },
|
|
69
|
-
{ keyword: 'register', weight: 3 },
|
|
70
|
-
{ keyword: 'create account', weight: 3 },
|
|
71
|
-
{ keyword: 'join', weight: 2 },
|
|
72
|
-
],
|
|
73
|
-
search: [
|
|
74
|
-
{ keyword: 'search', weight: 3 },
|
|
75
|
-
{ keyword: 'find', weight: 2 },
|
|
76
|
-
{ keyword: 'lookup', weight: 2 },
|
|
77
|
-
{ keyword: 'query', weight: 2 },
|
|
78
|
-
],
|
|
79
|
-
checkout: [
|
|
80
|
-
{ keyword: 'checkout', weight: 3 },
|
|
81
|
-
{ keyword: 'payment', weight: 2 },
|
|
82
|
-
{ keyword: 'order', weight: 2 },
|
|
83
|
-
{ keyword: 'purchase', weight: 2 },
|
|
84
|
-
{ keyword: 'buy now', weight: 3 },
|
|
85
|
-
],
|
|
86
|
-
filter: [
|
|
87
|
-
{ keyword: 'filter', weight: 3 },
|
|
88
|
-
{ keyword: 'sort', weight: 2 },
|
|
89
|
-
{ keyword: 'refine', weight: 2 },
|
|
90
|
-
{ keyword: 'narrow', weight: 2 },
|
|
91
|
-
],
|
|
92
|
-
settings: [
|
|
93
|
-
{ keyword: 'settings', weight: 3 },
|
|
94
|
-
{ keyword: 'preferences', weight: 2 },
|
|
95
|
-
{ keyword: 'configuration', weight: 2 },
|
|
96
|
-
{ keyword: 'options', weight: 1 },
|
|
97
|
-
],
|
|
98
|
-
contact: [
|
|
99
|
-
{ keyword: 'contact', weight: 3 },
|
|
100
|
-
{ keyword: 'message', weight: 1 },
|
|
101
|
-
{ keyword: 'feedback', weight: 2 },
|
|
102
|
-
{ keyword: 'inquiry', weight: 2 },
|
|
103
|
-
],
|
|
104
|
-
subscribe: [
|
|
105
|
-
{ keyword: 'subscribe', weight: 3 },
|
|
106
|
-
{ keyword: 'newsletter', weight: 3 },
|
|
107
|
-
{ keyword: 'email updates', weight: 2 },
|
|
108
|
-
],
|
|
109
|
-
shipping: [
|
|
110
|
-
{ keyword: 'shipping', weight: 3 },
|
|
111
|
-
{ keyword: 'delivery', weight: 2 },
|
|
112
|
-
{ keyword: 'address', weight: 1 },
|
|
113
|
-
],
|
|
114
|
-
payment: [
|
|
115
|
-
{ keyword: 'payment', weight: 3 },
|
|
116
|
-
{ keyword: 'credit card', weight: 3 },
|
|
117
|
-
{ keyword: 'billing', weight: 2 },
|
|
118
|
-
{ keyword: 'card number', weight: 3 },
|
|
119
|
-
],
|
|
120
|
-
profile: [
|
|
121
|
-
{ keyword: 'profile', weight: 3 },
|
|
122
|
-
{ keyword: 'account', weight: 1 },
|
|
123
|
-
{ keyword: 'personal info', weight: 2 },
|
|
124
|
-
],
|
|
125
|
-
unknown: [],
|
|
126
|
-
};
|
|
127
|
-
/**
|
|
128
|
-
* Submit button keywords
|
|
129
|
-
*/
|
|
130
|
-
const SUBMIT_KEYWORDS = [
|
|
131
|
-
'submit',
|
|
132
|
-
'send',
|
|
133
|
-
'continue',
|
|
134
|
-
'next',
|
|
135
|
-
'save',
|
|
136
|
-
'apply',
|
|
137
|
-
'confirm',
|
|
138
|
-
'add to',
|
|
139
|
-
'sign in',
|
|
140
|
-
'log in',
|
|
141
|
-
'sign up',
|
|
142
|
-
'register',
|
|
143
|
-
'search',
|
|
144
|
-
'buy',
|
|
145
|
-
'checkout',
|
|
146
|
-
'purchase',
|
|
147
|
-
'subscribe',
|
|
148
|
-
];
|
|
19
|
+
import { inferIntent, hasIntentKeywords, INTENT_KEYWORDS } from './intent-inference.js';
|
|
20
|
+
import { findSubmitButton, findSubmitButtonNearCluster } from './submit-detection.js';
|
|
21
|
+
import { clusterInputs } from './input-clustering.js';
|
|
22
|
+
import { extractFormActions, INPUT_KINDS, SIGNAL_WEIGHTS } from './form-actions.js';
|
|
149
23
|
/**
|
|
150
24
|
* Form Detector class
|
|
151
25
|
*/
|
|
@@ -232,7 +106,7 @@ export class FormDetector {
|
|
|
232
106
|
});
|
|
233
107
|
}
|
|
234
108
|
// Check for submit button
|
|
235
|
-
const submitButton =
|
|
109
|
+
const submitButton = findSubmitButton(snapshot, formNode, fieldEids, this.isNodeWithinForm.bind(this), this.computeClusterBbox.bind(this));
|
|
236
110
|
if (submitButton) {
|
|
237
111
|
signals.push({
|
|
238
112
|
type: 'submit_button',
|
|
@@ -243,7 +117,7 @@ export class FormDetector {
|
|
|
243
117
|
// Compute confidence
|
|
244
118
|
const confidence = this.computeConfidence(signals);
|
|
245
119
|
// Infer intent
|
|
246
|
-
const intent =
|
|
120
|
+
const intent = inferIntent(snapshot, fieldEids, formNode);
|
|
247
121
|
candidates.push({
|
|
248
122
|
root_node_id: formNode.node_id,
|
|
249
123
|
root_backend_node_id: formNode.backend_node_id,
|
|
@@ -274,7 +148,7 @@ export class FormDetector {
|
|
|
274
148
|
return candidates;
|
|
275
149
|
}
|
|
276
150
|
// Group inputs by proximity and structural context
|
|
277
|
-
const clusters =
|
|
151
|
+
const clusters = clusterInputs(unclaimedInputs, snapshot, this.config);
|
|
278
152
|
for (const cluster of clusters) {
|
|
279
153
|
if (cluster.length < 1)
|
|
280
154
|
continue;
|
|
@@ -298,7 +172,7 @@ export class FormDetector {
|
|
|
298
172
|
const allKeywords = Object.values(INTENT_KEYWORDS)
|
|
299
173
|
.flat()
|
|
300
174
|
.map((entry) => entry.keyword);
|
|
301
|
-
const hasFormKeywords = cluster.some((n) =>
|
|
175
|
+
const hasFormKeywords = cluster.some((n) => hasIntentKeywords(n.label, allKeywords));
|
|
302
176
|
if (hasFormKeywords) {
|
|
303
177
|
signals.push({
|
|
304
178
|
type: 'form_keywords',
|
|
@@ -308,7 +182,7 @@ export class FormDetector {
|
|
|
308
182
|
}
|
|
309
183
|
// Check for submit button near cluster
|
|
310
184
|
const fieldEids = cluster.map((n) => n.node_id);
|
|
311
|
-
const submitButton =
|
|
185
|
+
const submitButton = findSubmitButtonNearCluster(snapshot, cluster, this.computeClusterBbox.bind(this));
|
|
312
186
|
if (submitButton) {
|
|
313
187
|
signals.push({
|
|
314
188
|
type: 'submit_button',
|
|
@@ -319,7 +193,7 @@ export class FormDetector {
|
|
|
319
193
|
// Compute confidence
|
|
320
194
|
const confidence = this.computeConfidence(signals);
|
|
321
195
|
// Infer intent
|
|
322
|
-
const intent =
|
|
196
|
+
const intent = inferIntent(snapshot, fieldEids, undefined);
|
|
323
197
|
// Compute bounding box from cluster
|
|
324
198
|
const bbox = this.computeClusterBbox(cluster);
|
|
325
199
|
candidates.push({
|
|
@@ -332,58 +206,6 @@ export class FormDetector {
|
|
|
332
206
|
}
|
|
333
207
|
return candidates;
|
|
334
208
|
}
|
|
335
|
-
/**
|
|
336
|
-
* Cluster input nodes by proximity and structural context.
|
|
337
|
-
*/
|
|
338
|
-
clusterInputs(inputs, _snapshot) {
|
|
339
|
-
if (inputs.length === 0)
|
|
340
|
-
return [];
|
|
341
|
-
if (inputs.length === 1)
|
|
342
|
-
return [[inputs[0]]];
|
|
343
|
-
// Group by region first
|
|
344
|
-
const byRegion = new Map();
|
|
345
|
-
for (const input of inputs) {
|
|
346
|
-
const key = input.where.region ?? 'unknown';
|
|
347
|
-
const group = byRegion.get(key) ?? [];
|
|
348
|
-
group.push(input);
|
|
349
|
-
byRegion.set(key, group);
|
|
350
|
-
}
|
|
351
|
-
const clusters = [];
|
|
352
|
-
// Within each region, cluster by proximity
|
|
353
|
-
for (const regionInputs of byRegion.values()) {
|
|
354
|
-
if (regionInputs.length === 1) {
|
|
355
|
-
clusters.push(regionInputs);
|
|
356
|
-
continue;
|
|
357
|
-
}
|
|
358
|
-
// Simple clustering by vertical proximity
|
|
359
|
-
const sorted = [...regionInputs].sort((a, b) => {
|
|
360
|
-
const yA = a.layout?.bbox?.y ?? 0;
|
|
361
|
-
const yB = b.layout?.bbox?.y ?? 0;
|
|
362
|
-
return yA - yB;
|
|
363
|
-
});
|
|
364
|
-
let currentCluster = [sorted[0]];
|
|
365
|
-
for (let i = 1; i < sorted.length; i++) {
|
|
366
|
-
const prev = sorted[i - 1];
|
|
367
|
-
const curr = sorted[i];
|
|
368
|
-
const prevY = (prev.layout?.bbox?.y ?? 0) + (prev.layout?.bbox?.h ?? 0);
|
|
369
|
-
const currY = curr.layout?.bbox?.y ?? 0;
|
|
370
|
-
const distance = currY - prevY;
|
|
371
|
-
if (distance <= this.config.cluster_distance) {
|
|
372
|
-
currentCluster.push(curr);
|
|
373
|
-
}
|
|
374
|
-
else {
|
|
375
|
-
if (currentCluster.length > 0) {
|
|
376
|
-
clusters.push(currentCluster);
|
|
377
|
-
}
|
|
378
|
-
currentCluster = [curr];
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
if (currentCluster.length > 0) {
|
|
382
|
-
clusters.push(currentCluster);
|
|
383
|
-
}
|
|
384
|
-
}
|
|
385
|
-
return clusters;
|
|
386
|
-
}
|
|
387
209
|
/**
|
|
388
210
|
* Check if a node is likely within a form's scope.
|
|
389
211
|
*/
|
|
@@ -417,75 +239,6 @@ export class FormDetector {
|
|
|
417
239
|
}
|
|
418
240
|
return false;
|
|
419
241
|
}
|
|
420
|
-
/**
|
|
421
|
-
* Find a submit button associated with a form.
|
|
422
|
-
*/
|
|
423
|
-
findSubmitButton(snapshot, formNode, fieldEids) {
|
|
424
|
-
const buttons = snapshot.nodes.filter((n) => BUTTON_KINDS.has(n.kind));
|
|
425
|
-
for (const button of buttons) {
|
|
426
|
-
// Check if button is within form's scope
|
|
427
|
-
if (!this.isNodeWithinForm(button, formNode, snapshot)) {
|
|
428
|
-
continue;
|
|
429
|
-
}
|
|
430
|
-
// Check if button label suggests submission
|
|
431
|
-
if (this.isSubmitButton(button)) {
|
|
432
|
-
return button;
|
|
433
|
-
}
|
|
434
|
-
}
|
|
435
|
-
// Also check buttons near the fields
|
|
436
|
-
if (fieldEids.length > 0) {
|
|
437
|
-
const fieldNodes = fieldEids
|
|
438
|
-
.map((eid) => snapshot.nodes.find((n) => n.node_id === eid))
|
|
439
|
-
.filter((n) => n !== undefined);
|
|
440
|
-
return this.findSubmitButtonNearCluster(snapshot, fieldNodes);
|
|
441
|
-
}
|
|
442
|
-
return undefined;
|
|
443
|
-
}
|
|
444
|
-
/**
|
|
445
|
-
* Find a submit button near a cluster of inputs.
|
|
446
|
-
*/
|
|
447
|
-
findSubmitButtonNearCluster(snapshot, cluster) {
|
|
448
|
-
if (cluster.length === 0)
|
|
449
|
-
return undefined;
|
|
450
|
-
const buttons = snapshot.nodes.filter((n) => BUTTON_KINDS.has(n.kind));
|
|
451
|
-
const clusterBbox = this.computeClusterBbox(cluster);
|
|
452
|
-
if (!clusterBbox)
|
|
453
|
-
return undefined;
|
|
454
|
-
// Find buttons near the cluster
|
|
455
|
-
const nearbyButtons = buttons.filter((button) => {
|
|
456
|
-
if (!button.layout?.bbox)
|
|
457
|
-
return false;
|
|
458
|
-
const btnBbox = button.layout.bbox;
|
|
459
|
-
// Check if button is below or to the right of the cluster
|
|
460
|
-
const isNearX = btnBbox.x >= clusterBbox.x - 100 && btnBbox.x <= clusterBbox.x + clusterBbox.width + 100;
|
|
461
|
-
const isNearY = btnBbox.y >= clusterBbox.y - 50 && btnBbox.y <= clusterBbox.y + clusterBbox.height + 150;
|
|
462
|
-
return isNearX && isNearY;
|
|
463
|
-
});
|
|
464
|
-
// Find the best submit button candidate
|
|
465
|
-
for (const button of nearbyButtons) {
|
|
466
|
-
if (this.isSubmitButton(button)) {
|
|
467
|
-
return button;
|
|
468
|
-
}
|
|
469
|
-
}
|
|
470
|
-
return undefined;
|
|
471
|
-
}
|
|
472
|
-
/**
|
|
473
|
-
* Check if a button looks like a submit button.
|
|
474
|
-
*/
|
|
475
|
-
isSubmitButton(button) {
|
|
476
|
-
const label = button.label.toLowerCase();
|
|
477
|
-
// Check for submit keywords
|
|
478
|
-
for (const keyword of SUBMIT_KEYWORDS) {
|
|
479
|
-
if (label.includes(keyword)) {
|
|
480
|
-
return true;
|
|
481
|
-
}
|
|
482
|
-
}
|
|
483
|
-
// Check for type="submit" attribute
|
|
484
|
-
if (button.attributes?.input_type === 'submit') {
|
|
485
|
-
return true;
|
|
486
|
-
}
|
|
487
|
-
return false;
|
|
488
|
-
}
|
|
489
242
|
/**
|
|
490
243
|
* Compute confidence score from signals.
|
|
491
244
|
*/
|
|
@@ -498,57 +251,6 @@ export class FormDetector {
|
|
|
498
251
|
// Normalize to 0-1
|
|
499
252
|
return Math.min(1.0, score);
|
|
500
253
|
}
|
|
501
|
-
/**
|
|
502
|
-
* Infer the intent of a form.
|
|
503
|
-
*/
|
|
504
|
-
inferIntent(snapshot, fieldEids, formNode) {
|
|
505
|
-
// Collect all relevant text to analyze
|
|
506
|
-
const textToAnalyze = [];
|
|
507
|
-
// Add form node label if available
|
|
508
|
-
if (formNode?.label) {
|
|
509
|
-
textToAnalyze.push(formNode.label);
|
|
510
|
-
}
|
|
511
|
-
// Add form heading context
|
|
512
|
-
if (formNode?.where.heading_context) {
|
|
513
|
-
textToAnalyze.push(formNode.where.heading_context);
|
|
514
|
-
}
|
|
515
|
-
// Add field labels
|
|
516
|
-
for (const eid of fieldEids) {
|
|
517
|
-
const node = snapshot.nodes.find((n) => n.node_id === eid);
|
|
518
|
-
if (node?.label) {
|
|
519
|
-
textToAnalyze.push(node.label);
|
|
520
|
-
}
|
|
521
|
-
if (node?.attributes?.placeholder) {
|
|
522
|
-
textToAnalyze.push(node.attributes.placeholder);
|
|
523
|
-
}
|
|
524
|
-
}
|
|
525
|
-
const combinedText = textToAnalyze.join(' ').toLowerCase();
|
|
526
|
-
// Score each intent using weighted keywords
|
|
527
|
-
let bestIntent = 'unknown';
|
|
528
|
-
let bestScore = 0;
|
|
529
|
-
for (const [intent, keywordEntries] of Object.entries(INTENT_KEYWORDS)) {
|
|
530
|
-
if (intent === 'unknown')
|
|
531
|
-
continue;
|
|
532
|
-
let score = 0;
|
|
533
|
-
for (const entry of keywordEntries) {
|
|
534
|
-
if (combinedText.includes(entry.keyword)) {
|
|
535
|
-
score += entry.weight;
|
|
536
|
-
}
|
|
537
|
-
}
|
|
538
|
-
if (score > bestScore) {
|
|
539
|
-
bestScore = score;
|
|
540
|
-
bestIntent = intent;
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
return bestIntent;
|
|
544
|
-
}
|
|
545
|
-
/**
|
|
546
|
-
* Check if text contains any of the given keywords.
|
|
547
|
-
*/
|
|
548
|
-
hasIntentKeywords(text, keywords) {
|
|
549
|
-
const lower = text.toLowerCase();
|
|
550
|
-
return keywords.some((k) => lower.includes(k));
|
|
551
|
-
}
|
|
552
254
|
/**
|
|
553
255
|
* Compute bounding box for a cluster of nodes.
|
|
554
256
|
*/
|
|
@@ -584,7 +286,7 @@ export class FormDetector {
|
|
|
584
286
|
// Extract fields
|
|
585
287
|
const fields = extractFields(snapshot, candidate.field_eids, this.config);
|
|
586
288
|
// Find action buttons
|
|
587
|
-
const actions =
|
|
289
|
+
const actions = extractFormActions(snapshot, candidate, this.computeClusterBbox.bind(this));
|
|
588
290
|
// Compute form state
|
|
589
291
|
const state = computeFormState(fields);
|
|
590
292
|
// Determine form pattern
|
|
@@ -618,73 +320,6 @@ export class FormDetector {
|
|
|
618
320
|
const hash = createHash('sha256').update(components.join('::')).digest('hex');
|
|
619
321
|
return `form-${hash.substring(0, 8)}`;
|
|
620
322
|
}
|
|
621
|
-
/**
|
|
622
|
-
* Extract form action buttons.
|
|
623
|
-
*/
|
|
624
|
-
extractFormActions(snapshot, candidate) {
|
|
625
|
-
const actions = [];
|
|
626
|
-
const buttons = snapshot.nodes.filter((n) => BUTTON_KINDS.has(n.kind));
|
|
627
|
-
for (const button of buttons) {
|
|
628
|
-
// Skip disabled buttons for now but still include them
|
|
629
|
-
const isSubmit = this.isSubmitButton(button);
|
|
630
|
-
const isNearForm = candidate.bbox
|
|
631
|
-
? this.isButtonNearBbox(button, candidate.bbox)
|
|
632
|
-
: candidate.field_eids.length === 0 ||
|
|
633
|
-
this.isButtonNearFields(button, snapshot, candidate.field_eids);
|
|
634
|
-
if (!isNearForm)
|
|
635
|
-
continue;
|
|
636
|
-
// Determine action type
|
|
637
|
-
let type = 'action';
|
|
638
|
-
const label = button.label.toLowerCase();
|
|
639
|
-
if (isSubmit) {
|
|
640
|
-
type = 'submit';
|
|
641
|
-
}
|
|
642
|
-
else if (label.includes('cancel') || label.includes('close')) {
|
|
643
|
-
type = 'cancel';
|
|
644
|
-
}
|
|
645
|
-
else if (label.includes('back') || label.includes('previous')) {
|
|
646
|
-
type = 'back';
|
|
647
|
-
}
|
|
648
|
-
else if (label.includes('next') || label.includes('continue')) {
|
|
649
|
-
type = 'next';
|
|
650
|
-
}
|
|
651
|
-
else if (label.includes('reset') || label.includes('clear')) {
|
|
652
|
-
type = 'reset';
|
|
653
|
-
}
|
|
654
|
-
actions.push({
|
|
655
|
-
eid: button.node_id,
|
|
656
|
-
backend_node_id: button.backend_node_id,
|
|
657
|
-
label: button.label,
|
|
658
|
-
type,
|
|
659
|
-
enabled: button.state?.enabled ?? true,
|
|
660
|
-
is_primary: isSubmit,
|
|
661
|
-
});
|
|
662
|
-
}
|
|
663
|
-
return actions;
|
|
664
|
-
}
|
|
665
|
-
/**
|
|
666
|
-
* Check if a button is near a bounding box.
|
|
667
|
-
*/
|
|
668
|
-
isButtonNearBbox(button, bbox) {
|
|
669
|
-
if (!button.layout?.bbox)
|
|
670
|
-
return false;
|
|
671
|
-
const btnBbox = button.layout.bbox;
|
|
672
|
-
const isNearX = btnBbox.x >= bbox.x - 100 && btnBbox.x <= bbox.x + bbox.width + 100;
|
|
673
|
-
const isNearY = btnBbox.y >= bbox.y - 50 && btnBbox.y <= bbox.y + bbox.height + 150;
|
|
674
|
-
return isNearX && isNearY;
|
|
675
|
-
}
|
|
676
|
-
/**
|
|
677
|
-
* Check if a button is near a set of fields.
|
|
678
|
-
*/
|
|
679
|
-
isButtonNearFields(button, snapshot, fieldEids) {
|
|
680
|
-
const fieldNodes = fieldEids
|
|
681
|
-
.map((eid) => snapshot.nodes.find((n) => n.node_id === eid))
|
|
682
|
-
.filter((n) => n !== undefined);
|
|
683
|
-
const clusterBbox = this.computeClusterBbox(fieldNodes);
|
|
684
|
-
if (!clusterBbox)
|
|
685
|
-
return false;
|
|
686
|
-
return this.isButtonNearBbox(button, clusterBbox);
|
|
687
|
-
}
|
|
688
323
|
/**
|
|
689
324
|
* Infer form pattern (single page, multi-step, etc.)
|
|
690
325
|
*/
|