app-manager-edge-worker 1.0.4 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/package.json +45 -45
  2. package/src/banner.js +19 -5
  3. package/src/handlers/previewHandler.js +484 -346
  4. package/src/handlers/scrapeHandler.js +801 -433
  5. package/src/handlers/transformUtils.js +340 -0
package/src/handlers/previewHandler.js CHANGED
@@ -1,346 +1,484 @@
1
- import localBrowserManager from "../browser.js";
2
-
3
- /**
4
- * Handles PREVIEW_PAGE task: captures full-page screenshot and dimensions on the edge worker.
5
- */
6
- export async function handlePreviewPage(payload) {
7
- const { url, timeout = 25000 } = payload;
8
- const browser = await localBrowserManager.getBrowser();
9
- const page = await browser.newPage();
10
-
11
- try {
12
- await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
13
-
14
- await page.goto(url, { waitUntil: "domcontentloaded", timeout });
15
- try {
16
- await page.waitForNetworkIdle({ idleTime: 500, timeout: 3500 }).catch(() => {});
17
- } catch (_) {}
18
- await page.evaluate(() => document.fonts?.ready).catch(() => {});
19
- await new Promise((r) => setTimeout(r, 800));
20
-
21
- const title = (await page.title()) || new URL(url).hostname;
22
- const currentUrl = page.url();
23
-
24
- const rawDimensions = await page.evaluate(() => ({
25
- width: document.documentElement.scrollWidth || 1366,
26
- height: document.documentElement.scrollHeight || 900,
27
- }));
28
- const dimensions = {
29
- width: Math.min(rawDimensions.width, 3840),
30
- height: Math.min(rawDimensions.height, 12000),
31
- };
32
-
33
- // Hide scrollbars before screenshot
34
- await page
35
- .addStyleTag({
36
- content: `
37
- ::-webkit-scrollbar { display: none !important; width: 0 !important; height: 0 !important; }
38
- html, body, * { scrollbar-width: none !important; -ms-overflow-style: none !important; }
39
- `,
40
- })
41
- .catch(() => {});
42
-
43
- const screenshotBuffer = await page.screenshot({
44
- encoding: "binary",
45
- type: "jpeg",
46
- quality: 75,
47
- fullPage: true,
48
- });
49
-
50
- const screenshotBase64 = Buffer.from(screenshotBuffer).toString("base64");
51
-
52
- return {
53
- title,
54
- url: currentUrl,
55
- dimensions,
56
- screenshotBase64: `data:image/jpeg;base64,${screenshotBase64}`,
57
- };
58
- } finally {
59
- await page.close().catch(() => {});
60
- }
61
- }
62
-
63
- /**
64
- * Handles INSPECT_ELEMENT task: resolves DOM element coordinates or evaluates candidate selectors on the edge worker.
65
- */
66
- export async function handleInspectElement(payload) {
67
- const { url, selector, x, y, timeout = 20000 } = payload;
68
- const browser = await localBrowserManager.getBrowser();
69
- const page = await browser.newPage();
70
-
71
- try {
72
- await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
73
- await page.goto(url, { waitUntil: "domcontentloaded", timeout });
74
- await new Promise((r) => setTimeout(r, 1000));
75
-
76
- if (x !== undefined && y !== undefined) {
77
- const elementInfo = await page.evaluate((px, py) => {
78
- const scrollY = Math.max(0, py - 250);
79
- const scrollX = Math.max(0, px - 250);
80
- window.scrollTo(scrollX, scrollY);
81
-
82
- const clientX = px - window.scrollX;
83
- const clientY = py - window.scrollY;
84
-
85
- const el = document.elementFromPoint(clientX, clientY);
86
- if (!el) return null;
87
-
88
- const rect = el.getBoundingClientRect();
89
- const boundingRect = {
90
- left: Math.round(rect.left + window.scrollX),
91
- top: Math.round(rect.top + window.scrollY),
92
- width: Math.round(rect.width),
93
- height: Math.round(rect.height),
94
- };
95
-
96
- // ─── Helper: count matching elements for a CSS selector ───
97
- const countMatches = (sel) => {
98
- try { return document.querySelectorAll(sel).length; } catch { return 999; }
99
- };
100
-
101
- // ─── Helper: Full positional CSS path anchored at nearest ID ───
102
- const buildCssPath = (element, maxDepth = 8) => {
103
- const parts = [];
104
- let curr = element;
105
- while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
106
- if (curr === document.body) { parts.unshift("body"); break; }
107
- let seg = curr.tagName.toLowerCase();
108
- if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
109
- parts.unshift(`#${curr.id}`);
110
- break;
111
- }
112
- if (curr.parentElement) {
113
- const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
114
- seg += `:nth-child(${pos})`;
115
- }
116
- parts.unshift(seg);
117
- curr = curr.parentElement;
118
- if (parts.length >= maxDepth) break;
119
- }
120
- return parts.join(" > ");
121
- };
122
-
123
- // ─── Helper: Shortest unique CSS path using class combos + nth-child ───
124
- const buildShortUniqueSelector = (element) => {
125
- const pathParts = [];
126
- let curr = element;
127
- while (curr && curr !== document.documentElement) {
128
- if (curr === document.body) { pathParts.unshift("body"); break; }
129
- let seg = curr.tagName.toLowerCase();
130
- if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
131
- pathParts.unshift(`#${curr.id}`);
132
- break;
133
- }
134
- const classes = (curr.className && typeof curr.className === "string")
135
- ? curr.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
136
- : [];
137
- let foundUnique = false;
138
- for (let len = Math.min(classes.length, 5); len >= 1; len--) {
139
- const combo = classes.slice(0, len);
140
- const candidate = `${seg}.${combo.join(".")}`;
141
- const full = [...pathParts, candidate].join(" > ");
142
- if (countMatches(full) === 1) {
143
- pathParts.unshift(candidate);
144
- foundUnique = true;
145
- break;
146
- }
147
- }
148
- if (!foundUnique) {
149
- if (curr.parentElement) {
150
- const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
151
- seg += `:nth-child(${pos})`;
152
- }
153
- pathParts.unshift(seg);
154
- }
155
- const full = pathParts.join(" > ");
156
- if (countMatches(full) === 1) break;
157
- curr = curr.parentElement;
158
- }
159
- return pathParts.join(" > ");
160
- };
161
-
162
- // ─── Helper: Robust absolute XPath ───
163
- const buildXPath = (element) => {
164
- if (!element || element === document) return "";
165
- if (element.id && /^[a-zA-Z][\w-]*$/.test(element.id)) {
166
- return `//*[@id="${element.id}"]`;
167
- }
168
- if (element === document.body) return "/html/body";
169
- const parts = [];
170
- let curr = element;
171
- while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
172
- if (curr === document.body) { parts.unshift("body"); break; }
173
- if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
174
- parts.unshift(`//*[@id="${curr.id}"]`);
175
- return parts.join("/");
176
- }
177
- const tag = curr.tagName.toLowerCase();
178
- let idx = 1;
179
- let sib = curr.previousElementSibling;
180
- while (sib) {
181
- if (sib.tagName.toLowerCase() === tag) idx++;
182
- sib = sib.previousElementSibling;
183
- }
184
- const total = curr.parentElement
185
- ? Array.from(curr.parentElement.children).filter(c => c.tagName.toLowerCase() === tag).length
186
- : 1;
187
- parts.unshift(total > 1 ? `${tag}[${idx}]` : tag);
188
- curr = curr.parentElement;
189
- }
190
- return "/" + parts.join("/");
191
- };
192
-
193
- // ─── Build ranked candidates ───
194
- const candidates = [];
195
- const seen = new Set();
196
- const addCandidate = (type, value, extra = {}) => {
197
- if (!value || seen.has(value)) return;
198
- seen.add(value);
199
- const matchCount = type === "css" ? countMatches(value) : null;
200
- candidates.push({ type, value, attribute: "text", matchCount, ...extra });
201
- };
202
-
203
- // 1. ID (most reliable, unique)
204
- if (el.id && /^[a-zA-Z][\w-]*$/.test(el.id)) {
205
- addCandidate("css", `#${el.id}`, { specificity: "unique", label: "ID Selector" });
206
- }
207
-
208
- // 2. data-* and aria-* attributes
209
- for (const attr of el.attributes) {
210
- if ((attr.name.startsWith("data-") || attr.name.startsWith("aria-")) && attr.value && attr.value.length < 80) {
211
- addCandidate("css", `${el.tagName.toLowerCase()}[${attr.name}="${attr.value}"]`, {
212
- specificity: "high", label: `Attribute: ${attr.name}`
213
- });
214
- }
215
- }
216
-
217
- // 3. Shortest unique CSS path
218
- const shortUnique = buildShortUniqueSelector(el);
219
- if (shortUnique) {
220
- const mc = countMatches(shortUnique);
221
- addCandidate("css", shortUnique, {
222
- specificity: mc === 1 ? "unique" : "medium",
223
- label: "Smart CSS Path",
224
- matchCount: mc,
225
- });
226
- }
227
-
228
- // 4. Class-based selectors (increasing combos until unique)
229
- const classes = (el.className && typeof el.className === "string")
230
- ? el.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
231
- : [];
232
- for (let len = 1; len <= Math.min(classes.length, 5); len++) {
233
- const sel = `${el.tagName.toLowerCase()}.${classes.slice(0, len).join(".")}`;
234
- const mc = countMatches(sel);
235
- addCandidate("css", sel, {
236
- specificity: mc === 1 ? "unique" : mc <= 3 ? "medium" : "generic",
237
- label: "Class Selector",
238
- matchCount: mc,
239
- });
240
- if (mc === 1) break;
241
- }
242
-
243
- // 5. Table-specific selectors
244
- const tr = el.closest("tr");
245
- if (tr && tr.parentElement) {
246
- const rowIdx = Array.from(tr.parentElement.children).indexOf(tr) + 1;
247
- const td = el.closest("td, th");
248
- if (td) {
249
- const colIdx = Array.from(tr.children).indexOf(td) + 1;
250
- addCandidate("css", `table tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
251
- specificity: "medium", label: "Table Cell"
252
- });
253
- addCandidate("css", `table tbody tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
254
- specificity: "medium", label: "Table Cell (tbody)"
255
- });
256
- }
257
- }
258
-
259
- // 6. Full structural CSS path
260
- const fullCss = buildCssPath(el, 8);
261
- addCandidate("css", fullCss, { specificity: "unique", label: "Full CSS Path (Structural)" });
262
-
263
- // 7. XPath — most reliable for scrapers
264
- const xpath = buildXPath(el);
265
- addCandidate("xpath", xpath, { specificity: "high", label: "XPath" });
266
-
267
- // Sort by specificity: unique > high > medium > generic
268
- const rank = { unique: 0, high: 1, medium: 2, generic: 3 };
269
- candidates.sort((a, b) => (rank[a.specificity] ?? 4) - (rank[b.specificity] ?? 4));
270
-
271
- // ─── Breadcrumb hierarchy for UI ───
272
- const hierarchy = [];
273
- let curr = el;
274
- while (curr && curr !== document.body && hierarchy.length < 7) {
275
- let desc = curr.tagName.toLowerCase();
276
- if (curr.id) {
277
- desc += `#${curr.id}`;
278
- } else if (curr.classList && curr.classList.length > 0) {
279
- const cls = Array.from(curr.classList)
280
- .filter(c => /^[a-zA-Z_-][\w-]*$/.test(c))
281
- .slice(0, 2);
282
- if (cls.length) desc += `.${cls.join(".")}`;
283
- }
284
- hierarchy.unshift(desc);
285
- curr = curr.parentElement;
286
- }
287
-
288
- const text = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
289
-
290
- return {
291
- found: true,
292
- tagName: el.tagName.toLowerCase(),
293
- text,
294
- attributes: Array.from(el.attributes).reduce((acc, a) => {
295
- acc[a.name] = a.value;
296
- return acc;
297
- }, {}),
298
- candidates,
299
- hierarchy,
300
- boundingRect,
301
- };
302
- }, x, y);
303
-
304
- return elementInfo || { found: false, message: "No element found at specified coordinates." };
305
- }
306
-
307
- if (selector) {
308
- const testResult = await page.evaluate((sel) => {
309
- try {
310
- const el = document.querySelector(sel);
311
- if (!el) return { found: false, count: 0 };
312
-
313
- const rect = el.getBoundingClientRect();
314
- const boundingRect = {
315
- left: Math.round(rect.left + window.scrollX),
316
- top: Math.round(rect.top + window.scrollY),
317
- width: Math.round(rect.width),
318
- height: Math.round(rect.height),
319
- };
320
-
321
- const count = document.querySelectorAll(sel).length;
322
-
323
- return {
324
- found: true,
325
- count,
326
- tagName: el.tagName.toLowerCase(),
327
- text: el.innerText ? el.innerText.trim() : el.textContent?.trim() || "",
328
- attributes: Array.from(el.attributes).reduce((acc, a) => {
329
- acc[a.name] = a.value;
330
- return acc;
331
- }, {}),
332
- boundingRect,
333
- };
334
- } catch (e) {
335
- return { found: false, error: e.message };
336
- }
337
- }, selector);
338
-
339
- return testResult;
340
- }
341
-
342
- return { message: "Specify coordinates (x, y) or selector to inspect." };
343
- } finally {
344
- await page.close().catch(() => {});
345
- }
346
- }
1
+ import localBrowserManager from "../browser.js";
2
+
3
+ /**
4
+ * Handles PREVIEW_PAGE task: captures full-page screenshot and dimensions on the edge worker.
5
+ */
6
+ export async function handlePreviewPage(payload) {
7
+ const { url, timeout = 25000 } = payload;
8
+ const browser = await localBrowserManager.getBrowser();
9
+ const page = await browser.newPage();
10
+
11
+ try {
12
+ await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
13
+
14
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout });
15
+ try {
16
+ await page.waitForNetworkIdle({ idleTime: 500, timeout: 3500 }).catch(() => {});
17
+ } catch (_) {}
18
+ await page.evaluate(() => document.fonts?.ready).catch(() => {});
19
+ await new Promise((r) => setTimeout(r, 800));
20
+
21
+ const title = (await page.title()) || new URL(url).hostname;
22
+ const currentUrl = page.url();
23
+
24
+ const rawDimensions = await page.evaluate(() => ({
25
+ width: document.documentElement.scrollWidth || 1366,
26
+ height: document.documentElement.scrollHeight || 900,
27
+ }));
28
+ const dimensions = {
29
+ width: Math.min(rawDimensions.width, 3840),
30
+ height: Math.min(rawDimensions.height, 12000),
31
+ };
32
+
33
+ // Hide scrollbars before screenshot
34
+ await page
35
+ .addStyleTag({
36
+ content: `
37
+ ::-webkit-scrollbar { display: none !important; width: 0 !important; height: 0 !important; }
38
+ html, body, * { scrollbar-width: none !important; -ms-overflow-style: none !important; }
39
+ `,
40
+ })
41
+ .catch(() => {});
42
+
43
+ const screenshotBuffer = await page.screenshot({
44
+ encoding: "binary",
45
+ type: "jpeg",
46
+ quality: 75,
47
+ fullPage: true,
48
+ });
49
+
50
+ const screenshotBase64 = Buffer.from(screenshotBuffer).toString("base64");
51
+
52
+ return {
53
+ title,
54
+ url: currentUrl,
55
+ dimensions,
56
+ screenshotBase64: `data:image/jpeg;base64,${screenshotBase64}`,
57
+ };
58
+ } finally {
59
+ await page.close().catch(() => {});
60
+ }
61
+ }
62
+
63
+ /**
64
+ * Handles INSPECT_ELEMENT task: resolves DOM element coordinates or evaluates candidate selectors on the edge worker.
65
+ */
66
+ export async function handleInspectElement(payload) {
67
+ const { url, selector, x, y, timeout = 20000 } = payload;
68
+ const browser = await localBrowserManager.getBrowser();
69
+ const page = await browser.newPage();
70
+
71
+ try {
72
+ await page.setViewport({ width: 1366, height: 900, deviceScaleFactor: 1 });
73
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout });
74
+ await new Promise((r) => setTimeout(r, 1000));
75
+
76
+ if (x !== undefined && y !== undefined) {
77
+ const elementInfo = await page.evaluate((px, py) => {
78
+ const scrollY = Math.max(0, py - 250);
79
+ const scrollX = Math.max(0, px - 250);
80
+ window.scrollTo(scrollX, scrollY);
81
+
82
+ const clientX = px - window.scrollX;
83
+ const clientY = py - window.scrollY;
84
+
85
+ const el = document.elementFromPoint(clientX, clientY);
86
+ if (!el) return null;
87
+
88
+ const rect = el.getBoundingClientRect();
89
+ const boundingRect = {
90
+ left: Math.round(rect.left + window.scrollX),
91
+ top: Math.round(rect.top + window.scrollY),
92
+ width: Math.round(rect.width),
93
+ height: Math.round(rect.height),
94
+ };
95
+
96
+ // ─── Helper: count matching elements for a CSS selector ───
97
+ const countMatches = (sel) => {
98
+ try { return document.querySelectorAll(sel).length; } catch { return 999; }
99
+ };
100
+
101
+ // ─── Helper: Full positional CSS path anchored at nearest ID ───
102
+ const buildCssPath = (element, maxDepth = 8) => {
103
+ const parts = [];
104
+ let curr = element;
105
+ while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
106
+ if (curr === document.body) { parts.unshift("body"); break; }
107
+ let seg = curr.tagName.toLowerCase();
108
+ if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
109
+ parts.unshift(`#${curr.id}`);
110
+ break;
111
+ }
112
+ if (curr.parentElement) {
113
+ const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
114
+ seg += `:nth-child(${pos})`;
115
+ }
116
+ parts.unshift(seg);
117
+ curr = curr.parentElement;
118
+ if (parts.length >= maxDepth) break;
119
+ }
120
+ return parts.join(" > ");
121
+ };
122
+
123
+ // ─── Helper: Shortest unique CSS path using class combos + nth-child ───
124
+ const buildShortUniqueSelector = (element) => {
125
+ const pathParts = [];
126
+ let curr = element;
127
+ while (curr && curr !== document.documentElement) {
128
+ if (curr === document.body) { pathParts.unshift("body"); break; }
129
+ let seg = curr.tagName.toLowerCase();
130
+ if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
131
+ pathParts.unshift(`#${curr.id}`);
132
+ break;
133
+ }
134
+ const classes = (curr.className && typeof curr.className === "string")
135
+ ? curr.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
136
+ : [];
137
+ let foundUnique = false;
138
+ for (let len = Math.min(classes.length, 5); len >= 1; len--) {
139
+ const combo = classes.slice(0, len);
140
+ const candidate = `${seg}.${combo.join(".")}`;
141
+ const full = [...pathParts, candidate].join(" > ");
142
+ if (countMatches(full) === 1) {
143
+ pathParts.unshift(candidate);
144
+ foundUnique = true;
145
+ break;
146
+ }
147
+ }
148
+ if (!foundUnique) {
149
+ if (curr.parentElement) {
150
+ const pos = Array.from(curr.parentElement.children).indexOf(curr) + 1;
151
+ seg += `:nth-child(${pos})`;
152
+ }
153
+ pathParts.unshift(seg);
154
+ }
155
+ const full = pathParts.join(" > ");
156
+ if (countMatches(full) === 1) break;
157
+ curr = curr.parentElement;
158
+ }
159
+ return pathParts.join(" > ");
160
+ };
161
+
162
+ // ─── Helper: Robust absolute XPath ───
163
+ const buildXPath = (element) => {
164
+ if (!element || element === document) return "";
165
+ if (element.id && /^[a-zA-Z][\w-]*$/.test(element.id)) {
166
+ return `//*[@id="${element.id}"]`;
167
+ }
168
+ if (element === document.body) return "/html/body";
169
+ const parts = [];
170
+ let curr = element;
171
+ while (curr && curr.nodeType === 1 && curr !== document.documentElement) {
172
+ if (curr === document.body) { parts.unshift("body"); break; }
173
+ if (curr.id && /^[a-zA-Z][\w-]*$/.test(curr.id)) {
174
+ parts.unshift(`//*[@id="${curr.id}"]`);
175
+ return parts.join("/");
176
+ }
177
+ const tag = curr.tagName.toLowerCase();
178
+ let idx = 1;
179
+ let sib = curr.previousElementSibling;
180
+ while (sib) {
181
+ if (sib.tagName.toLowerCase() === tag) idx++;
182
+ sib = sib.previousElementSibling;
183
+ }
184
+ const total = curr.parentElement
185
+ ? Array.from(curr.parentElement.children).filter(c => c.tagName.toLowerCase() === tag).length
186
+ : 1;
187
+ parts.unshift(total > 1 ? `${tag}[${idx}]` : tag);
188
+ curr = curr.parentElement;
189
+ }
190
+ return "/" + parts.join("/");
191
+ };
192
+
193
+ // ─── Build ranked candidates ───
194
+ const candidates = [];
195
+ const seen = new Set();
196
+ const addCandidate = (type, value, extra = {}) => {
197
+ if (!value || seen.has(value)) return;
198
+ seen.add(value);
199
+ const matchCount = type === "css" ? countMatches(value) : null;
200
+ candidates.push({ type, value, attribute: "text", matchCount, ...extra });
201
+ };
202
+
203
+ // 1. ID (most reliable, unique)
204
+ if (el.id && /^[a-zA-Z][\w-]*$/.test(el.id)) {
205
+ addCandidate("css", `#${el.id}`, { specificity: "unique", label: "ID Selector" });
206
+ }
207
+
208
+ // 2. data-* and aria-* attributes
209
+ for (const attr of el.attributes) {
210
+ if ((attr.name.startsWith("data-") || attr.name.startsWith("aria-")) && attr.value && attr.value.length < 80) {
211
+ addCandidate("css", `${el.tagName.toLowerCase()}[${attr.name}="${attr.value}"]`, {
212
+ specificity: "high", label: `Attribute: ${attr.name}`
213
+ });
214
+ }
215
+ }
216
+
217
+ // 3. Shortest unique CSS path
218
+ const shortUnique = buildShortUniqueSelector(el);
219
+ if (shortUnique) {
220
+ const mc = countMatches(shortUnique);
221
+ addCandidate("css", shortUnique, {
222
+ specificity: mc === 1 ? "unique" : "medium",
223
+ label: "Smart CSS Path",
224
+ matchCount: mc,
225
+ });
226
+ }
227
+
228
+ // 4. Class-based selectors (increasing combos until unique)
229
+ const classes = (el.className && typeof el.className === "string")
230
+ ? el.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
231
+ : [];
232
+ for (let len = 1; len <= Math.min(classes.length, 5); len++) {
233
+ const sel = `${el.tagName.toLowerCase()}.${classes.slice(0, len).join(".")}`;
234
+ const mc = countMatches(sel);
235
+ addCandidate("css", sel, {
236
+ specificity: mc === 1 ? "unique" : mc <= 3 ? "medium" : "generic",
237
+ label: "Class Selector",
238
+ matchCount: mc,
239
+ });
240
+ if (mc === 1) break;
241
+ }
242
+
243
+ // 5. Table-specific selectors
244
+ const tr = el.closest("tr");
245
+ if (tr && tr.parentElement) {
246
+ const rowIdx = Array.from(tr.parentElement.children).indexOf(tr) + 1;
247
+ const td = el.closest("td, th");
248
+ if (td) {
249
+ const colIdx = Array.from(tr.children).indexOf(td) + 1;
250
+ addCandidate("css", `table tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
251
+ specificity: "medium", label: "Table Cell"
252
+ });
253
+ addCandidate("css", `table tbody tr:nth-child(${rowIdx}) td:nth-child(${colIdx})`, {
254
+ specificity: "medium", label: "Table Cell (tbody)"
255
+ });
256
+ }
257
+ }
258
+
259
+ // 6. Full structural CSS path
260
+ const fullCss = buildCssPath(el, 8);
261
+ addCandidate("css", fullCss, { specificity: "unique", label: "Full CSS Path (Structural)" });
262
+
263
+ // 7. XPath — most reliable for scrapers
264
+ const xpath = buildXPath(el);
265
+ addCandidate("xpath", xpath, { specificity: "high", label: "XPath" });
266
+
267
+ // Sort by specificity: unique > high > medium > generic
268
+ const rank = { unique: 0, high: 1, medium: 2, generic: 3 };
269
+ candidates.sort((a, b) => (rank[a.specificity] ?? 4) - (rank[b.specificity] ?? 4));
270
+
271
+ // ─── Breadcrumb hierarchy for UI ───
272
+ const hierarchy = [];
273
+ let curr = el;
274
+ while (curr && curr !== document.body && hierarchy.length < 7) {
275
+ let desc = curr.tagName.toLowerCase();
276
+ if (curr.id) {
277
+ desc += `#${curr.id}`;
278
+ } else if (curr.classList && curr.classList.length > 0) {
279
+ const cls = Array.from(curr.classList)
280
+ .filter(c => /^[a-zA-Z_-][\w-]*$/.test(c))
281
+ .slice(0, 2);
282
+ if (cls.length) desc += `.${cls.join(".")}`;
283
+ }
284
+ hierarchy.unshift(desc);
285
+ curr = curr.parentElement;
286
+ }
287
+
288
+ const text = el.innerText ? el.innerText.trim() : el.textContent?.trim() || "";
289
+
290
+ // ─── Table Detection (US-01, US-02, US-03) ───
291
+ let tableDetection = null;
292
+ const tableEl = el.closest("table");
293
+ if (tableEl) {
294
+ const rows = Array.from(tableEl.querySelectorAll("tr"));
295
+ if (rows.length > 0) {
296
+ let tableSelector = "table";
297
+ if (tableEl.id) {
298
+ tableSelector = `table#${tableEl.id}`;
299
+ } else if (tableEl.className && typeof tableEl.className === "string") {
300
+ const cls = tableEl.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c));
301
+ if (cls.length > 0) tableSelector = `table.${cls.slice(0, 2).join(".")}`;
302
+ }
303
+
304
+ const theadThs = Array.from(tableEl.querySelectorAll("thead th, thead td"));
305
+ const firstRowCells = Array.from(rows[0].querySelectorAll("th, td"));
306
+ const headerCells = theadThs.length > 0 ? theadThs : firstRowCells;
307
+
308
+ const columns = headerCells.map((c, colIdx) => {
309
+ const rawHeader = (c.innerText?.trim() || c.textContent?.trim() || "").replace(/\s+/g, " ");
310
+ const normalizedKey = rawHeader.toLowerCase().replace(/[^a-z0-9]+/g, "_").replace(/^_+|_+$/g, "") || `column_${colIdx + 1}`;
311
+ return { key: normalizedKey, label: rawHeader || `Column ${colIdx + 1}`, columnIndex: colIdx, type: "string", sampleValues: [] };
312
+ });
313
+
314
+ const sampleRows = [];
315
+ for (const dr of rows.slice(1, 6)) {
316
+ const cells = Array.from(dr.querySelectorAll("td, th")).map(c => c.innerText?.trim() || c.textContent?.trim() || "");
317
+ const rec = {};
318
+ columns.forEach((col, idx) => {
319
+ rec[col.key] = cells[idx] !== undefined ? cells[idx] : "";
320
+ });
321
+ sampleRows.push(rec);
322
+ }
323
+
324
+ tableDetection = {
325
+ isTable: true,
326
+ tableSelector,
327
+ hasHeaders: true,
328
+ headerRowIndex: 0,
329
+ totalRows: rows.length,
330
+ columns,
331
+ sampleRows,
332
+ };
333
+ }
334
+ }
335
+
336
+ // ─── Repeating Structure Detection (US-04, US-05, US-23) ───
337
+ let repeatingDetection = null;
338
+ if (!tableEl) {
339
+ let currR = el;
340
+ let depth = 0;
341
+ while (currR && currR !== document.body && depth < 5) {
342
+ const parent = currR.parentElement;
343
+ if (parent && parent !== document.documentElement) {
344
+ const tag = currR.tagName.toLowerCase();
345
+ const classes = (currR.className && typeof currR.className === "string")
346
+ ? currR.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c))
347
+ : [];
348
+ let selCandidate = tag;
349
+ if (classes.length > 0) selCandidate = `${tag}.${classes.slice(0, 2).join(".")}`;
350
+
351
+ const siblings = Array.from(parent.children).filter(c => {
352
+ if (c.tagName.toLowerCase() !== tag) return false;
353
+ if (classes.length > 0 && c.classList) {
354
+ return classes.some(cls => c.classList.contains(cls));
355
+ }
356
+ return true;
357
+ });
358
+
359
+ if (siblings.length >= 2) {
360
+ let parentSel = parent.tagName.toLowerCase();
361
+ if (parent.id) parentSel = `#${parent.id}`;
362
+ else if (parent.className && typeof parent.className === "string") {
363
+ const pCls = parent.className.split(/\s+/).filter(c => c && /^[a-zA-Z_-][\w-]*$/.test(c));
364
+ if (pCls.length > 0) parentSel = `${parentSel}.${pCls.slice(0, 2).join(".")}`;
365
+ }
366
+ const containerSelector = `${parentSel} > ${selCandidate}`;
367
+
368
+ const repCard = siblings[0];
369
+ const suggestedFields = [];
370
+
371
+ const heading = repCard.querySelector("h1, h2, h3, h4, h5, .title, [class*='title'], .name, [class*='name']");
372
+ if (heading) {
373
+ const tagH = heading.tagName.toLowerCase();
374
+ const clsH = heading.className && typeof heading.className === "string" ? heading.className.split(/\s+/)[0] : "";
375
+ suggestedFields.push({ key: "title", label: "Title", relativeSelector: clsH ? `.${clsH}` : tagH, attribute: "text", type: "string" });
376
+ }
377
+
378
+ const priceEl = repCard.querySelector(".price, [class*='price'], [data-price]");
379
+ if (priceEl) {
380
+ const clsP = priceEl.className && typeof priceEl.className === "string" ? priceEl.className.split(/\s+/)[0] : "";
381
+ suggestedFields.push({ key: "price", label: "Price", relativeSelector: clsP ? `.${clsP}` : "[class*='price']", attribute: "text", type: "number" });
382
+ }
383
+
384
+ const imgEl = repCard.querySelector("img");
385
+ if (imgEl && imgEl.getAttribute("src")) {
386
+ suggestedFields.push({ key: "image", label: "Image URL", relativeSelector: "img", attribute: "src", type: "string" });
387
+ }
388
+
389
+ const linkEl = repCard.querySelector("a[href]") || (repCard.tagName.toLowerCase() === "a" ? repCard : null);
390
+ if (linkEl && linkEl.getAttribute("href")) {
391
+ suggestedFields.push({ key: "url", label: "Link", relativeSelector: repCard.tagName.toLowerCase() === "a" ? "" : "a", attribute: "href", type: "string" });
392
+ }
393
+
394
+ const sampleItems = siblings.slice(0, 3).map((item) => {
395
+ const rec = {};
396
+ suggestedFields.forEach((sf) => {
397
+ let val = "";
398
+ if (sf.relativeSelector) {
399
+ const sub = item.querySelector(sf.relativeSelector);
400
+ if (sub) {
401
+ val = sf.attribute === "text" ? (sub.innerText?.trim() || sub.textContent?.trim()) : sub.getAttribute(sf.attribute);
402
+ }
403
+ } else {
404
+ val = sf.attribute === "text" ? (item.innerText?.trim() || item.textContent?.trim()) : item.getAttribute(sf.attribute);
405
+ }
406
+ rec[sf.key] = val || "";
407
+ });
408
+ return rec;
409
+ });
410
+
411
+ repeatingDetection = {
412
+ isRepeating: true,
413
+ containerSelector,
414
+ itemCount: siblings.length,
415
+ suggestedFields,
416
+ sampleItems,
417
+ };
418
+ break;
419
+ }
420
+ }
421
+ currR = currR.parentElement;
422
+ depth++;
423
+ }
424
+ }
425
+
426
+ return {
427
+ found: true,
428
+ tagName: el.tagName.toLowerCase(),
429
+ text,
430
+ attributes: Array.from(el.attributes).reduce((acc, a) => {
431
+ acc[a.name] = a.value;
432
+ return acc;
433
+ }, {}),
434
+ candidates,
435
+ hierarchy,
436
+ boundingRect,
437
+ tableDetection,
438
+ repeatingDetection,
439
+ };
440
+ }, x, y);
441
+
442
+ return elementInfo || { found: false, message: "No element found at specified coordinates." };
443
+ }
444
+
445
+ if (selector) {
446
+ const testResult = await page.evaluate((sel) => {
447
+ try {
448
+ const el = document.querySelector(sel);
449
+ if (!el) return { found: false, count: 0 };
450
+
451
+ const rect = el.getBoundingClientRect();
452
+ const boundingRect = {
453
+ left: Math.round(rect.left + window.scrollX),
454
+ top: Math.round(rect.top + window.scrollY),
455
+ width: Math.round(rect.width),
456
+ height: Math.round(rect.height),
457
+ };
458
+
459
+ const count = document.querySelectorAll(sel).length;
460
+
461
+ return {
462
+ found: true,
463
+ count,
464
+ tagName: el.tagName.toLowerCase(),
465
+ text: el.innerText ? el.innerText.trim() : el.textContent?.trim() || "",
466
+ attributes: Array.from(el.attributes).reduce((acc, a) => {
467
+ acc[a.name] = a.value;
468
+ return acc;
469
+ }, {}),
470
+ boundingRect,
471
+ };
472
+ } catch (e) {
473
+ return { found: false, error: e.message };
474
+ }
475
+ }, selector);
476
+
477
+ return testResult;
478
+ }
479
+
480
+ return { message: "Specify coordinates (x, y) or selector to inspect." };
481
+ } finally {
482
+ await page.close().catch(() => {});
483
+ }
484
+ }