@detiq/crawler 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-login.js +12 -1
- package/dist/ai-login.js.map +2 -2
- package/dist/crawler.js +89 -8
- package/dist/crawler.js.map +2 -2
- package/dist/http-engine.js +2 -1
- package/dist/http-engine.js.map +2 -2
- package/dist/job-queue.js +5 -2
- package/dist/job-queue.js.map +2 -2
- package/dist/skyvern.js +0 -4
- package/dist/skyvern.js.map +2 -2
- package/dist/test-executor.js +18 -6
- package/dist/test-executor.js.map +2 -2
- package/package.json +1 -1
- package/src/crawler.ts +97 -7
- package/src/crawler.ts.rej +12 -0
- package/src/http-engine.ts +2 -1
- package/src/job-queue.ts +5 -2
- package/src/test-executor.ts +20 -6
- package/types/crawler.d.ts +14 -0
- package/types/test-executor.d.ts +7 -0
package/src/test-executor.ts
CHANGED
|
@@ -17,6 +17,8 @@ export interface StepResult {
|
|
|
17
17
|
healedVia?: 'mechanical' | 'ai';
|
|
18
18
|
healReasoning?: string;
|
|
19
19
|
healConfidence?: number;
|
|
20
|
+
elementBoundingRect?: { x: number; y: number; width: number; height: number } | null;
|
|
21
|
+
elementSelector?: string;
|
|
20
22
|
}
|
|
21
23
|
|
|
22
24
|
export interface ExecutionResult {
|
|
@@ -84,14 +86,14 @@ async function resolveLocator(
|
|
|
84
86
|
if (!selector) return null;
|
|
85
87
|
|
|
86
88
|
const primary = buildLocator(page, selector, selectorStrategy);
|
|
87
|
-
const attached = await primary.first().isVisible({ timeout: 1500 }).catch(() => false);
|
|
89
|
+
const attached = await primary.first().isVisible({ timeout: parseInt(process.env.TEST_WAIT_TIMEOUT_MS ?? '1500', 10) }).catch(() => false);
|
|
88
90
|
if (attached) return { loc: primary, healed: false };
|
|
89
91
|
|
|
90
92
|
// Tier 2 — mechanical: try other strategies before giving up
|
|
91
93
|
for (const strategy of LOCATOR_STRATEGIES) {
|
|
92
94
|
if (strategy === selectorStrategy) continue;
|
|
93
95
|
const loc = buildLocator(page, selector, strategy);
|
|
94
|
-
const visible = await loc.first().isVisible({ timeout: 800 }).catch(() => false);
|
|
96
|
+
const visible = await loc.first().isVisible({ timeout: parseInt(process.env.TEST_SHORT_TIMEOUT_MS ?? '800', 10) }).catch(() => false);
|
|
95
97
|
if (visible) return { loc, healed: true, healedVia: 'mechanical' };
|
|
96
98
|
}
|
|
97
99
|
|
|
@@ -108,7 +110,7 @@ async function resolveLocator(
|
|
|
108
110
|
});
|
|
109
111
|
if (result.healed && result.updatedTest) {
|
|
110
112
|
const healedLoc = page.getByText(result.updatedTest.elementMeaning, { exact: false });
|
|
111
|
-
const visible = await healedLoc.first().isVisible({ timeout: 1500 }).catch(() => false);
|
|
113
|
+
const visible = await healedLoc.first().isVisible({ timeout: parseInt(process.env.TEST_WAIT_TIMEOUT_MS ?? '1500', 10) }).catch(() => false);
|
|
112
114
|
if (visible) {
|
|
113
115
|
return {
|
|
114
116
|
loc: healedLoc,
|
|
@@ -157,7 +159,7 @@ async function runPlan(
|
|
|
157
159
|
};
|
|
158
160
|
|
|
159
161
|
try {
|
|
160
|
-
const timeout = step.timeout ?? 8000;
|
|
162
|
+
const timeout = step.timeout ?? parseInt(process.env.TEST_STEP_TIMEOUT_MS ?? '8000', 10);
|
|
161
163
|
|
|
162
164
|
switch (step.action) {
|
|
163
165
|
case 'navigate':
|
|
@@ -169,6 +171,8 @@ async function runPlan(
|
|
|
169
171
|
case 'click': {
|
|
170
172
|
const r = await resolveLocator(page, step, options.healCtx);
|
|
171
173
|
if (!r) throw new Error('Could not resolve locator');
|
|
174
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
175
|
+
result.elementSelector = step.selector;
|
|
172
176
|
await r.loc.first().click({ timeout });
|
|
173
177
|
applyHealInfo(result, r);
|
|
174
178
|
break;
|
|
@@ -177,6 +181,8 @@ async function runPlan(
|
|
|
177
181
|
case 'fill': {
|
|
178
182
|
const r = await resolveLocator(page, step, options.healCtx);
|
|
179
183
|
if (!r) throw new Error('Could not resolve locator');
|
|
184
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
185
|
+
result.elementSelector = step.selector;
|
|
180
186
|
await r.loc.first().fill(step.value ?? '', { timeout });
|
|
181
187
|
applyHealInfo(result, r);
|
|
182
188
|
break;
|
|
@@ -185,6 +191,8 @@ async function runPlan(
|
|
|
185
191
|
case 'select': {
|
|
186
192
|
const r = await resolveLocator(page, step, options.healCtx);
|
|
187
193
|
if (!r) throw new Error('Could not resolve locator');
|
|
194
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
195
|
+
result.elementSelector = step.selector;
|
|
188
196
|
await r.loc.first().selectOption(step.value ?? '', { timeout });
|
|
189
197
|
applyHealInfo(result, r);
|
|
190
198
|
break;
|
|
@@ -193,6 +201,8 @@ async function runPlan(
|
|
|
193
201
|
case 'hover': {
|
|
194
202
|
const r = await resolveLocator(page, step, options.healCtx);
|
|
195
203
|
if (!r) throw new Error('Could not resolve locator');
|
|
204
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
205
|
+
result.elementSelector = step.selector;
|
|
196
206
|
await r.loc.first().hover({ timeout });
|
|
197
207
|
applyHealInfo(result, r);
|
|
198
208
|
break;
|
|
@@ -206,6 +216,8 @@ async function runPlan(
|
|
|
206
216
|
const r = await resolveLocator(page, step, options.healCtx);
|
|
207
217
|
if (!r) throw new Error('Could not resolve locator');
|
|
208
218
|
await r.loc.first().waitFor({ state: 'visible', timeout });
|
|
219
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
220
|
+
result.elementSelector = step.selector;
|
|
209
221
|
applyHealInfo(result, r);
|
|
210
222
|
break;
|
|
211
223
|
}
|
|
@@ -217,6 +229,8 @@ async function runPlan(
|
|
|
217
229
|
if (!text?.includes(step.value ?? '')) {
|
|
218
230
|
throw new Error(`Expected text "${step.value}" not found. Got: "${text?.slice(0, 100)}"`);
|
|
219
231
|
}
|
|
232
|
+
result.elementBoundingRect = await r.loc.first().boundingBox().catch(() => null);
|
|
233
|
+
result.elementSelector = step.selector;
|
|
220
234
|
applyHealInfo(result, r);
|
|
221
235
|
break;
|
|
222
236
|
}
|
|
@@ -230,7 +244,7 @@ async function runPlan(
|
|
|
230
244
|
}
|
|
231
245
|
|
|
232
246
|
case 'wait':
|
|
233
|
-
await page.waitForTimeout(step.timeout ?? 2000);
|
|
247
|
+
await page.waitForTimeout(step.timeout ?? parseInt(process.env.TEST_STEP_DELAY_MS ?? '2000', 10));
|
|
234
248
|
break;
|
|
235
249
|
|
|
236
250
|
case 'screenshot': {
|
|
@@ -251,7 +265,7 @@ async function runPlan(
|
|
|
251
265
|
}
|
|
252
266
|
|
|
253
267
|
try {
|
|
254
|
-
await page.waitForLoadState('networkidle', { timeout: 1500 }).catch(() => {});
|
|
268
|
+
await page.waitForLoadState('networkidle', { timeout: parseInt(process.env.TEST_NETWORK_IDLE_TIMEOUT_MS ?? '1500', 10) }).catch(() => {});
|
|
255
269
|
if (!result.screenshotPath) {
|
|
256
270
|
const ssPath = `${options.screenshotDir}/${options.testRunId}-step-${step.stepNumber}.png`;
|
|
257
271
|
await page.screenshot({ path: ssPath, fullPage: true });
|
package/types/crawler.d.ts
CHANGED
|
@@ -35,10 +35,24 @@ export declare function crawlProject(opts: {
|
|
|
35
35
|
tenantId: string;
|
|
36
36
|
projectId: string;
|
|
37
37
|
appUrl: string;
|
|
38
|
+
/** The viewport used for this crawl pass */
|
|
39
|
+
viewport?: 'DESKTOP' | 'MOBILE_PORTRAIT' | 'TABLET';
|
|
38
40
|
credentials?: Credentials;
|
|
39
41
|
/** Playwright storageState JSON string — if present, session is restored and AI login is skipped. */
|
|
40
42
|
storageState?: string;
|
|
41
43
|
screenshotDir?: string;
|
|
44
|
+
/** Cloud storage config for immediate screenshot upload. If set, screenshots are uploaded to cloud immediately after capture and cloudUrl is persisted. */
|
|
45
|
+
storageConfig?: {
|
|
46
|
+
provider: string;
|
|
47
|
+
bucket: string;
|
|
48
|
+
region?: string;
|
|
49
|
+
accountId?: string;
|
|
50
|
+
endpointUrl?: string;
|
|
51
|
+
accessKeyId?: string;
|
|
52
|
+
secretAccessKey?: string;
|
|
53
|
+
prefix?: string;
|
|
54
|
+
publicUrl?: string;
|
|
55
|
+
} | null;
|
|
42
56
|
/** When true, run a fast URL-only discovery crawl — skips screenshots, DB saves, and AI element extraction. Returns discoveredUrls in the result object. */
|
|
43
57
|
discoveryMode?: boolean;
|
|
44
58
|
maxScreens?: number;
|
package/types/test-executor.d.ts
CHANGED
|
@@ -13,6 +13,13 @@ export interface StepResult {
|
|
|
13
13
|
healedVia?: 'mechanical' | 'ai';
|
|
14
14
|
healReasoning?: string;
|
|
15
15
|
healConfidence?: number;
|
|
16
|
+
elementBoundingRect?: {
|
|
17
|
+
x: number;
|
|
18
|
+
y: number;
|
|
19
|
+
width: number;
|
|
20
|
+
height: number;
|
|
21
|
+
} | null;
|
|
22
|
+
elementSelector?: string;
|
|
16
23
|
}
|
|
17
24
|
export interface ExecutionResult {
|
|
18
25
|
stepResults: StepResult[];
|