@detiq/crawler 1.1.25 → 1.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/crawler.js +2 -3
- package/dist/crawler.js.map +2 -2
- package/dist/job-queue.js +299 -64
- package/dist/job-queue.js.map +3 -3
- package/package.json +1 -1
- package/src/crawler.ts +5 -2
- package/src/failure-analysis-agent.ts +2 -1
- package/src/job-queue.ts +373 -74
- package/types/crawler.d.ts +4 -0
- package/types/job-queue.d.ts +44 -3
package/package.json
CHANGED
package/src/crawler.ts
CHANGED
|
@@ -373,7 +373,6 @@ async function launchChromeCDP(
|
|
|
373
373
|
'--remote-allow-origins=*',
|
|
374
374
|
'--no-first-run',
|
|
375
375
|
'--no-default-browser-check',
|
|
376
|
-
'--window-size=1280,900',
|
|
377
376
|
],
|
|
378
377
|
});
|
|
379
378
|
|
|
@@ -634,6 +633,10 @@ export async function crawlProject(opts: {
|
|
|
634
633
|
/** When false, disables the puppeteer-extra stealth plugin (playwright-extra) for this crawl.
|
|
635
634
|
* Default true — stealth is applied unless explicitly set to false. */
|
|
636
635
|
stealth?: boolean;
|
|
636
|
+
/** Desktop viewport width in pixels (default: 1440). */
|
|
637
|
+
viewportWidth?: number;
|
|
638
|
+
/** Desktop viewport height in pixels (default: 900). */
|
|
639
|
+
viewportHeight?: number;
|
|
637
640
|
/** Called every 10 successful screen captures with current visited URLs and remaining queue — used for checkpoint persistence. */
|
|
638
641
|
onCheckpoint?: (visitedUrls: string[], pendingQueue: string[]) => Promise<void>;
|
|
639
642
|
/** Discovery mode only: called after each page with the current set of discovered original URLs — enables streaming partial results to callers. */
|
|
@@ -829,7 +832,7 @@ export async function crawlProject(opts: {
|
|
|
829
832
|
'--disable-blink-features=AutomationControlled',
|
|
830
833
|
'--disable-http2',
|
|
831
834
|
'--lang=en-US,en',
|
|
832
|
-
|
|
835
|
+
`--window-size=${opts.viewportWidth ?? 1440},${opts.viewportHeight ?? 900}`,
|
|
833
836
|
'--disable-features=IsolateOrigins,site-per-process',
|
|
834
837
|
];
|
|
835
838
|
|
|
@@ -36,7 +36,8 @@ Error: ${ctx.error}${consoleSection}${networkSection}`;
|
|
|
36
36
|
if (ctx.screenshotPath) {
|
|
37
37
|
try {
|
|
38
38
|
const img = readFileSync(ctx.screenshotPath);
|
|
39
|
-
const
|
|
39
|
+
const imgSeed = deterministicSeed(`failure-img:${ctx.tenantId ?? ''}:${(ctx.stepDescription ?? '').slice(0, 100)}:${ctx.error.slice(0, 200)}`);
|
|
40
|
+
const result = await askTrackedWithImages('', userText, [{ base64: img.toString('base64'), mediaType: 'image/png' }], llmCfg, { temperature: 0, seed: imgSeed });
|
|
40
41
|
return result.text || null;
|
|
41
42
|
} catch { /* fall through to text-only */ }
|
|
42
43
|
}
|