@doppelgangerdev/doppelganger 0.5.6 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +177 -157
- package/README.md +278 -273
- package/agent.js +42 -27
- package/dist/assets/index-Cwmqk52G.js +19 -0
- package/dist/assets/{index-isZw-0dm.css → index-CxzMazJO.css} +1 -1
- package/dist/captures/run_1769734411613_783_scrape_1769734425256.png +0 -0
- package/dist/captures/run_1769734411613_783_scrape_1769734428068.webm +0 -0
- package/dist/captures/run_1769734522774_unknown_scrape_1769734535501.png +0 -0
- package/dist/captures/run_1769734522774_unknown_scrape_1769734538775.webm +0 -0
- package/dist/index.html +133 -22
- package/headful.js +49 -39
- package/package.json +1 -1
- package/public/captures/run_1770084709375_263_scrape_1770084720880.png +0 -0
- package/public/captures/run_1770084753714_765_agent_1770084772039.png +0 -0
- package/public/captures/run_1770084753714_765_agent_1770084774318.webm +0 -0
- package/public/captures/run_1770084826401_32_scrape_1770084832653.png +0 -0
- package/public/captures/run_1770084826401_32_scrape_1770084835345.webm +0 -0
- package/public/captures/run_1770084861758_434_scrape_1770084869777.png +0 -0
- package/public/captures/run_1770084861758_434_scrape_1770084875604.webm +0 -0
- package/public/captures/run_1770084870793_97_scrape_1770084879360.png +0 -0
- package/public/captures/run_1770084870793_97_scrape_1770084882219.webm +0 -0
- package/scrape.js +34 -17
- package/server.js +50 -30
- package/dist/assets/index-BKB-zmAO.js +0 -19
package/agent.js
CHANGED
|
@@ -110,11 +110,17 @@ async function overshootScroll(page, targetY) {
|
|
|
110
110
|
}
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
const punctuationPause = /[.,!?;:]/;
|
|
114
|
-
|
|
115
|
-
const randomBetween = (min, max) => min + Math.random() * (max - min);
|
|
116
|
-
|
|
117
|
-
|
|
113
|
+
const punctuationPause = /[.,!?;:]/;
|
|
114
|
+
|
|
115
|
+
const randomBetween = (min, max) => min + Math.random() * (max - min);
|
|
116
|
+
const parseBooleanFlag = (value) => {
|
|
117
|
+
if (typeof value === 'boolean') return value;
|
|
118
|
+
if (value === undefined || value === null) return false;
|
|
119
|
+
const normalized = String(value).toLowerCase();
|
|
120
|
+
return normalized === 'true' || normalized === '1';
|
|
121
|
+
};
|
|
122
|
+
|
|
123
|
+
async function humanType(page, selector, text, options = {}) {
|
|
118
124
|
const { allowTypos = false, naturalTyping = false, fatigue = false } = options;
|
|
119
125
|
if (selector) await page.focus(selector);
|
|
120
126
|
const chars = text.split('');
|
|
@@ -166,12 +172,16 @@ async function handleAgent(req, res) {
|
|
|
166
172
|
let { url, actions, wait: globalWait, rotateUserAgents, rotateProxies, humanTyping, stealth = {} } = data;
|
|
167
173
|
const runId = data.runId ? String(data.runId) : null;
|
|
168
174
|
const captureRunId = runId || `run_${Date.now()}_unknown`;
|
|
169
|
-
const includeShadowDomRaw = data.includeShadowDom ?? req.query.includeShadowDom;
|
|
170
|
-
const includeShadowDom = includeShadowDomRaw === undefined
|
|
171
|
-
? true
|
|
172
|
-
: !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
|
|
173
|
-
const
|
|
174
|
-
|
|
175
|
+
const includeShadowDomRaw = data.includeShadowDom ?? req.query.includeShadowDom;
|
|
176
|
+
const includeShadowDom = includeShadowDomRaw === undefined
|
|
177
|
+
? true
|
|
178
|
+
: !(String(includeShadowDomRaw).toLowerCase() === 'false' || includeShadowDomRaw === false);
|
|
179
|
+
const disableRecordingRaw = data.disableRecording ?? req.query.disableRecording;
|
|
180
|
+
const disableRecording = parseBooleanFlag(disableRecordingRaw);
|
|
181
|
+
const statelessExecutionRaw = data.statelessExecution ?? req.query.statelessExecution;
|
|
182
|
+
const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
|
|
183
|
+
const {
|
|
184
|
+
allowTypos = false,
|
|
175
185
|
idleMovements = false,
|
|
176
186
|
overscroll = false,
|
|
177
187
|
deadClicks = false,
|
|
@@ -434,21 +444,24 @@ async function handleAgent(req, res) {
|
|
|
434
444
|
? { width: 1280 + Math.floor(Math.random() * 640), height: 720 + Math.floor(Math.random() * 360) }
|
|
435
445
|
: { width: 1366, height: 768 };
|
|
436
446
|
|
|
437
|
-
const contextOptions = {
|
|
438
|
-
userAgent: selectedUA,
|
|
439
|
-
viewport,
|
|
440
|
-
deviceScaleFactor: 1,
|
|
441
|
-
locale: 'en-US',
|
|
442
|
-
timezoneId: 'America/New_York',
|
|
443
|
-
colorScheme: 'dark',
|
|
444
|
-
permissions: ['geolocation'],
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
if (
|
|
449
|
-
contextOptions.storageState = STORAGE_STATE_FILE;
|
|
450
|
-
}
|
|
451
|
-
|
|
447
|
+
const contextOptions = {
|
|
448
|
+
userAgent: selectedUA,
|
|
449
|
+
viewport,
|
|
450
|
+
deviceScaleFactor: 1,
|
|
451
|
+
locale: 'en-US',
|
|
452
|
+
timezoneId: 'America/New_York',
|
|
453
|
+
colorScheme: 'dark',
|
|
454
|
+
permissions: ['geolocation'],
|
|
455
|
+
};
|
|
456
|
+
|
|
457
|
+
const shouldUseStorageState = !statelessExecution && fs.existsSync(STORAGE_STATE_FILE);
|
|
458
|
+
if (shouldUseStorageState) {
|
|
459
|
+
contextOptions.storageState = STORAGE_STATE_FILE;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
if (!disableRecording) {
|
|
463
|
+
contextOptions.recordVideo = { dir: recordingsDir, size: viewport };
|
|
464
|
+
}
|
|
452
465
|
context = await browser.newContext(contextOptions);
|
|
453
466
|
|
|
454
467
|
await context.addInitScript(() => {
|
|
@@ -1434,7 +1447,9 @@ async function handleAgent(req, res) {
|
|
|
1434
1447
|
};
|
|
1435
1448
|
|
|
1436
1449
|
const video = page.video();
|
|
1437
|
-
|
|
1450
|
+
if (!statelessExecution) {
|
|
1451
|
+
try { await context.storageState({ path: STORAGE_STATE_FILE }); } catch {}
|
|
1452
|
+
}
|
|
1438
1453
|
try { await context.close(); } catch {}
|
|
1439
1454
|
if (video) {
|
|
1440
1455
|
try {
|