@doppelgangerdev/doppelganger 0.5.7 → 0.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +2 -2
- package/README.md +9 -29
- package/agent.js +200 -101
- package/headful.js +126 -126
- package/package.json +2 -2
- package/scrape.js +249 -284
- package/server.js +469 -359
package/headful.js
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
1
|
const { chromium } = require('playwright');
|
|
2
2
|
const fs = require('fs');
|
|
3
3
|
const path = require('path');
|
|
4
|
-
const { getProxySelection } = require('./proxy-rotation');
|
|
5
|
-
const { selectUserAgent } = require('./user-agent-settings');
|
|
4
|
+
const { getProxySelection } = require('./proxy-rotation');
|
|
5
|
+
const { selectUserAgent } = require('./user-agent-settings');
|
|
6
6
|
|
|
7
7
|
const STORAGE_STATE_PATH = path.join(__dirname, 'storage_state.json');
|
|
8
|
-
const STORAGE_STATE_FILE = (() => {
|
|
9
|
-
try {
|
|
10
|
-
if (fs.existsSync(STORAGE_STATE_PATH)) {
|
|
11
|
-
const stat = fs.statSync(STORAGE_STATE_PATH);
|
|
12
|
-
if (stat.isDirectory()) {
|
|
13
|
-
return path.join(STORAGE_STATE_PATH, 'storage_state.json');
|
|
14
|
-
}
|
|
15
|
-
}
|
|
16
|
-
} catch {}
|
|
17
|
-
return STORAGE_STATE_PATH;
|
|
18
|
-
})();
|
|
19
|
-
|
|
20
|
-
const parseBooleanFlag = (value) => {
|
|
21
|
-
if (typeof value === 'boolean') return value;
|
|
22
|
-
if (value === undefined || value === null) return false;
|
|
23
|
-
const normalized = String(value).toLowerCase();
|
|
24
|
-
return normalized === 'true' || normalized === '1';
|
|
25
|
-
};
|
|
26
|
-
|
|
27
|
-
let activeSession = null;
|
|
28
|
-
|
|
29
|
-
const teardownActiveSession = async () => {
|
|
8
|
+
const STORAGE_STATE_FILE = (() => {
|
|
9
|
+
try {
|
|
10
|
+
if (fs.existsSync(STORAGE_STATE_PATH)) {
|
|
11
|
+
const stat = fs.statSync(STORAGE_STATE_PATH);
|
|
12
|
+
if (stat.isDirectory()) {
|
|
13
|
+
return path.join(STORAGE_STATE_PATH, 'storage_state.json');
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
} catch {}
|
|
17
|
+
return STORAGE_STATE_PATH;
|
|
18
|
+
})();
|
|
19
|
+
|
|
20
|
+
const parseBooleanFlag = (value) => {
|
|
21
|
+
if (typeof value === 'boolean') return value;
|
|
22
|
+
if (value === undefined || value === null) return false;
|
|
23
|
+
const normalized = String(value).toLowerCase();
|
|
24
|
+
return normalized === 'true' || normalized === '1';
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
let activeSession = null;
|
|
28
|
+
|
|
29
|
+
const teardownActiveSession = async () => {
|
|
30
30
|
if (!activeSession) return;
|
|
31
31
|
try {
|
|
32
32
|
if (activeSession.interval) clearInterval(activeSession.interval);
|
|
33
33
|
} catch {}
|
|
34
|
-
try {
|
|
35
|
-
if (activeSession.context && !activeSession.stateless) {
|
|
36
|
-
await activeSession.context.storageState({ path: STORAGE_STATE_FILE });
|
|
37
|
-
}
|
|
38
|
-
} catch {}
|
|
34
|
+
try {
|
|
35
|
+
if (activeSession.context && !activeSession.stateless) {
|
|
36
|
+
await activeSession.context.storageState({ path: STORAGE_STATE_FILE });
|
|
37
|
+
}
|
|
38
|
+
} catch {}
|
|
39
39
|
try {
|
|
40
40
|
if (activeSession.browser) {
|
|
41
41
|
await activeSession.browser.close();
|
|
@@ -49,15 +49,15 @@ async function handleHeadful(req, res) {
|
|
|
49
49
|
await teardownActiveSession();
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
const url = req.body.url || req.query.url || 'https://www.google.com';
|
|
53
|
-
const rotateProxiesRaw = req.body.rotateProxies ?? req.query.rotateProxies;
|
|
54
|
-
const rotateProxies = String(rotateProxiesRaw).toLowerCase() === 'true' || rotateProxiesRaw === true;
|
|
55
|
-
const statelessExecutionRaw = req.body.statelessExecution ?? req.query.statelessExecution;
|
|
56
|
-
const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
|
|
57
|
-
|
|
58
|
-
activeSession = { status: 'starting', startedAt: Date.now(), stateless: statelessExecution };
|
|
52
|
+
const url = req.body.url || req.query.url || 'https://www.google.com';
|
|
53
|
+
const rotateProxiesRaw = req.body.rotateProxies ?? req.query.rotateProxies;
|
|
54
|
+
const rotateProxies = String(rotateProxiesRaw).toLowerCase() === 'true' || rotateProxiesRaw === true;
|
|
55
|
+
const statelessExecutionRaw = req.body.statelessExecution ?? req.query.statelessExecution;
|
|
56
|
+
const statelessExecution = parseBooleanFlag(statelessExecutionRaw);
|
|
57
|
+
|
|
58
|
+
activeSession = { status: 'starting', startedAt: Date.now(), stateless: statelessExecution };
|
|
59
59
|
|
|
60
|
-
const selectedUA = selectUserAgent(false);
|
|
60
|
+
const selectedUA = await selectUserAgent(false);
|
|
61
61
|
|
|
62
62
|
console.log(`Opening headful browser for: ${url}`);
|
|
63
63
|
|
|
@@ -87,84 +87,84 @@ async function handleHeadful(req, res) {
|
|
|
87
87
|
timezoneId: 'America/New_York'
|
|
88
88
|
};
|
|
89
89
|
|
|
90
|
-
if (!statelessExecution && fs.existsSync(STORAGE_STATE_FILE)) {
|
|
91
|
-
console.log('Loading existing storage state...');
|
|
92
|
-
contextOptions.storageState = STORAGE_STATE_FILE;
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
const context = await browser.newContext(contextOptions);
|
|
96
|
-
await context.addInitScript(() => {
|
|
97
|
-
window.open = () => null;
|
|
98
|
-
document.addEventListener('click', (event) => {
|
|
99
|
-
const target = event.target;
|
|
100
|
-
const anchor = target && target.closest ? target.closest('a[target="_blank"]') : null;
|
|
101
|
-
if (anchor) {
|
|
102
|
-
event.preventDefault();
|
|
103
|
-
}
|
|
104
|
-
}, true);
|
|
105
|
-
});
|
|
106
|
-
await context.addInitScript(() => {
|
|
107
|
-
const cursorId = 'dg-cursor-overlay';
|
|
108
|
-
const dotId = 'dg-click-dot';
|
|
109
|
-
if (document.getElementById(cursorId)) return;
|
|
110
|
-
const cursor = document.createElement('div');
|
|
111
|
-
cursor.id = cursorId;
|
|
112
|
-
cursor.style.cssText = [
|
|
113
|
-
'position:fixed',
|
|
114
|
-
'top:0',
|
|
115
|
-
'left:0',
|
|
116
|
-
'width:18px',
|
|
117
|
-
'height:18px',
|
|
118
|
-
'margin-left:-9px',
|
|
119
|
-
'margin-top:-9px',
|
|
120
|
-
'border:2px solid rgba(56,189,248,0.7)',
|
|
121
|
-
'background:rgba(56,189,248,0.25)',
|
|
122
|
-
'border-radius:50%',
|
|
123
|
-
'box-shadow:0 0 10px rgba(56,189,248,0.6)',
|
|
124
|
-
'pointer-events:none',
|
|
125
|
-
'z-index:2147483647',
|
|
126
|
-
'transform:translate3d(0,0,0)',
|
|
127
|
-
'transition:transform 60ms ease-out'
|
|
128
|
-
].join(';');
|
|
129
|
-
const dot = document.createElement('div');
|
|
130
|
-
dot.id = dotId;
|
|
131
|
-
dot.style.cssText = [
|
|
132
|
-
'position:fixed',
|
|
133
|
-
'top:0',
|
|
134
|
-
'left:0',
|
|
135
|
-
'width:10px',
|
|
136
|
-
'height:10px',
|
|
137
|
-
'margin-left:-5px',
|
|
138
|
-
'margin-top:-5px',
|
|
139
|
-
'background:rgba(239,68,68,0.9)',
|
|
140
|
-
'border-radius:50%',
|
|
141
|
-
'box-shadow:0 0 12px rgba(239,68,68,0.8)',
|
|
142
|
-
'pointer-events:none',
|
|
143
|
-
'z-index:2147483647',
|
|
144
|
-
'opacity:0',
|
|
145
|
-
'transform:translate3d(0,0,0) scale(0.6)',
|
|
146
|
-
'transition:opacity 120ms ease, transform 120ms ease'
|
|
147
|
-
].join(';');
|
|
148
|
-
document.documentElement.appendChild(cursor);
|
|
149
|
-
document.documentElement.appendChild(dot);
|
|
150
|
-
const move = (x, y) => {
|
|
151
|
-
cursor.style.transform = `translate3d(${x}px, ${y}px, 0)`;
|
|
152
|
-
};
|
|
153
|
-
window.addEventListener('mousemove', (e) => move(e.clientX, e.clientY), { passive: true });
|
|
154
|
-
window.addEventListener('click', (e) => {
|
|
155
|
-
dot.style.left = `${e.clientX}px`;
|
|
156
|
-
dot.style.top = `${e.clientY}px`;
|
|
157
|
-
dot.style.opacity = '1';
|
|
158
|
-
dot.style.transform = 'translate3d(0,0,0) scale(1)';
|
|
159
|
-
cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(0.65)`;
|
|
160
|
-
setTimeout(() => {
|
|
161
|
-
dot.style.opacity = '0';
|
|
162
|
-
dot.style.transform = 'translate3d(0,0,0) scale(0.6)';
|
|
163
|
-
cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(1)`;
|
|
164
|
-
}, 180);
|
|
165
|
-
}, true);
|
|
166
|
-
});
|
|
167
|
-
const page = await context.newPage();
|
|
90
|
+
if (!statelessExecution && fs.existsSync(STORAGE_STATE_FILE)) {
|
|
91
|
+
console.log('Loading existing storage state...');
|
|
92
|
+
contextOptions.storageState = STORAGE_STATE_FILE;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const context = await browser.newContext(contextOptions);
|
|
96
|
+
await context.addInitScript(() => {
|
|
97
|
+
window.open = () => null;
|
|
98
|
+
document.addEventListener('click', (event) => {
|
|
99
|
+
const target = event.target;
|
|
100
|
+
const anchor = target && target.closest ? target.closest('a[target="_blank"]') : null;
|
|
101
|
+
if (anchor) {
|
|
102
|
+
event.preventDefault();
|
|
103
|
+
}
|
|
104
|
+
}, true);
|
|
105
|
+
});
|
|
106
|
+
await context.addInitScript(() => {
|
|
107
|
+
const cursorId = 'dg-cursor-overlay';
|
|
108
|
+
const dotId = 'dg-click-dot';
|
|
109
|
+
if (document.getElementById(cursorId)) return;
|
|
110
|
+
const cursor = document.createElement('div');
|
|
111
|
+
cursor.id = cursorId;
|
|
112
|
+
cursor.style.cssText = [
|
|
113
|
+
'position:fixed',
|
|
114
|
+
'top:0',
|
|
115
|
+
'left:0',
|
|
116
|
+
'width:18px',
|
|
117
|
+
'height:18px',
|
|
118
|
+
'margin-left:-9px',
|
|
119
|
+
'margin-top:-9px',
|
|
120
|
+
'border:2px solid rgba(56,189,248,0.7)',
|
|
121
|
+
'background:rgba(56,189,248,0.25)',
|
|
122
|
+
'border-radius:50%',
|
|
123
|
+
'box-shadow:0 0 10px rgba(56,189,248,0.6)',
|
|
124
|
+
'pointer-events:none',
|
|
125
|
+
'z-index:2147483647',
|
|
126
|
+
'transform:translate3d(0,0,0)',
|
|
127
|
+
'transition:transform 60ms ease-out'
|
|
128
|
+
].join(';');
|
|
129
|
+
const dot = document.createElement('div');
|
|
130
|
+
dot.id = dotId;
|
|
131
|
+
dot.style.cssText = [
|
|
132
|
+
'position:fixed',
|
|
133
|
+
'top:0',
|
|
134
|
+
'left:0',
|
|
135
|
+
'width:10px',
|
|
136
|
+
'height:10px',
|
|
137
|
+
'margin-left:-5px',
|
|
138
|
+
'margin-top:-5px',
|
|
139
|
+
'background:rgba(239,68,68,0.9)',
|
|
140
|
+
'border-radius:50%',
|
|
141
|
+
'box-shadow:0 0 12px rgba(239,68,68,0.8)',
|
|
142
|
+
'pointer-events:none',
|
|
143
|
+
'z-index:2147483647',
|
|
144
|
+
'opacity:0',
|
|
145
|
+
'transform:translate3d(0,0,0) scale(0.6)',
|
|
146
|
+
'transition:opacity 120ms ease, transform 120ms ease'
|
|
147
|
+
].join(';');
|
|
148
|
+
document.documentElement.appendChild(cursor);
|
|
149
|
+
document.documentElement.appendChild(dot);
|
|
150
|
+
const move = (x, y) => {
|
|
151
|
+
cursor.style.transform = `translate3d(${x}px, ${y}px, 0)`;
|
|
152
|
+
};
|
|
153
|
+
window.addEventListener('mousemove', (e) => move(e.clientX, e.clientY), { passive: true });
|
|
154
|
+
window.addEventListener('click', (e) => {
|
|
155
|
+
dot.style.left = `${e.clientX}px`;
|
|
156
|
+
dot.style.top = `${e.clientY}px`;
|
|
157
|
+
dot.style.opacity = '1';
|
|
158
|
+
dot.style.transform = 'translate3d(0,0,0) scale(1)';
|
|
159
|
+
cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(0.65)`;
|
|
160
|
+
setTimeout(() => {
|
|
161
|
+
dot.style.opacity = '0';
|
|
162
|
+
dot.style.transform = 'translate3d(0,0,0) scale(0.6)';
|
|
163
|
+
cursor.style.transform = `translate3d(${e.clientX}px, ${e.clientY}px, 0) scale(1)`;
|
|
164
|
+
}, 180);
|
|
165
|
+
}, true);
|
|
166
|
+
});
|
|
167
|
+
const page = await context.newPage();
|
|
168
168
|
|
|
169
169
|
const closeIfExtra = async (extraPage) => {
|
|
170
170
|
if (!extraPage || extraPage === page) return;
|
|
@@ -182,11 +182,11 @@ async function handleHeadful(req, res) {
|
|
|
182
182
|
console.log('IMPORTANT: Close the page/tab or wait for saves.');
|
|
183
183
|
|
|
184
184
|
// Function to save state
|
|
185
|
-
const saveState = async () => {
|
|
186
|
-
if (statelessExecution) return;
|
|
187
|
-
try {
|
|
188
|
-
await context.storageState({ path: STORAGE_STATE_FILE });
|
|
189
|
-
console.log('Storage state saved successfully.');
|
|
185
|
+
const saveState = async () => {
|
|
186
|
+
if (statelessExecution) return;
|
|
187
|
+
try {
|
|
188
|
+
await context.storageState({ path: STORAGE_STATE_FILE });
|
|
189
|
+
console.log('Storage state saved successfully.');
|
|
190
190
|
} catch (e) {
|
|
191
191
|
// If context is closed, this will fail, which is expected during shutdown
|
|
192
192
|
}
|
|
@@ -195,7 +195,7 @@ async function handleHeadful(req, res) {
|
|
|
195
195
|
// Auto-save every 10 seconds while the window is open
|
|
196
196
|
const interval = setInterval(saveState, 10000);
|
|
197
197
|
|
|
198
|
-
activeSession = { browser, context, interval, status: 'running', startedAt: activeSession.startedAt, stateless: statelessExecution };
|
|
198
|
+
activeSession = { browser, context, interval, status: 'running', startedAt: activeSession.startedAt, stateless: statelessExecution };
|
|
199
199
|
|
|
200
200
|
// Save when the page is closed
|
|
201
201
|
page.on('close', async () => {
|
|
@@ -204,11 +204,11 @@ async function handleHeadful(req, res) {
|
|
|
204
204
|
});
|
|
205
205
|
|
|
206
206
|
// Respond immediately; cleanup runs after disconnect
|
|
207
|
-
res.json({
|
|
208
|
-
message: 'Headful session started. Close the browser window or call /headful/stop to end.',
|
|
209
|
-
userAgentUsed: selectedUA,
|
|
210
|
-
path: statelessExecution ? null : STORAGE_STATE_FILE
|
|
211
|
-
});
|
|
207
|
+
res.json({
|
|
208
|
+
message: 'Headful session started. Close the browser window or call /headful/stop to end.',
|
|
209
|
+
userAgentUsed: selectedUA,
|
|
210
|
+
path: statelessExecution ? null : STORAGE_STATE_FILE
|
|
211
|
+
});
|
|
212
212
|
|
|
213
213
|
// Wait for the browser to disconnect (user closes the last window)
|
|
214
214
|
await new Promise((resolve) => browser.on('disconnected', resolve));
|
package/package.json
CHANGED