@imenam/simple-scraper 1.0.8 → 1.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gui.js +115 -8
- package/dist/index.js +15 -5
- package/dist/session-rpc.js +26 -0
- package/package.json +2 -1
- package/public/dist/assets/index-DAjr-PsH.js +12 -0
- package/public/dist/assets/index-IkSxq6_f.css +1 -0
- package/public/dist/index.html +2 -2
- package/public/dist/assets/index-C9ZfKLEL.css +0 -1
- package/public/dist/assets/index-COlwJN-b.js +0 -10
package/dist/gui.js
CHANGED
|
@@ -3,7 +3,16 @@ import bodyParser from 'body-parser';
|
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import fs from 'fs';
|
|
5
5
|
import { fileURLToPath } from 'url';
|
|
6
|
-
import { setupLogging, ProxyClient } from '@imenam/mcp-gui-interface';
|
|
6
|
+
import { setupLogging, ProxyClient, IpcHub } from '@imenam/mcp-gui-interface';
|
|
7
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
8
|
+
const ipc = new IpcHub(process);
|
|
9
|
+
async function callMaster(type, data, timeoutMs = 5000) {
|
|
10
|
+
const resp = await ipc.request({ type, data }, timeoutMs);
|
|
11
|
+
if (resp.error)
|
|
12
|
+
throw new Error(resp.error);
|
|
13
|
+
return resp.data;
|
|
14
|
+
}
|
|
15
|
+
const SESSION_ID_RE = /^sess_[a-z0-9]{6}$/;
|
|
7
16
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
8
17
|
const rootDir = path.resolve(__dirname, '..');
|
|
9
18
|
setupLogging({ processLabel: 'GUI', logDir: process.env.MCP_LOG_DIR });
|
|
@@ -11,6 +20,47 @@ if (!process.env.PROXY_URL) {
|
|
|
11
20
|
process.stderr.write('[GUI] PROXY_URL non défini — arrêt.\n');
|
|
12
21
|
process.exit(0);
|
|
13
22
|
}
|
|
23
|
+
const APP_PATH = process.env.PROXY_APP_PATH ?? '/simple-scraper-mcp';
|
|
24
|
+
const APP_NAME = process.env.PROXY_APP_NAME ?? 'Simple Scraper MCP';
|
|
25
|
+
// ─── Cycle de vie du process ─────────────────────────────────────────────────
|
|
26
|
+
let proxyClient = null;
|
|
27
|
+
/**
|
|
28
|
+
* Libère la route avant de mourir. Sans ça le proxy continue de router vers un
|
|
29
|
+
* port mort (502 opaques) jusqu'à ce que son health check finisse par nettoyer.
|
|
30
|
+
*/
|
|
31
|
+
async function cleanupProxy() {
|
|
32
|
+
if (proxyClient)
|
|
33
|
+
await proxyClient.unregister();
|
|
34
|
+
process.exit(0);
|
|
35
|
+
}
|
|
36
|
+
process.on('SIGTERM', () => void cleanupProxy());
|
|
37
|
+
process.on('SIGINT', () => void cleanupProxy());
|
|
38
|
+
// Le master peut disparaître sans nous tuer (crash, kill -9) : sans ce watchdog
|
|
39
|
+
// la GUI reste orpheline et garde le port et la route du proxy.
|
|
40
|
+
const parentPid = process.ppid;
|
|
41
|
+
const parentCheckInterval = setInterval(() => {
|
|
42
|
+
try {
|
|
43
|
+
process.kill(parentPid, 0);
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
console.error(`[GUI] Master (PID ${parentPid}) disparu — arrêt.`);
|
|
47
|
+
clearInterval(parentCheckInterval);
|
|
48
|
+
void cleanupProxy();
|
|
49
|
+
}
|
|
50
|
+
}, 2000);
|
|
51
|
+
process.on('disconnect', () => {
|
|
52
|
+
console.error('[GUI] Canal IPC fermé par le master — arrêt.');
|
|
53
|
+
void cleanupProxy();
|
|
54
|
+
});
|
|
55
|
+
process.on('uncaughtException', (err) => {
|
|
56
|
+
console.error(`[GUI] Uncaught Exception: ${err.message}`);
|
|
57
|
+
console.error(err.stack ?? 'No stack trace');
|
|
58
|
+
process.exit(1);
|
|
59
|
+
});
|
|
60
|
+
process.on('unhandledRejection', (reason) => {
|
|
61
|
+
console.error(`[GUI] Unhandled Rejection: ${reason}`);
|
|
62
|
+
process.exit(1);
|
|
63
|
+
});
|
|
14
64
|
const getCookiesDir = () => process.env.COOKIES_DIR ?? null;
|
|
15
65
|
const app = express();
|
|
16
66
|
app.use(bodyParser.json());
|
|
@@ -59,6 +109,23 @@ app.post('/api/cookies', (req, res) => {
|
|
|
59
109
|
res.status(500).json({ success: false, error: message });
|
|
60
110
|
}
|
|
61
111
|
});
|
|
112
|
+
// ─── GET /api/cookies/:filename — lit le contenu d'un fichier ────────────────
|
|
113
|
+
app.get('/api/cookies/:filename', (req, res) => {
|
|
114
|
+
const cookiesDir = getCookiesDir();
|
|
115
|
+
if (!cookiesDir) {
|
|
116
|
+
res.status(400).json({ success: false, error: 'COOKIES_DIR not configured' });
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const safeName = path.basename(req.params.filename);
|
|
120
|
+
try {
|
|
121
|
+
const content = fs.readFileSync(path.join(cookiesDir, safeName), 'utf-8');
|
|
122
|
+
res.json({ success: true, content });
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
126
|
+
res.status(500).json({ success: false, error: message });
|
|
127
|
+
}
|
|
128
|
+
});
|
|
62
129
|
// ─── DELETE /api/cookies/:filename — supprime un fichier ──────────────────────
|
|
63
130
|
app.delete('/api/cookies/:filename', (req, res) => {
|
|
64
131
|
const cookiesDir = getCookiesDir();
|
|
@@ -76,21 +143,61 @@ app.delete('/api/cookies/:filename', (req, res) => {
|
|
|
76
143
|
res.status(500).json({ success: false, error: message });
|
|
77
144
|
}
|
|
78
145
|
});
|
|
146
|
+
// ─── GET /api/sessions — liste les sessions actives ──────────────────────────
|
|
147
|
+
app.get('/api/sessions', async (_req, res) => {
|
|
148
|
+
try {
|
|
149
|
+
const sessions = await callMaster('SESSION_LIST');
|
|
150
|
+
res.json({ success: true, sessions });
|
|
151
|
+
}
|
|
152
|
+
catch (error) {
|
|
153
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
154
|
+
res.status(500).json({ success: false, error: message });
|
|
155
|
+
}
|
|
156
|
+
});
|
|
157
|
+
// ─── GET /api/sessions/:id/screenshot — capture JPEG de la session ───────────
|
|
158
|
+
app.get('/api/sessions/:id/screenshot', async (req, res) => {
|
|
159
|
+
const { id } = req.params;
|
|
160
|
+
if (!SESSION_ID_RE.test(id)) {
|
|
161
|
+
res.status(400).json({ success: false, error: 'Invalid session ID' });
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
try {
|
|
165
|
+
const data = await callMaster('SESSION_SCREENSHOT', { sessionId: id });
|
|
166
|
+
res.type('image/jpeg').send(Buffer.from(data.base64, 'base64'));
|
|
167
|
+
}
|
|
168
|
+
catch (error) {
|
|
169
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
170
|
+
res.status(500).json({ success: false, error: message });
|
|
171
|
+
}
|
|
172
|
+
});
|
|
79
173
|
async function startServer() {
|
|
80
|
-
const
|
|
81
|
-
const result = await
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
174
|
+
const client = new ProxyClient(process.env.PROXY_URL);
|
|
175
|
+
const result = await client.register({ path: APP_PATH, name: APP_NAME });
|
|
176
|
+
// Une autre instance du serveur MCP sert déjà cette GUI. C'est le cas nominal
|
|
177
|
+
// quand le client redémarre le serveur : il lance le nouveau master avant de
|
|
178
|
+
// couper l'ancien, dont la GUI tient encore la route. Ce n'est pas un crash —
|
|
179
|
+
// on sort en 0 (sinon le launcher log "Giving up") sans toucher au proxy, dont
|
|
180
|
+
// la route appartient à l'autre instance.
|
|
181
|
+
if (result.error === 'HTTP 409') {
|
|
182
|
+
console.error('[GUI] Une autre instance sert déjà cette GUI — arrêt.');
|
|
183
|
+
ipc.send({
|
|
184
|
+
type: 'ALREADY_RUNNING',
|
|
185
|
+
data: { url: `${process.env.PROXY_URL}${APP_PATH}` },
|
|
186
|
+
timestamp: new Date().toISOString(),
|
|
187
|
+
});
|
|
188
|
+
process.exit(0);
|
|
189
|
+
}
|
|
85
190
|
if (!result.success) {
|
|
86
|
-
process.stderr.write(
|
|
191
|
+
process.stderr.write(`[GUI] Enregistrement proxy échoué (${result.error}) — arrêt.\n`);
|
|
87
192
|
process.exit(1);
|
|
88
193
|
}
|
|
194
|
+
// À partir d'ici la route nous appartient : cleanupProxy peut la libérer.
|
|
195
|
+
proxyClient = client;
|
|
89
196
|
const port = result.port;
|
|
90
197
|
app.post('/api/stop', async (_req, res) => {
|
|
91
198
|
console.error('POST /api/stop called');
|
|
92
199
|
res.json({ success: true, message: 'Stopping...' });
|
|
93
|
-
await
|
|
200
|
+
await client.unregister();
|
|
94
201
|
setTimeout(() => {
|
|
95
202
|
if (process.send && process.connected) {
|
|
96
203
|
process.send({ type: 'STOP' });
|
package/dist/index.js
CHANGED
|
@@ -11,6 +11,7 @@ import { getBrowser, closeBrowser, getDefaultTimeout } from './browser.js';
|
|
|
11
11
|
import { loadAllCookies } from './cookies.js';
|
|
12
12
|
import { GuiLauncher } from '@imenam/mcp-gui-interface';
|
|
13
13
|
import { createSession, getSession, closeSession, closeAllSessions, listSessions, getSessionLogs, clearSessionLogs } from './sessions.js';
|
|
14
|
+
import { registerSessionRpc } from './session-rpc.js';
|
|
14
15
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
15
16
|
const rootDir = path.resolve(__dirname, '..');
|
|
16
17
|
const envPath = path.join(rootDir, '.env');
|
|
@@ -19,8 +20,12 @@ setupLogging('MASTER');
|
|
|
19
20
|
const launcher = new GuiLauncher({
|
|
20
21
|
guiPath: path.join(__dirname, 'gui.js'),
|
|
21
22
|
onMessage: (msg) => {
|
|
22
|
-
if (msg.type === 'STOP')
|
|
23
|
+
if (msg.type === 'STOP') {
|
|
23
24
|
process.exit(0);
|
|
25
|
+
}
|
|
26
|
+
else if (msg.type === 'ALREADY_RUNNING') {
|
|
27
|
+
console.error(`[MASTER] GUI déjà servie par une autre instance : ${msg.data?.url ?? 'URL inconnue'}`);
|
|
28
|
+
}
|
|
24
29
|
},
|
|
25
30
|
});
|
|
26
31
|
const server = new McpServer({
|
|
@@ -608,16 +613,21 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
608
613
|
}
|
|
609
614
|
}
|
|
610
615
|
});
|
|
611
|
-
|
|
616
|
+
function launchGUI() {
|
|
612
617
|
if (!process.env.PROXY_URL) {
|
|
613
618
|
console.error('[MASTER] PROXY_URL non défini — GUI désactivée.');
|
|
619
|
+
return;
|
|
614
620
|
}
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
621
|
+
launcher.start();
|
|
622
|
+
registerSessionRpc(launcher);
|
|
623
|
+
}
|
|
624
|
+
async function main() {
|
|
618
625
|
const transport = new StdioServerTransport();
|
|
619
626
|
await server.connect(transport);
|
|
620
627
|
console.error('[MCP] Simple Scraper MCP server started');
|
|
628
|
+
// Après le connect, comme mcp-documentor : au redémarrage du serveur par le
|
|
629
|
+
// client, le master sortant a alors eu le temps de libérer la route.
|
|
630
|
+
launchGUI();
|
|
621
631
|
}
|
|
622
632
|
main().catch(async (err) => {
|
|
623
633
|
console.error('[MCP] Fatal error:', err);
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { getSession, listSessions } from './sessions.js';
|
|
2
|
+
export function registerSessionRpc(launcher) {
|
|
3
|
+
launcher.getIpcHub().onMessage(async (msg) => {
|
|
4
|
+
const reply = (data, error) => launcher.getIpcHub().send({
|
|
5
|
+
type: `${msg.type}_RESPONSE`,
|
|
6
|
+
correlationId: msg.correlationId,
|
|
7
|
+
data,
|
|
8
|
+
error,
|
|
9
|
+
timestamp: new Date().toISOString(),
|
|
10
|
+
});
|
|
11
|
+
try {
|
|
12
|
+
if (msg.type === 'SESSION_LIST') {
|
|
13
|
+
reply(listSessions());
|
|
14
|
+
}
|
|
15
|
+
else if (msg.type === 'SESSION_SCREENSHOT') {
|
|
16
|
+
const { sessionId } = msg.data;
|
|
17
|
+
const page = getSession(sessionId);
|
|
18
|
+
const buffer = await page.screenshot({ type: 'jpeg', quality: 60 });
|
|
19
|
+
reply({ base64: Buffer.from(buffer).toString('base64') });
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
catch (err) {
|
|
23
|
+
reply(undefined, err instanceof Error ? err.message : String(err));
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@imenam/simple-scraper",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.12",
|
|
4
4
|
"description": "MCP server for web scraping and JavaScript execution using Puppeteer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"dotenv": "^17.3.1",
|
|
42
42
|
"express": "^5.2.1",
|
|
43
43
|
"puppeteer": "^24.40.0",
|
|
44
|
+
"react-router-dom": "^7.15.1",
|
|
44
45
|
"zod": "^4.3.6"
|
|
45
46
|
},
|
|
46
47
|
"devDependencies": {
|