@imenam/simple-scraper 1.0.24 → 1.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +444 -398
- package/dist/browser.js +40 -11
- package/dist/cli/setup-mcp.js +2 -2
- package/dist/cookie-sync.js +67 -0
- package/dist/cookies.js +28 -40
- package/dist/data-dir.js +24 -0
- package/dist/gui.js +83 -85
- package/dist/index.js +36 -38
- package/dist/keep-alive.js +51 -0
- package/dist/login.js +157 -0
- package/dist/relay.js +11 -7
- package/dist/remote-input.js +67 -0
- package/dist/rpc.js +184 -0
- package/dist/sessions.js +23 -11
- package/dist/signal-picker.js +144 -0
- package/dist/site-check.js +88 -0
- package/dist/site-store.js +210 -0
- package/package.json +2 -1
- package/public/dist/assets/index-BAs4dkz0.css +1 -0
- package/public/dist/assets/index-D44p8v3x.js +11 -0
- package/public/dist/index.html +13 -13
- package/public/dist/assets/index-DAjr-PsH.js +0 -12
- package/public/dist/assets/index-IkSxq6_f.css +0 -1
package/dist/index.js
CHANGED
|
@@ -9,10 +9,12 @@ import { fileURLToPath } from 'url';
|
|
|
9
9
|
import { z } from 'zod';
|
|
10
10
|
import { setupLogging } from './logger.js';
|
|
11
11
|
import { getBrowser, closeBrowser, getDefaultTimeout } from './browser.js';
|
|
12
|
-
import {
|
|
12
|
+
import { scheduleJarSync, startPeriodicJarSync } from './cookie-sync.js';
|
|
13
|
+
import { getDataDir } from './data-dir.js';
|
|
14
|
+
import { startKeepAlive } from './keep-alive.js';
|
|
13
15
|
import { GuiLauncher, reconnectGuiToolDefinition } from '@imenam/mcp-gui-interface';
|
|
14
16
|
import { createSession, getSession, closeSession, closeAllSessions, listSessions, getSessionLogs, clearSessionLogs, resolveViewport } from './sessions.js';
|
|
15
|
-
import {
|
|
17
|
+
import { registerRpc } from './rpc.js';
|
|
16
18
|
import { deliverFile, RELAY_SHAPE_DESCRIPTIONS } from './relay.js';
|
|
17
19
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
18
20
|
const rootDir = path.resolve(__dirname, '..');
|
|
@@ -62,22 +64,6 @@ function toViewportOverride(v) {
|
|
|
62
64
|
return undefined;
|
|
63
65
|
return { width: v.width, height: v.height, deviceScaleFactor: v.device_scale_factor };
|
|
64
66
|
}
|
|
65
|
-
// ─── One-shot helpers ─────────────────────────────────────────────────────────
|
|
66
|
-
async function setCookies(page, cookies, pageUrl) {
|
|
67
|
-
const prepared = cookies.map(c => {
|
|
68
|
-
if (!c.domain) {
|
|
69
|
-
return { ...c, url: pageUrl };
|
|
70
|
-
}
|
|
71
|
-
return c;
|
|
72
|
-
});
|
|
73
|
-
await page.setCookie(...prepared);
|
|
74
|
-
}
|
|
75
|
-
async function applyCookies(page, pageUrl) {
|
|
76
|
-
const cookies = await loadAllCookies();
|
|
77
|
-
if (cookies.length) {
|
|
78
|
-
await setCookies(page, cookies, pageUrl);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
67
|
// ─── Page-level helpers (shared between one-shot and session tools) ───────────
|
|
82
68
|
async function extractInputs(page, opts) {
|
|
83
69
|
return page.evaluate(({ selector: sel, showHidden }) => {
|
|
@@ -211,7 +197,6 @@ server.tool('scrape_page', 'Navigate to a URL using a headless browser and retur
|
|
|
211
197
|
const browser = await getBrowser();
|
|
212
198
|
const page = await browser.newPage();
|
|
213
199
|
try {
|
|
214
|
-
await applyCookies(page, url);
|
|
215
200
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
216
201
|
if (wait_for) {
|
|
217
202
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -225,6 +210,7 @@ server.tool('scrape_page', 'Navigate to a URL using a headless browser and retur
|
|
|
225
210
|
}
|
|
226
211
|
finally {
|
|
227
212
|
await page.close();
|
|
213
|
+
scheduleJarSync();
|
|
228
214
|
}
|
|
229
215
|
});
|
|
230
216
|
// ─── reconnect_gui ──────────────────────────────────────────────────────────────
|
|
@@ -250,7 +236,6 @@ server.tool('execute_js', 'Navigate to a URL and execute custom JavaScript in th
|
|
|
250
236
|
const browser = await getBrowser();
|
|
251
237
|
const page = await browser.newPage();
|
|
252
238
|
try {
|
|
253
|
-
await applyCookies(page, url);
|
|
254
239
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
255
240
|
if (wait_for) {
|
|
256
241
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -269,6 +254,7 @@ server.tool('execute_js', 'Navigate to a URL and execute custom JavaScript in th
|
|
|
269
254
|
}
|
|
270
255
|
finally {
|
|
271
256
|
await page.close();
|
|
257
|
+
scheduleJarSync();
|
|
272
258
|
}
|
|
273
259
|
});
|
|
274
260
|
// ─── get_page_inputs ──────────────────────────────────────────────────────────
|
|
@@ -286,7 +272,7 @@ server.tool('get_page_inputs', 'Return a structured JSON object of all form inpu
|
|
|
286
272
|
}
|
|
287
273
|
if (session_id) {
|
|
288
274
|
try {
|
|
289
|
-
const page = getSession(session_id);
|
|
275
|
+
const page = getSession(session_id, 'get_page_inputs');
|
|
290
276
|
const result = await extractInputs(page, { selector, showHidden: show_hidden ?? false });
|
|
291
277
|
return { content: [{ type: 'text', text: JSON.stringify(result, null, 2) }] };
|
|
292
278
|
}
|
|
@@ -298,7 +284,6 @@ server.tool('get_page_inputs', 'Return a structured JSON object of all form inpu
|
|
|
298
284
|
const browser = await getBrowser();
|
|
299
285
|
const page = await browser.newPage();
|
|
300
286
|
try {
|
|
301
|
-
await applyCookies(page, url);
|
|
302
287
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
303
288
|
if (wait_for) {
|
|
304
289
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -312,6 +297,7 @@ server.tool('get_page_inputs', 'Return a structured JSON object of all form inpu
|
|
|
312
297
|
}
|
|
313
298
|
finally {
|
|
314
299
|
await page.close();
|
|
300
|
+
scheduleJarSync();
|
|
315
301
|
}
|
|
316
302
|
});
|
|
317
303
|
// ─── get_show_page ────────────────────────────────────────────────────────────
|
|
@@ -333,7 +319,6 @@ server.tool('get_show_page', 'Navigate to a URL and automatically parse a show/d
|
|
|
333
319
|
const browser = await getBrowser();
|
|
334
320
|
const page = await browser.newPage();
|
|
335
321
|
try {
|
|
336
|
-
await applyCookies(page, url);
|
|
337
322
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
338
323
|
if (wait_for) {
|
|
339
324
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -351,6 +336,7 @@ server.tool('get_show_page', 'Navigate to a URL and automatically parse a show/d
|
|
|
351
336
|
}
|
|
352
337
|
finally {
|
|
353
338
|
await page.close();
|
|
339
|
+
scheduleJarSync();
|
|
354
340
|
}
|
|
355
341
|
});
|
|
356
342
|
// ─── open_session ─────────────────────────────────────────────────────────────
|
|
@@ -400,7 +386,7 @@ server.tool('session_goto', 'Navigate the browser session to a new URL without c
|
|
|
400
386
|
}, async ({ session_id, url, wait_for, timeout }) => {
|
|
401
387
|
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
|
402
388
|
try {
|
|
403
|
-
const page = getSession(session_id);
|
|
389
|
+
const page = getSession(session_id, 'session_goto');
|
|
404
390
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
405
391
|
if (wait_for) {
|
|
406
392
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -420,7 +406,7 @@ server.tool('session_click', 'Click on an element in the browser session identif
|
|
|
420
406
|
}, async ({ session_id, selector, timeout }) => {
|
|
421
407
|
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
|
422
408
|
try {
|
|
423
|
-
const page = getSession(session_id);
|
|
409
|
+
const page = getSession(session_id, 'session_click');
|
|
424
410
|
await page.waitForSelector(selector, { timeout: effectiveTimeout });
|
|
425
411
|
await page.click(selector);
|
|
426
412
|
return { content: [{ type: 'text', text: `Clicked "${selector}"` }] };
|
|
@@ -440,7 +426,7 @@ server.tool('session_type', 'Type text into an input element in the browser sess
|
|
|
440
426
|
}, async ({ session_id, selector, text, clear, timeout }) => {
|
|
441
427
|
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
|
442
428
|
try {
|
|
443
|
-
const page = getSession(session_id);
|
|
429
|
+
const page = getSession(session_id, 'session_type');
|
|
444
430
|
await page.waitForSelector(selector, { timeout: effectiveTimeout });
|
|
445
431
|
if (clear) {
|
|
446
432
|
await page.click(selector, { clickCount: 3 });
|
|
@@ -461,7 +447,7 @@ server.tool('session_wait_for', 'Wait for a CSS selector to appear in the active
|
|
|
461
447
|
}, async ({ session_id, selector, timeout }) => {
|
|
462
448
|
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
|
463
449
|
try {
|
|
464
|
-
const page = getSession(session_id);
|
|
450
|
+
const page = getSession(session_id, 'session_wait_for');
|
|
465
451
|
await page.waitForSelector(selector, { timeout: effectiveTimeout });
|
|
466
452
|
return { content: [{ type: 'text', text: `Selector "${selector}" found.` }] };
|
|
467
453
|
}
|
|
@@ -479,7 +465,7 @@ server.tool('session_evaluate', 'Execute custom JavaScript in the context of an
|
|
|
479
465
|
}, async ({ session_id, script, wait_for, timeout }) => {
|
|
480
466
|
const effectiveTimeout = timeout ?? getDefaultTimeout();
|
|
481
467
|
try {
|
|
482
|
-
const page = getSession(session_id);
|
|
468
|
+
const page = getSession(session_id, 'session_evaluate');
|
|
483
469
|
if (wait_for) {
|
|
484
470
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
485
471
|
}
|
|
@@ -501,7 +487,7 @@ server.tool('session_html', 'Return the current full rendered HTML of an active
|
|
|
501
487
|
session_id: z.string().describe('Session ID returned by open_session'),
|
|
502
488
|
}, async ({ session_id }) => {
|
|
503
489
|
try {
|
|
504
|
-
const page = getSession(session_id);
|
|
490
|
+
const page = getSession(session_id, 'session_html');
|
|
505
491
|
const html = await page.content();
|
|
506
492
|
return { content: [{ type: 'text', text: html }] };
|
|
507
493
|
}
|
|
@@ -517,7 +503,7 @@ server.tool('session_query_selector', 'Query the active session page with a CSS
|
|
|
517
503
|
limit: z.number().optional().describe('Max number of elements to return (default: 10)'),
|
|
518
504
|
}, async ({ session_id, selector, limit }) => {
|
|
519
505
|
try {
|
|
520
|
-
const page = getSession(session_id);
|
|
506
|
+
const page = getSession(session_id, 'session_query_selector');
|
|
521
507
|
const results = await page.$$eval(selector, (elements, maxItems) => elements.slice(0, maxItems).map(el => {
|
|
522
508
|
const text = (el.textContent ?? '').trim().slice(0, 100);
|
|
523
509
|
return {
|
|
@@ -566,7 +552,7 @@ server.tool('session_scroll', 'Scroll the active session page. Use selector to s
|
|
|
566
552
|
return { content: [{ type: 'text', text: 'Error: provide at least one of selector, to_bottom, x, or y.' }], isError: true };
|
|
567
553
|
}
|
|
568
554
|
try {
|
|
569
|
-
const page = getSession(session_id);
|
|
555
|
+
const page = getSession(session_id, 'session_scroll');
|
|
570
556
|
if (selector) {
|
|
571
557
|
await page.$eval(selector, el => el.scrollIntoView());
|
|
572
558
|
return { content: [{ type: 'text', text: `Scrolled "${selector}" into view.` }] };
|
|
@@ -606,7 +592,7 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
606
592
|
destination_path: z
|
|
607
593
|
.string()
|
|
608
594
|
.optional()
|
|
609
|
-
.describe(`${RELAY_SHAPE_DESCRIPTIONS.destination_path}. Required when output is "file" or "both"`),
|
|
595
|
+
.describe(`${RELAY_SHAPE_DESCRIPTIONS.destination_path}. Required when output is "file" or "both", except behind the mcp-http-gateway relay, which keeps it on the agent side`),
|
|
610
596
|
filename: z
|
|
611
597
|
.string()
|
|
612
598
|
.optional()
|
|
@@ -623,10 +609,17 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
623
609
|
// Écrire un fichier suppose de savoir où, et sur quelle machine. Il n'y a pas de
|
|
624
610
|
// destination par défaut : elle atterrirait sur le disque de ce serveur, qui n'est
|
|
625
611
|
// pas forcément celui de l'agent — un fichier introuvable plutôt qu'une erreur.
|
|
612
|
+
// Derrière le relais, la destination reste côté agent : seul le nom du fichier arrive ici.
|
|
626
613
|
const wantsFile = output === 'file' || output === 'both';
|
|
627
|
-
if (wantsFile &&
|
|
614
|
+
if (wantsFile && !filename) {
|
|
615
|
+
return {
|
|
616
|
+
content: [{ type: 'text', text: 'Error: filename is required when output is "file" or "both".' }],
|
|
617
|
+
isError: true,
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
if (wantsFile && !destination_relay && !destination_path) {
|
|
628
621
|
return {
|
|
629
|
-
content: [{ type: 'text', text: 'Error: destination_path
|
|
622
|
+
content: [{ type: 'text', text: 'Error: destination_path is required when output is "file" or "both".' }],
|
|
630
623
|
isError: true,
|
|
631
624
|
};
|
|
632
625
|
}
|
|
@@ -636,7 +629,7 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
636
629
|
let previousViewport;
|
|
637
630
|
try {
|
|
638
631
|
if (session_id) {
|
|
639
|
-
page = getSession(session_id);
|
|
632
|
+
page = getSession(session_id, 'screenshot');
|
|
640
633
|
if (viewport) {
|
|
641
634
|
previousViewport = page.viewport();
|
|
642
635
|
await page.setViewport(resolveViewport(toViewportOverride(viewport)));
|
|
@@ -652,7 +645,6 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
652
645
|
if (viewport) {
|
|
653
646
|
await page.setViewport(resolveViewport(toViewportOverride(viewport)));
|
|
654
647
|
}
|
|
655
|
-
await applyCookies(page, url);
|
|
656
648
|
await page.goto(url, { waitUntil: 'networkidle2', timeout: effectiveTimeout });
|
|
657
649
|
if (wait_for) {
|
|
658
650
|
await page.waitForSelector(wait_for, { timeout: effectiveTimeout });
|
|
@@ -681,7 +673,7 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
681
673
|
}
|
|
682
674
|
if (wantsFile) {
|
|
683
675
|
resultContent.push(...deliverFile(imageBuffer, mimeType, {
|
|
684
|
-
destination_path
|
|
676
|
+
destination_path,
|
|
685
677
|
filename: filename,
|
|
686
678
|
destination_overwrite,
|
|
687
679
|
destination_relay,
|
|
@@ -696,6 +688,7 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
696
688
|
finally {
|
|
697
689
|
if (ownedPage && page) {
|
|
698
690
|
await page.close();
|
|
691
|
+
scheduleJarSync();
|
|
699
692
|
}
|
|
700
693
|
else if (page && previousViewport) {
|
|
701
694
|
await page.setViewport(previousViewport).catch(() => undefined);
|
|
@@ -708,15 +701,20 @@ function launchGUI() {
|
|
|
708
701
|
return;
|
|
709
702
|
}
|
|
710
703
|
launcher.start();
|
|
711
|
-
|
|
704
|
+
registerRpc(launcher);
|
|
712
705
|
}
|
|
713
706
|
async function main() {
|
|
707
|
+
// Un dossier de données mal configuré arrête le serveur avant qu'aucun outil ne serve.
|
|
708
|
+
const dataDir = getDataDir();
|
|
709
|
+
console.error(`[MCP] Dossier des sites : ${dataDir ?? 'non configuré (sites désactivés)'}`);
|
|
714
710
|
const transport = new StdioServerTransport();
|
|
715
711
|
await server.connect(transport);
|
|
716
712
|
console.error('[MCP] Simple Scraper MCP server started');
|
|
717
713
|
// Après le connect, comme mcp-documentor : au redémarrage du serveur par le
|
|
718
714
|
// client, le master sortant a alors eu le temps de libérer la route.
|
|
719
715
|
launchGUI();
|
|
716
|
+
startKeepAlive();
|
|
717
|
+
startPeriodicJarSync();
|
|
720
718
|
}
|
|
721
719
|
main().catch(async (err) => {
|
|
722
720
|
console.error('[MCP] Fatal error:', err);
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { checkSite } from './site-check.js';
|
|
2
|
+
import { getStore } from './site-store.js';
|
|
3
|
+
const TICK_MS = 60_000;
|
|
4
|
+
/** Le maintien visite ce site : activé, avec une page de vérification, et pas déjà déconnecté. */
|
|
5
|
+
function isMaintained(site) {
|
|
6
|
+
return site.keepAlive.enabled && !!site.keepAlive.pageUrl && site.state.status !== 'disconnected';
|
|
7
|
+
}
|
|
8
|
+
/** Prochaine visite du maintien pour ce site, ou null s'il n'en fait pas. */
|
|
9
|
+
export function nextVisitAt(site, now = Date.now()) {
|
|
10
|
+
if (!isMaintained(site))
|
|
11
|
+
return null;
|
|
12
|
+
const { lastCheckAt } = site.state;
|
|
13
|
+
return lastCheckAt ? Math.max(now, lastCheckAt + site.keepAlive.intervalMin * 60_000) : now;
|
|
14
|
+
}
|
|
15
|
+
/** Prochaine visite du maintien, tous sites confondus. */
|
|
16
|
+
export function nextKeepAliveAt(now = Date.now()) {
|
|
17
|
+
const store = getStore();
|
|
18
|
+
if (!store)
|
|
19
|
+
return null;
|
|
20
|
+
const times = store.list().map(s => nextVisitAt(s, now)).filter((t) => t !== null);
|
|
21
|
+
return times.length ? Math.min(...times) : null;
|
|
22
|
+
}
|
|
23
|
+
let ticking = false;
|
|
24
|
+
async function tick() {
|
|
25
|
+
if (ticking)
|
|
26
|
+
return;
|
|
27
|
+
ticking = true;
|
|
28
|
+
try {
|
|
29
|
+
const store = getStore();
|
|
30
|
+
if (!store)
|
|
31
|
+
return;
|
|
32
|
+
const now = Date.now();
|
|
33
|
+
for (const site of store.list()) {
|
|
34
|
+
const next = nextVisitAt(site, now);
|
|
35
|
+
if (next === null || next > now)
|
|
36
|
+
continue;
|
|
37
|
+
await checkSite(site.id);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
console.error(`[KeepAlive] ${err instanceof Error ? err.message : String(err)}`);
|
|
42
|
+
}
|
|
43
|
+
finally {
|
|
44
|
+
ticking = false;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
/** Démarre le maintien : chaque minute, visite les sites dont le passage est dû. */
|
|
48
|
+
export function startKeepAlive() {
|
|
49
|
+
setInterval(() => void tick(), TICK_MS).unref();
|
|
50
|
+
setTimeout(() => void tick(), 10_000).unref();
|
|
51
|
+
}
|
package/dist/login.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
import fs from 'fs';
|
|
2
|
+
import os from 'os';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import puppeteer from 'puppeteer';
|
|
5
|
+
import { BROWSER_LAUNCH_ARGS, getDefaultViewport } from './browser.js';
|
|
6
|
+
import { replaceSiteCookies } from './cookie-sync.js';
|
|
7
|
+
import { applyRemoteAction } from './remote-input.js';
|
|
8
|
+
import { pageShowsSignal } from './site-check.js';
|
|
9
|
+
import { cookieCountLabel, requireStore } from './site-store.js';
|
|
10
|
+
const POLL_MS = 1_000;
|
|
11
|
+
let active = null;
|
|
12
|
+
let last = null;
|
|
13
|
+
/** Onglet le plus récent du navigateur de connexion. */
|
|
14
|
+
async function currentPage(browser) {
|
|
15
|
+
const pages = (await browser.pages()).filter(p => !p.isClosed());
|
|
16
|
+
return pages.length ? pages[pages.length - 1] : null;
|
|
17
|
+
}
|
|
18
|
+
async function requirePage() {
|
|
19
|
+
if (!active)
|
|
20
|
+
throw new Error('Aucune connexion en cours.');
|
|
21
|
+
const page = await currentPage(active.browser);
|
|
22
|
+
if (!page)
|
|
23
|
+
throw new Error('Le navigateur de connexion n\'a aucun onglet.');
|
|
24
|
+
return page;
|
|
25
|
+
}
|
|
26
|
+
/** Ferme le navigateur de connexion et supprime son profil temporaire. */
|
|
27
|
+
async function teardown(login) {
|
|
28
|
+
clearInterval(login.timer);
|
|
29
|
+
await login.browser.close().catch(() => undefined);
|
|
30
|
+
fs.rmSync(login.profileDir, { recursive: true, force: true, maxRetries: 10, retryDelay: 200 });
|
|
31
|
+
}
|
|
32
|
+
function settle(login, patch) {
|
|
33
|
+
login.status = { ...login.status, ...patch };
|
|
34
|
+
last = login.status;
|
|
35
|
+
if (active === login)
|
|
36
|
+
active = null;
|
|
37
|
+
}
|
|
38
|
+
/** Récupère les cookies du navigateur de connexion et les enregistre pour le site. */
|
|
39
|
+
async function finish(login) {
|
|
40
|
+
if (login.status.phase !== 'waiting')
|
|
41
|
+
return;
|
|
42
|
+
login.status.phase = 'saving';
|
|
43
|
+
clearInterval(login.timer);
|
|
44
|
+
try {
|
|
45
|
+
const cookies = await login.browser.cookies();
|
|
46
|
+
if (!cookies.length)
|
|
47
|
+
throw new Error('Le navigateur de connexion ne contient aucun cookie.');
|
|
48
|
+
const count = await replaceSiteCookies(login.status.siteId, cookies);
|
|
49
|
+
const store = requireStore();
|
|
50
|
+
const now = Date.now();
|
|
51
|
+
store.setState(login.status.siteId, {
|
|
52
|
+
status: 'connected',
|
|
53
|
+
sessionOpenedAt: now,
|
|
54
|
+
lostAt: null,
|
|
55
|
+
lastCheckAt: now,
|
|
56
|
+
lastCheckError: null,
|
|
57
|
+
});
|
|
58
|
+
store.addEvent(login.status.siteId, 'login', cookieCountLabel(count, 'enregistré'));
|
|
59
|
+
settle(login, { phase: 'done', cookieCount: count });
|
|
60
|
+
}
|
|
61
|
+
catch (err) {
|
|
62
|
+
settle(login, { phase: 'error', error: err instanceof Error ? err.message : String(err) });
|
|
63
|
+
}
|
|
64
|
+
await teardown(login);
|
|
65
|
+
}
|
|
66
|
+
/** Surveille la page : connexion terminée quand le signe du site apparaît. */
|
|
67
|
+
async function poll(login) {
|
|
68
|
+
if (login.status.phase !== 'waiting')
|
|
69
|
+
return;
|
|
70
|
+
if (!login.browser.connected) {
|
|
71
|
+
settle(login, { phase: 'cancelled' });
|
|
72
|
+
await teardown(login);
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
const page = await currentPage(login.browser).catch(() => null);
|
|
76
|
+
if (!page)
|
|
77
|
+
return;
|
|
78
|
+
login.status.url = page.url();
|
|
79
|
+
const site = requireStore().get(login.status.siteId);
|
|
80
|
+
const signalled = await pageShowsSignal(page, site.signal).catch(() => false);
|
|
81
|
+
if (signalled)
|
|
82
|
+
await finish(login);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Ouvre un navigateur sans fenêtre, sur un profil vierge, à la page de connexion
|
|
86
|
+
* du site. L'utilisateur s'y connecte depuis l'interface, qui affiche la page et
|
|
87
|
+
* lui relaie ses clics et sa frappe ; aucun écran n'est nécessaire. Les cookies
|
|
88
|
+
* sont enregistrés pour le site dès que le signe de connexion apparaît.
|
|
89
|
+
*/
|
|
90
|
+
export async function startLogin(siteId) {
|
|
91
|
+
if (active)
|
|
92
|
+
throw new Error('Une connexion est déjà en cours.');
|
|
93
|
+
const site = requireStore().get(siteId);
|
|
94
|
+
if (!site.loginUrl)
|
|
95
|
+
throw new Error(`Le site "${site.name}" n'a pas de page de connexion.`);
|
|
96
|
+
const profileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'simple-scraper-login-'));
|
|
97
|
+
const browser = await puppeteer.launch({
|
|
98
|
+
headless: true,
|
|
99
|
+
defaultViewport: getDefaultViewport(),
|
|
100
|
+
userDataDir: profileDir,
|
|
101
|
+
args: BROWSER_LAUNCH_ARGS,
|
|
102
|
+
});
|
|
103
|
+
const page = (await browser.pages())[0] ?? (await browser.newPage());
|
|
104
|
+
const login = {
|
|
105
|
+
status: { siteId, phase: 'waiting', url: site.loginUrl, error: null, cookieCount: null, startedAt: Date.now() },
|
|
106
|
+
browser,
|
|
107
|
+
profileDir,
|
|
108
|
+
timer: setInterval(() => void poll(login), POLL_MS),
|
|
109
|
+
};
|
|
110
|
+
active = login;
|
|
111
|
+
last = login.status;
|
|
112
|
+
browser.on('disconnected', () => {
|
|
113
|
+
if (login.status.phase === 'waiting') {
|
|
114
|
+
settle(login, { phase: 'cancelled' });
|
|
115
|
+
void teardown(login);
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
page.goto(site.loginUrl).catch(err => {
|
|
119
|
+
if (login.status.phase !== 'waiting')
|
|
120
|
+
return;
|
|
121
|
+
settle(login, { phase: 'error', error: err instanceof Error ? err.message : String(err) });
|
|
122
|
+
void teardown(login);
|
|
123
|
+
});
|
|
124
|
+
return login.status;
|
|
125
|
+
}
|
|
126
|
+
/** Connexion en cours, ou la dernière terminée. */
|
|
127
|
+
export function getLoginStatus() {
|
|
128
|
+
return active?.status ?? last;
|
|
129
|
+
}
|
|
130
|
+
/** Enregistre la connexion sans attendre le signe du site. */
|
|
131
|
+
export async function confirmLogin() {
|
|
132
|
+
if (!active)
|
|
133
|
+
throw new Error('Aucune connexion en cours.');
|
|
134
|
+
const login = active;
|
|
135
|
+
await finish(login);
|
|
136
|
+
return login.status;
|
|
137
|
+
}
|
|
138
|
+
export async function cancelLogin() {
|
|
139
|
+
if (!active)
|
|
140
|
+
return;
|
|
141
|
+
const login = active;
|
|
142
|
+
settle(login, { phase: 'cancelled' });
|
|
143
|
+
await teardown(login);
|
|
144
|
+
}
|
|
145
|
+
/** Rejoue dans la page de connexion un clic, une frappe ou une navigation de l'utilisateur. */
|
|
146
|
+
export async function loginInput(action) {
|
|
147
|
+
const page = await requirePage();
|
|
148
|
+
await applyRemoteAction(page, action);
|
|
149
|
+
if (active)
|
|
150
|
+
active.status.url = page.url();
|
|
151
|
+
}
|
|
152
|
+
/** Capture JPEG de l'onglet courant du navigateur de connexion. */
|
|
153
|
+
export async function loginScreenshot() {
|
|
154
|
+
const page = await requirePage();
|
|
155
|
+
const buffer = await page.screenshot({ type: 'jpeg', quality: 70 });
|
|
156
|
+
return Buffer.from(buffer).toString('base64');
|
|
157
|
+
}
|
package/dist/relay.js
CHANGED
|
@@ -50,18 +50,22 @@ export function deliverFile(data, mimeType, target) {
|
|
|
50
50
|
];
|
|
51
51
|
}
|
|
52
52
|
// Pas de relais : le MCP tourne à côté de l'agent, son disque est le bon.
|
|
53
|
-
if (!
|
|
54
|
-
throw new Error(
|
|
53
|
+
if (!target.destination_path) {
|
|
54
|
+
throw new Error('destination_path is required when no relay delivers the file.');
|
|
55
55
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
throw new Error(`destination_path must be
|
|
56
|
+
const destination = target.destination_path;
|
|
57
|
+
if (!path.isAbsolute(destination)) {
|
|
58
|
+
throw new Error(`destination_path must be an absolute directory path: ${destination}`);
|
|
59
59
|
}
|
|
60
|
-
|
|
60
|
+
if (fs.existsSync(destination) &&
|
|
61
|
+
!fs.statSync(destination).isDirectory()) {
|
|
62
|
+
throw new Error(`destination_path must be a directory, but a file already exists there: ${destination}`);
|
|
63
|
+
}
|
|
64
|
+
const written = path.join(destination, name);
|
|
61
65
|
if (!target.destination_overwrite && fs.existsSync(written)) {
|
|
62
66
|
throw new Error(`Destination already exists: ${written}. Pass destination_overwrite to replace it.`);
|
|
63
67
|
}
|
|
64
|
-
fs.mkdirSync(
|
|
68
|
+
fs.mkdirSync(destination, { recursive: true });
|
|
65
69
|
fs.writeFileSync(written, data);
|
|
66
70
|
return [{ type: "text", text: `File written to ${written} (${data.length} bytes).` }];
|
|
67
71
|
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { getDefaultTimeout } from './browser.js';
|
|
3
|
+
/**
|
|
4
|
+
* Action de l'utilisateur sur une page affichée dans l'interface. Les
|
|
5
|
+
* coordonnées d'un clic sont des fractions (0 à 1) de la capture, donc
|
|
6
|
+
* indépendantes de la taille à laquelle l'interface l'affiche.
|
|
7
|
+
*/
|
|
8
|
+
export const remoteActionSchema = z.discriminatedUnion('type', [
|
|
9
|
+
z.object({ type: z.literal('click'), x: z.number().min(0).max(1), y: z.number().min(0).max(1) }),
|
|
10
|
+
z.object({ type: z.literal('scroll'), dy: z.number().min(-5000).max(5000) }),
|
|
11
|
+
z.object({ type: z.literal('text'), text: z.string().min(1).max(2000) }),
|
|
12
|
+
z.object({
|
|
13
|
+
type: z.literal('key'),
|
|
14
|
+
key: z.string().min(1).max(32),
|
|
15
|
+
modifiers: z.array(z.enum(['Control', 'Shift', 'Alt', 'Meta'])).optional(),
|
|
16
|
+
}),
|
|
17
|
+
z.object({ type: z.literal('navigate'), url: z.url() }),
|
|
18
|
+
z.object({ type: z.literal('back') }),
|
|
19
|
+
]);
|
|
20
|
+
export function parseRemoteAction(data) {
|
|
21
|
+
const result = remoteActionSchema.safeParse(data);
|
|
22
|
+
if (!result.success)
|
|
23
|
+
throw new Error(`Action invalide : ${result.error.issues.map(i => i.message).join(' ')}`);
|
|
24
|
+
return result.data;
|
|
25
|
+
}
|
|
26
|
+
/** Point de la page, en pixels CSS, désigné par des fractions de sa capture. */
|
|
27
|
+
export function pointOf(page, x, y) {
|
|
28
|
+
const viewport = page.viewport();
|
|
29
|
+
if (!viewport)
|
|
30
|
+
throw new Error('La page n\'a pas de taille d\'affichage fixe.');
|
|
31
|
+
return { x: x * viewport.width, y: y * viewport.height };
|
|
32
|
+
}
|
|
33
|
+
/** Rejoue l'action dans la page. */
|
|
34
|
+
export async function applyRemoteAction(page, action) {
|
|
35
|
+
switch (action.type) {
|
|
36
|
+
case 'click': {
|
|
37
|
+
const point = pointOf(page, action.x, action.y);
|
|
38
|
+
await page.mouse.click(point.x, point.y);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
case 'scroll':
|
|
42
|
+
await page.mouse.wheel({ deltaY: action.dy });
|
|
43
|
+
return;
|
|
44
|
+
case 'text':
|
|
45
|
+
await page.keyboard.type(action.text);
|
|
46
|
+
return;
|
|
47
|
+
case 'key': {
|
|
48
|
+
const modifiers = action.modifiers ?? [];
|
|
49
|
+
for (const m of modifiers)
|
|
50
|
+
await page.keyboard.down(m);
|
|
51
|
+
try {
|
|
52
|
+
await page.keyboard.press(action.key);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
for (const m of modifiers.slice().reverse())
|
|
56
|
+
await page.keyboard.up(m);
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
case 'navigate':
|
|
61
|
+
await page.goto(action.url, { waitUntil: 'domcontentloaded', timeout: getDefaultTimeout() });
|
|
62
|
+
return;
|
|
63
|
+
case 'back':
|
|
64
|
+
await page.goBack({ waitUntil: 'domcontentloaded', timeout: getDefaultTimeout() });
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
}
|