@imenam/simple-scraper 1.0.27 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -5
- package/dist/browser.js +8 -1
- package/dist/capture.js +37 -0
- package/dist/gui.js +8 -0
- package/dist/index.js +4 -3
- package/dist/login.js +35 -18
- package/dist/remote-input.js +67 -0
- package/dist/rpc.js +31 -6
- package/dist/signal-picker.js +144 -0
- package/package.json +3 -2
- package/public/dist/assets/index-AyzOgMwI.js +11 -0
- package/public/dist/assets/{index-olIqry-_.css → index-BAs4dkz0.css} +1 -1
- package/public/dist/index.html +2 -2
- package/dist/session-rpc.js +0 -26
- package/public/dist/assets/index-CmU4YRqF.js +0 -11
package/README.md
CHANGED
|
@@ -355,6 +355,8 @@ Capture a screenshot of a page. Use `session_id` to capture an active session in
|
|
|
355
355
|
|
|
356
356
|
The capture area is resolved by precedence: `selector` → `clip` → `full_page` → the visible viewport.
|
|
357
357
|
|
|
358
|
+
Captures run one at a time across the whole server (this tool and the GUI views alike), each after bringing its page to the front: with several pages open, headless Chrome stalls the capture of a page that is not in front. A capture that gets no answer within 20 s fails without holding up the next ones.
|
|
359
|
+
|
|
358
360
|
| Parameter | Type | Required | Description |
|
|
359
361
|
|-----------|------|----------|-------------|
|
|
360
362
|
| `session_id` | string | | Session ID. If provided, `url` is ignored and the current page state is captured. |
|
|
@@ -396,11 +398,17 @@ A **site** is an account the scraper uses while logged in: a name, the domains i
|
|
|
396
398
|
The sign tells the scraper whether a page is seen logged in:
|
|
397
399
|
|
|
398
400
|
- **L'adresse contient** (the address contains): the page address (host and path, without query or anchor) contains the value, e.g. `crm.example.com/app`;
|
|
399
|
-
- **Un élément est présent** (an element is present): a
|
|
401
|
+
- **Un élément est présent** (an element is present): a selector that only matches once logged in, e.g. `.user-menu`. Puppeteer's extended selectors are accepted, so an element can be matched by its text: `.user-menu span::-p-text(Galadrim)`.
|
|
402
|
+
|
|
403
|
+
**Choisir sur la page** opens the visited page as the scraper sees it logged in; clicking an element (your name, for instance) proposes selectors for it, from the most precise (its text inside its container) to the most general. **Tester** checks the sign as it is typed. Both check each selector twice: it must be found on the visited page with the site's cookies, and absent from the same page without cookies; a selector passing both is a reliable login sign.
|
|
400
404
|
|
|
401
405
|
### Assisted login
|
|
402
406
|
|
|
403
|
-
**Se reconnecter** opens a
|
|
407
|
+
**Se reconnecter** opens the site's login page in a headless Chromium on a blank temporary profile, on the machine running the MCP, and shows it in the GUI. You log in there yourself (credentials, two-factor authentication, SSO): clicks, typing, pasting and scrolling on the capture are replayed in the page, and the address bar allows going back or opening another address. No display is needed, so this works the same on a remote server. As soon as the login sign appears, the browser's cookies are saved for the site and it closes; **Je suis connecté** saves them without waiting for the sign. **Tout reconnecter** chains every site to reconnect, one after the other.
|
|
408
|
+
|
|
409
|
+
What you type travels from the GUI to the MCP (through the proxy when the GUI is reached remotely, which should then be served over HTTPS); it is never stored. Some sites detect headless browsers and may show a captcha or refuse the login.
|
|
410
|
+
|
|
411
|
+
Every browser of the scraper starts with Chromium's sandbox disabled (`--no-sandbox`): under a service account, on a Linux that restricts unprivileged user namespaces (Ubuntu 23.10+ with AppArmor), the sandbox cannot start.
|
|
404
412
|
|
|
405
413
|
### Keep-alive
|
|
406
414
|
|
|
@@ -422,9 +430,9 @@ Netscape cookie files (`.txt`) placed in this folder are not loaded. To start fr
|
|
|
422
430
|
|
|
423
431
|
When `PROXY_URL` is set, a web interface is registered with the MCP proxy:
|
|
424
432
|
|
|
425
|
-
- **Sites** — every site with its state (logged in, to reconnect, not checked), its session and its keep-alive; banner and **Tout reconnecter** when sessions are lost; assisted login
|
|
426
|
-
- **Site page** — domains, login page, login sign, keep-alive (interval, visited page), recent activity, cookie import and **Effacer la session**.
|
|
427
|
-
- **Sessions** — the sessions opened by the agent, with a live view (
|
|
433
|
+
- **Sites** — every site with its state (logged in, to reconnect, not checked), its session and its keep-alive; banner and **Tout reconnecter** when sessions are lost; assisted login in a remotely controlled view of the login page, which can be shown full screen.
|
|
434
|
+
- **Site page** — domains, login page, login sign (picked on the page and tested), keep-alive (interval, visited page), recent activity, cookie import and **Effacer la session**.
|
|
435
|
+
- **Sessions** — the sessions opened by the agent, with a live view (refresh rate selectable from 0.5 s to 10 s, 2.5 s by default; can be paused or shown full screen), the last action, the matching site and the console messages.
|
|
428
436
|
- **Activity** — the history of every site, filterable by site.
|
|
429
437
|
|
|
430
438
|
The interface refreshes by polling: no connection stays open.
|
package/dist/browser.js
CHANGED
|
@@ -29,6 +29,13 @@ export function getDefaultViewport() {
|
|
|
29
29
|
deviceScaleFactor: envNumber('PUPPETEER_DEVICE_SCALE_FACTOR', 1),
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
|
+
/**
|
|
33
|
+
* Arguments de lancement de tous les navigateurs du scraper. La sandbox de
|
|
34
|
+
* Chromium est désactivée : sous un compte de service, sur un Linux qui
|
|
35
|
+
* restreint les espaces de noms utilisateur (Ubuntu 23.10+ avec AppArmor),
|
|
36
|
+
* elle ne peut pas démarrer.
|
|
37
|
+
*/
|
|
38
|
+
export const BROWSER_LAUNCH_ARGS = ['--no-sandbox', '--disable-setuid-sandbox'];
|
|
32
39
|
/** Lance le navigateur partagé et y charge les cookies de tous les sites. */
|
|
33
40
|
async function launchBrowser() {
|
|
34
41
|
const cookies = getStore()?.allCookies() ?? [];
|
|
@@ -36,7 +43,7 @@ async function launchBrowser() {
|
|
|
36
43
|
const launched = await puppeteer.launch({
|
|
37
44
|
headless,
|
|
38
45
|
defaultViewport: getDefaultViewport(),
|
|
39
|
-
args:
|
|
46
|
+
args: BROWSER_LAUNCH_ARGS,
|
|
40
47
|
});
|
|
41
48
|
if (cookies.length)
|
|
42
49
|
await launched.setCookie(...cookies);
|
package/dist/capture.js
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
const CAPTURE_TIMEOUT_MS = 20_000;
|
|
2
|
+
let queue = Promise.resolve();
|
|
3
|
+
/**
|
|
4
|
+
* Exécute une capture d'écran sur la page, une seule à la fois pour tout le
|
|
5
|
+
* process, après avoir mis la page au premier plan. Chrome sans fenêtre bloque
|
|
6
|
+
* la capture d'une page qui n'est pas au premier plan dès que plusieurs pages
|
|
7
|
+
* sont ouvertes, et deux mises au premier plan simultanées se contrarient.
|
|
8
|
+
* Une capture qui dépasse le délai échoue sans bloquer les suivantes.
|
|
9
|
+
*/
|
|
10
|
+
export function runCapture(page, capture) {
|
|
11
|
+
const run = async () => {
|
|
12
|
+
let timer;
|
|
13
|
+
const timeout = new Promise((_, reject) => {
|
|
14
|
+
timer = setTimeout(() => reject(new Error(`Capture d'écran sans réponse après ${CAPTURE_TIMEOUT_MS / 1000} s.`)), CAPTURE_TIMEOUT_MS);
|
|
15
|
+
});
|
|
16
|
+
try {
|
|
17
|
+
return await Promise.race([
|
|
18
|
+
(async () => {
|
|
19
|
+
await page.bringToFront();
|
|
20
|
+
return capture();
|
|
21
|
+
})(),
|
|
22
|
+
timeout,
|
|
23
|
+
]);
|
|
24
|
+
}
|
|
25
|
+
finally {
|
|
26
|
+
clearTimeout(timer);
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
const next = queue.then(run, run);
|
|
30
|
+
queue = next.catch(() => undefined);
|
|
31
|
+
return next;
|
|
32
|
+
}
|
|
33
|
+
/** Capture JPEG de la partie visible de la page, encodée en base64. */
|
|
34
|
+
export async function captureJpegBase64(page, quality) {
|
|
35
|
+
const buffer = await runCapture(page, () => page.screenshot({ type: 'jpeg', quality }));
|
|
36
|
+
return Buffer.from(buffer).toString('base64');
|
|
37
|
+
}
|
package/dist/gui.js
CHANGED
|
@@ -148,7 +148,15 @@ app.post('/api/login', (req, res) => {
|
|
|
148
148
|
});
|
|
149
149
|
app.get('/api/login', (_req, res) => forward(res, 'LOGIN_STATUS'));
|
|
150
150
|
app.get('/api/login/screenshot', (_req, res) => sendJpeg(res, 'LOGIN_SCREENSHOT'));
|
|
151
|
+
app.post('/api/login/input', (req, res) => forward(res, 'LOGIN_INPUT', { action: req.body.action }, 45_000));
|
|
151
152
|
app.post('/api/login/confirm', (_req, res) => forward(res, 'LOGIN_CONFIRM', undefined, 15_000));
|
|
153
|
+
// ─── Choix du signe de connexion ──────────────────────────────────────────────
|
|
154
|
+
app.post('/api/picker', (req, res) => forward(res, 'PICKER_OPEN', { pageUrl: req.body.pageUrl }, 45_000));
|
|
155
|
+
app.get('/api/picker/screenshot', (_req, res) => sendJpeg(res, 'PICKER_SCREENSHOT'));
|
|
156
|
+
app.post('/api/picker/scroll', (req, res) => forward(res, 'PICKER_SCROLL', { dy: req.body.dy }));
|
|
157
|
+
app.post('/api/picker/pick', (req, res) => forward(res, 'PICKER_PICK', { x: req.body.x, y: req.body.y }, 60_000));
|
|
158
|
+
app.post('/api/picker/close', (_req, res) => forward(res, 'PICKER_CLOSE'));
|
|
159
|
+
app.post('/api/signal/test', (req, res) => forward(res, 'SIGNAL_TEST', { pageUrl: req.body.pageUrl, signal: req.body.signal }, 75_000));
|
|
152
160
|
app.post('/api/login/cancel', (_req, res) => forward(res, 'LOGIN_CANCEL', undefined, 15_000));
|
|
153
161
|
// ─── Sessions ─────────────────────────────────────────────────────────────────
|
|
154
162
|
app.get('/api/sessions', (_req, res) => forward(res, 'SESSION_LIST'));
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,7 @@ import { fileURLToPath } from 'url';
|
|
|
9
9
|
import { z } from 'zod';
|
|
10
10
|
import { setupLogging } from './logger.js';
|
|
11
11
|
import { getBrowser, closeBrowser, getDefaultTimeout } from './browser.js';
|
|
12
|
+
import { runCapture } from './capture.js';
|
|
12
13
|
import { scheduleJarSync, startPeriodicJarSync } from './cookie-sync.js';
|
|
13
14
|
import { getDataDir } from './data-dir.js';
|
|
14
15
|
import { startKeepAlive } from './keep-alive.js';
|
|
@@ -657,15 +658,15 @@ server.tool('screenshot', 'Take a screenshot of a page. Provide either session_i
|
|
|
657
658
|
if (!element) {
|
|
658
659
|
return { content: [{ type: 'text', text: `Error: selector "${selector}" not found.` }], isError: true };
|
|
659
660
|
}
|
|
660
|
-
imageBuffer = Buffer.from(await element.screenshot({ type: fmt }));
|
|
661
|
+
imageBuffer = Buffer.from(await runCapture(page, () => element.screenshot({ type: fmt })));
|
|
661
662
|
}
|
|
662
663
|
else if (clip) {
|
|
663
664
|
// captureBeyondViewport lets the region extend past the visible area
|
|
664
665
|
// without having to scroll the page first.
|
|
665
|
-
imageBuffer = Buffer.from(await page.screenshot({ type: fmt, clip, captureBeyondViewport: true }));
|
|
666
|
+
imageBuffer = Buffer.from(await runCapture(page, () => page.screenshot({ type: fmt, clip, captureBeyondViewport: true })));
|
|
666
667
|
}
|
|
667
668
|
else {
|
|
668
|
-
imageBuffer = Buffer.from(await page.screenshot({ type: fmt, fullPage: full_page ?? false }));
|
|
669
|
+
imageBuffer = Buffer.from(await runCapture(page, () => page.screenshot({ type: fmt, fullPage: full_page ?? false })));
|
|
669
670
|
}
|
|
670
671
|
const resultContent = [];
|
|
671
672
|
if (output === 'inline' || output === 'both') {
|
package/dist/login.js
CHANGED
|
@@ -2,18 +2,29 @@ import fs from 'fs';
|
|
|
2
2
|
import os from 'os';
|
|
3
3
|
import path from 'path';
|
|
4
4
|
import puppeteer from 'puppeteer';
|
|
5
|
+
import { BROWSER_LAUNCH_ARGS, getDefaultViewport } from './browser.js';
|
|
6
|
+
import { captureJpegBase64 } from './capture.js';
|
|
5
7
|
import { replaceSiteCookies } from './cookie-sync.js';
|
|
8
|
+
import { applyRemoteAction } from './remote-input.js';
|
|
6
9
|
import { pageShowsSignal } from './site-check.js';
|
|
7
10
|
import { cookieCountLabel, requireStore } from './site-store.js';
|
|
8
11
|
const POLL_MS = 1_000;
|
|
9
12
|
let active = null;
|
|
10
13
|
let last = null;
|
|
11
|
-
/** Onglet le plus récent
|
|
14
|
+
/** Onglet le plus récent du navigateur de connexion. */
|
|
12
15
|
async function currentPage(browser) {
|
|
13
16
|
const pages = (await browser.pages()).filter(p => !p.isClosed());
|
|
14
17
|
return pages.length ? pages[pages.length - 1] : null;
|
|
15
18
|
}
|
|
16
|
-
|
|
19
|
+
async function requirePage() {
|
|
20
|
+
if (!active)
|
|
21
|
+
throw new Error('Aucune connexion en cours.');
|
|
22
|
+
const page = await currentPage(active.browser);
|
|
23
|
+
if (!page)
|
|
24
|
+
throw new Error('Le navigateur de connexion n\'a aucun onglet.');
|
|
25
|
+
return page;
|
|
26
|
+
}
|
|
27
|
+
/** Ferme le navigateur de connexion et supprime son profil temporaire. */
|
|
17
28
|
async function teardown(login) {
|
|
18
29
|
clearInterval(login.timer);
|
|
19
30
|
await login.browser.close().catch(() => undefined);
|
|
@@ -25,7 +36,7 @@ function settle(login, patch) {
|
|
|
25
36
|
if (active === login)
|
|
26
37
|
active = null;
|
|
27
38
|
}
|
|
28
|
-
/** Récupère les cookies
|
|
39
|
+
/** Récupère les cookies du navigateur de connexion et les enregistre pour le site. */
|
|
29
40
|
async function finish(login) {
|
|
30
41
|
if (login.status.phase !== 'waiting')
|
|
31
42
|
return;
|
|
@@ -34,7 +45,7 @@ async function finish(login) {
|
|
|
34
45
|
try {
|
|
35
46
|
const cookies = await login.browser.cookies();
|
|
36
47
|
if (!cookies.length)
|
|
37
|
-
throw new Error('
|
|
48
|
+
throw new Error('Le navigateur de connexion ne contient aucun cookie.');
|
|
38
49
|
const count = await replaceSiteCookies(login.status.siteId, cookies);
|
|
39
50
|
const store = requireStore();
|
|
40
51
|
const now = Date.now();
|
|
@@ -53,7 +64,7 @@ async function finish(login) {
|
|
|
53
64
|
}
|
|
54
65
|
await teardown(login);
|
|
55
66
|
}
|
|
56
|
-
/** Surveille la
|
|
67
|
+
/** Surveille la page : connexion terminée quand le signe du site apparaît. */
|
|
57
68
|
async function poll(login) {
|
|
58
69
|
if (login.status.phase !== 'waiting')
|
|
59
70
|
return;
|
|
@@ -72,9 +83,10 @@ async function poll(login) {
|
|
|
72
83
|
await finish(login);
|
|
73
84
|
}
|
|
74
85
|
/**
|
|
75
|
-
* Ouvre
|
|
76
|
-
*
|
|
77
|
-
*
|
|
86
|
+
* Ouvre un navigateur sans fenêtre, sur un profil vierge, à la page de connexion
|
|
87
|
+
* du site. L'utilisateur s'y connecte depuis l'interface, qui affiche la page et
|
|
88
|
+
* lui relaie ses clics et sa frappe ; aucun écran n'est nécessaire. Les cookies
|
|
89
|
+
* sont enregistrés pour le site dès que le signe de connexion apparaît.
|
|
78
90
|
*/
|
|
79
91
|
export async function startLogin(siteId) {
|
|
80
92
|
if (active)
|
|
@@ -83,7 +95,12 @@ export async function startLogin(siteId) {
|
|
|
83
95
|
if (!site.loginUrl)
|
|
84
96
|
throw new Error(`Le site "${site.name}" n'a pas de page de connexion.`);
|
|
85
97
|
const profileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'simple-scraper-login-'));
|
|
86
|
-
const browser = await puppeteer.launch({
|
|
98
|
+
const browser = await puppeteer.launch({
|
|
99
|
+
headless: true,
|
|
100
|
+
defaultViewport: getDefaultViewport(),
|
|
101
|
+
userDataDir: profileDir,
|
|
102
|
+
args: BROWSER_LAUNCH_ARGS,
|
|
103
|
+
});
|
|
87
104
|
const page = (await browser.pages())[0] ?? (await browser.newPage());
|
|
88
105
|
const login = {
|
|
89
106
|
status: { siteId, phase: 'waiting', url: site.loginUrl, error: null, cookieCount: null, startedAt: Date.now() },
|
|
@@ -105,7 +122,6 @@ export async function startLogin(siteId) {
|
|
|
105
122
|
settle(login, { phase: 'error', error: err instanceof Error ? err.message : String(err) });
|
|
106
123
|
void teardown(login);
|
|
107
124
|
});
|
|
108
|
-
await page.bringToFront();
|
|
109
125
|
return login.status;
|
|
110
126
|
}
|
|
111
127
|
/** Connexion en cours, ou la dernière terminée. */
|
|
@@ -127,13 +143,14 @@ export async function cancelLogin() {
|
|
|
127
143
|
settle(login, { phase: 'cancelled' });
|
|
128
144
|
await teardown(login);
|
|
129
145
|
}
|
|
130
|
-
/**
|
|
146
|
+
/** Rejoue dans la page de connexion un clic, une frappe ou une navigation de l'utilisateur. */
|
|
147
|
+
export async function loginInput(action) {
|
|
148
|
+
const page = await requirePage();
|
|
149
|
+
await applyRemoteAction(page, action);
|
|
150
|
+
if (active)
|
|
151
|
+
active.status.url = page.url();
|
|
152
|
+
}
|
|
153
|
+
/** Capture JPEG de l'onglet courant du navigateur de connexion. */
|
|
131
154
|
export async function loginScreenshot() {
|
|
132
|
-
|
|
133
|
-
throw new Error('Aucune connexion en cours.');
|
|
134
|
-
const page = await currentPage(active.browser);
|
|
135
|
-
if (!page)
|
|
136
|
-
throw new Error('La fenêtre de connexion n\'a aucun onglet.');
|
|
137
|
-
const buffer = await page.screenshot({ type: 'jpeg', quality: 60 });
|
|
138
|
-
return Buffer.from(buffer).toString('base64');
|
|
155
|
+
return captureJpegBase64(await requirePage(), 70);
|
|
139
156
|
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { getDefaultTimeout } from './browser.js';
|
|
3
|
+
/**
|
|
4
|
+
* Action de l'utilisateur sur une page affichée dans l'interface. Les
|
|
5
|
+
* coordonnées d'un clic sont des fractions (0 à 1) de la capture, donc
|
|
6
|
+
* indépendantes de la taille à laquelle l'interface l'affiche.
|
|
7
|
+
*/
|
|
8
|
+
export const remoteActionSchema = z.discriminatedUnion('type', [
|
|
9
|
+
z.object({ type: z.literal('click'), x: z.number().min(0).max(1), y: z.number().min(0).max(1) }),
|
|
10
|
+
z.object({ type: z.literal('scroll'), dy: z.number().min(-5000).max(5000) }),
|
|
11
|
+
z.object({ type: z.literal('text'), text: z.string().min(1).max(2000) }),
|
|
12
|
+
z.object({
|
|
13
|
+
type: z.literal('key'),
|
|
14
|
+
key: z.string().min(1).max(32),
|
|
15
|
+
modifiers: z.array(z.enum(['Control', 'Shift', 'Alt', 'Meta'])).optional(),
|
|
16
|
+
}),
|
|
17
|
+
z.object({ type: z.literal('navigate'), url: z.url() }),
|
|
18
|
+
z.object({ type: z.literal('back') }),
|
|
19
|
+
]);
|
|
20
|
+
export function parseRemoteAction(data) {
|
|
21
|
+
const result = remoteActionSchema.safeParse(data);
|
|
22
|
+
if (!result.success)
|
|
23
|
+
throw new Error(`Action invalide : ${result.error.issues.map(i => i.message).join(' ')}`);
|
|
24
|
+
return result.data;
|
|
25
|
+
}
|
|
26
|
+
/** Point de la page, en pixels CSS, désigné par des fractions de sa capture. */
|
|
27
|
+
export function pointOf(page, x, y) {
|
|
28
|
+
const viewport = page.viewport();
|
|
29
|
+
if (!viewport)
|
|
30
|
+
throw new Error('La page n\'a pas de taille d\'affichage fixe.');
|
|
31
|
+
return { x: x * viewport.width, y: y * viewport.height };
|
|
32
|
+
}
|
|
33
|
+
/** Rejoue l'action dans la page. */
|
|
34
|
+
export async function applyRemoteAction(page, action) {
|
|
35
|
+
switch (action.type) {
|
|
36
|
+
case 'click': {
|
|
37
|
+
const point = pointOf(page, action.x, action.y);
|
|
38
|
+
await page.mouse.click(point.x, point.y);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
case 'scroll':
|
|
42
|
+
await page.mouse.wheel({ deltaY: action.dy });
|
|
43
|
+
return;
|
|
44
|
+
case 'text':
|
|
45
|
+
await page.keyboard.type(action.text);
|
|
46
|
+
return;
|
|
47
|
+
case 'key': {
|
|
48
|
+
const modifiers = action.modifiers ?? [];
|
|
49
|
+
for (const m of modifiers)
|
|
50
|
+
await page.keyboard.down(m);
|
|
51
|
+
try {
|
|
52
|
+
await page.keyboard.press(action.key);
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
for (const m of modifiers.slice().reverse())
|
|
56
|
+
await page.keyboard.up(m);
|
|
57
|
+
}
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
case 'navigate':
|
|
61
|
+
await page.goto(action.url, { waitUntil: 'domcontentloaded', timeout: getDefaultTimeout() });
|
|
62
|
+
return;
|
|
63
|
+
case 'back':
|
|
64
|
+
await page.goBack({ waitUntil: 'domcontentloaded', timeout: getDefaultTimeout() });
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
}
|
package/dist/rpc.js
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { captureJpegBase64 } from './capture.js';
|
|
2
3
|
import { clearSiteCookies, replaceSiteCookies } from './cookie-sync.js';
|
|
3
4
|
import { parseNetscapeCookies } from './cookies.js';
|
|
4
5
|
import { getDataDir } from './data-dir.js';
|
|
5
6
|
import { nextKeepAliveAt, nextVisitAt } from './keep-alive.js';
|
|
6
|
-
import { cancelLogin, confirmLogin, getLoginStatus, loginScreenshot, startLogin } from './login.js';
|
|
7
|
+
import { cancelLogin, confirmLogin, getLoginStatus, loginInput, loginScreenshot, startLogin } from './login.js';
|
|
8
|
+
import { parseRemoteAction } from './remote-input.js';
|
|
9
|
+
import { closePicker, openPicker, pickAt, pickerScreenshot, pickerScroll, testSignal } from './signal-picker.js';
|
|
7
10
|
import { checkSite, isChecking } from './site-check.js';
|
|
8
11
|
import { cookieCountLabel, getStore, requireStore } from './site-store.js';
|
|
9
12
|
import { closeAllSessions, closeSession, getSessionLogs, listSessions, peekSession } from './sessions.js';
|
|
13
|
+
const signalSchema = z.object({ kind: z.enum(['url', 'selector']), value: z.string().trim() });
|
|
10
14
|
const siteConfigSchema = z.object({
|
|
11
15
|
name: z.string().trim().min(1, 'Le nom est requis.'),
|
|
12
16
|
domains: z
|
|
13
17
|
.array(z.string().trim().toLowerCase().regex(/^[a-z0-9.-]+$/, 'Domaine invalide.'))
|
|
14
18
|
.min(1, 'Au moins un domaine est requis.'),
|
|
15
19
|
loginUrl: z.union([z.literal(''), z.url('Page de connexion invalide.')]),
|
|
16
|
-
signal:
|
|
20
|
+
signal: signalSchema,
|
|
17
21
|
keepAlive: z.object({
|
|
18
22
|
enabled: z.boolean(),
|
|
19
23
|
intervalMin: z.number().int().min(1).max(24 * 60),
|
|
@@ -21,12 +25,18 @@ const siteConfigSchema = z.object({
|
|
|
21
25
|
}),
|
|
22
26
|
flagWhenLost: z.boolean(),
|
|
23
27
|
});
|
|
24
|
-
|
|
25
|
-
|
|
28
|
+
/** Valide une donnée de la GUI ; l'erreur reprend les messages du schéma. */
|
|
29
|
+
function parse(schema, data) {
|
|
30
|
+
const result = schema.safeParse(data);
|
|
26
31
|
if (!result.success)
|
|
27
32
|
throw new Error(result.error.issues.map(i => i.message).join(' '));
|
|
28
33
|
return result.data;
|
|
29
34
|
}
|
|
35
|
+
const pageUrlSchema = z.url('Page visitée invalide.');
|
|
36
|
+
const ratioSchema = z.number().min(0).max(1);
|
|
37
|
+
function parseSiteConfig(data) {
|
|
38
|
+
return parse(siteConfigSchema, data);
|
|
39
|
+
}
|
|
30
40
|
/** Site enrichi de ce que l'interface affiche à côté de ses réglages. */
|
|
31
41
|
function siteView(site) {
|
|
32
42
|
const store = requireStore();
|
|
@@ -116,15 +126,30 @@ const handlers = {
|
|
|
116
126
|
LOGIN_START: ({ siteId }) => startLogin(siteId),
|
|
117
127
|
LOGIN_STATUS: () => getLoginStatus(),
|
|
118
128
|
LOGIN_SCREENSHOT: async () => ({ base64: await loginScreenshot() }),
|
|
129
|
+
LOGIN_INPUT: async ({ action }) => {
|
|
130
|
+
await loginInput(parseRemoteAction(action));
|
|
131
|
+
return getLoginStatus();
|
|
132
|
+
},
|
|
119
133
|
LOGIN_CONFIRM: () => confirmLogin(),
|
|
120
134
|
LOGIN_CANCEL: async () => {
|
|
121
135
|
await cancelLogin();
|
|
122
136
|
return null;
|
|
123
137
|
},
|
|
138
|
+
PICKER_OPEN: ({ pageUrl }) => openPicker(parse(pageUrlSchema, pageUrl)),
|
|
139
|
+
PICKER_SCREENSHOT: async () => ({ base64: await pickerScreenshot() }),
|
|
140
|
+
PICKER_SCROLL: async ({ dy }) => {
|
|
141
|
+
await pickerScroll(parse(z.number().min(-5000).max(5000), dy));
|
|
142
|
+
return null;
|
|
143
|
+
},
|
|
144
|
+
PICKER_PICK: ({ x, y }) => pickAt(parse(ratioSchema, x), parse(ratioSchema, y)),
|
|
145
|
+
PICKER_CLOSE: async () => {
|
|
146
|
+
await closePicker();
|
|
147
|
+
return null;
|
|
148
|
+
},
|
|
149
|
+
SIGNAL_TEST: ({ pageUrl, signal }) => testSignal(parse(pageUrlSchema, pageUrl), parse(signalSchema, signal)),
|
|
124
150
|
SESSION_LIST: () => sessionViews(),
|
|
125
151
|
SESSION_SCREENSHOT: async ({ sessionId }) => {
|
|
126
|
-
|
|
127
|
-
return { base64: Buffer.from(buffer).toString('base64') };
|
|
152
|
+
return { base64: await captureJpegBase64(peekSession(sessionId), 60) };
|
|
128
153
|
},
|
|
129
154
|
SESSION_LOGS: ({ sessionId }) => getSessionLogs(sessionId),
|
|
130
155
|
SESSION_CLOSE: async ({ sessionId }) => {
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { getBrowser, getDefaultTimeout } from './browser.js';
|
|
2
|
+
import { captureJpegBase64 } from './capture.js';
|
|
3
|
+
import { applyRemoteAction, pointOf } from './remote-input.js';
|
|
4
|
+
import { pageShowsSignal, urlMatchesSignal } from './site-check.js';
|
|
5
|
+
const SIGNAL_WAIT_MS = 3_000;
|
|
6
|
+
let picker = null;
|
|
7
|
+
function requirePicker() {
|
|
8
|
+
if (!picker || picker.page.isClosed())
|
|
9
|
+
throw new Error('Aucune page ouverte pour choisir le signe de connexion.');
|
|
10
|
+
return picker;
|
|
11
|
+
}
|
|
12
|
+
/** Ouvre la page visitée du site dans le navigateur partagé, donc avec les cookies des sites. */
|
|
13
|
+
export async function openPicker(pageUrl) {
|
|
14
|
+
await closePicker();
|
|
15
|
+
const page = await (await getBrowser()).newPage();
|
|
16
|
+
picker = { page, pageUrl };
|
|
17
|
+
await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
|
|
18
|
+
return { url: page.url() };
|
|
19
|
+
}
|
|
20
|
+
export async function closePicker() {
|
|
21
|
+
const current = picker;
|
|
22
|
+
picker = null;
|
|
23
|
+
await current?.page.close().catch(() => undefined);
|
|
24
|
+
}
|
|
25
|
+
export async function pickerScreenshot() {
|
|
26
|
+
return captureJpegBase64(requirePicker().page, 70);
|
|
27
|
+
}
|
|
28
|
+
/** Fait défiler la page du sélecteur ; seul le défilement y est relayé. */
|
|
29
|
+
export async function pickerScroll(dy) {
|
|
30
|
+
const action = { type: 'scroll', dy };
|
|
31
|
+
await applyRemoteAction(requirePicker().page, action);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Sélecteurs désignant l'élément situé au point donné, du plus précis au plus
|
|
35
|
+
* général : son texte dans son conteneur repérable, son chemin depuis ce
|
|
36
|
+
* conteneur, le conteneur seul, l'élément seul.
|
|
37
|
+
*/
|
|
38
|
+
function candidatesAt(x, y) {
|
|
39
|
+
const target = document.elementFromPoint(x, y);
|
|
40
|
+
if (!target)
|
|
41
|
+
return [];
|
|
42
|
+
const stable = (name) => /^[a-zA-Z_][\w-]*$/.test(name) &&
|
|
43
|
+
!/\d{3,}/.test(name) &&
|
|
44
|
+
!/^(active|open|show|hover|focus|focused|in|collapsed|selected|disabled|is-|has-|ng-)/.test(name);
|
|
45
|
+
const count = (selector) => document.querySelectorAll(selector).length;
|
|
46
|
+
/** Classe stable de l'élément la plus rare dans la page ; à égalité, la plus longue, souvent la plus parlante. */
|
|
47
|
+
const ownClass = (el) => Array.from(el.classList)
|
|
48
|
+
.filter(stable)
|
|
49
|
+
.sort((a, b) => count(`.${CSS.escape(a)}`) - count(`.${CSS.escape(b)}`) || b.length - a.length)[0];
|
|
50
|
+
const selfSel = (el) => {
|
|
51
|
+
if (el.id && stable(el.id))
|
|
52
|
+
return `#${CSS.escape(el.id)}`;
|
|
53
|
+
const cls = ownClass(el);
|
|
54
|
+
return cls ? `.${CSS.escape(cls)}` : null;
|
|
55
|
+
};
|
|
56
|
+
const part = (el) => {
|
|
57
|
+
if (el.id && stable(el.id))
|
|
58
|
+
return `#${CSS.escape(el.id)}`;
|
|
59
|
+
const cls = ownClass(el);
|
|
60
|
+
return el.tagName.toLowerCase() + (cls ? `.${CSS.escape(cls)}` : '');
|
|
61
|
+
};
|
|
62
|
+
// Conteneur : l'ancêtre repérable le plus proche qui soit unique dans la page,
|
|
63
|
+
// sinon le plus proche repérable.
|
|
64
|
+
let anchorSel = null;
|
|
65
|
+
let firstSel = null;
|
|
66
|
+
for (let el = target.parentElement, depth = 0; el && depth < 6; el = el.parentElement, depth++) {
|
|
67
|
+
if (el === document.body || el === document.documentElement)
|
|
68
|
+
break;
|
|
69
|
+
const sel = selfSel(el);
|
|
70
|
+
if (!sel)
|
|
71
|
+
continue;
|
|
72
|
+
firstSel ??= sel;
|
|
73
|
+
if (count(sel) === 1) {
|
|
74
|
+
anchorSel = sel;
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
anchorSel ??= firstSel;
|
|
79
|
+
const targetSel = part(target);
|
|
80
|
+
const scoped = anchorSel ? `${anchorSel} ${targetSel}` : targetSel;
|
|
81
|
+
const out = [];
|
|
82
|
+
const text = (target.textContent ?? '').replace(/\s+/g, ' ').trim();
|
|
83
|
+
if (text && text.length <= 60 && target.children.length === 0) {
|
|
84
|
+
const quoted = JSON.stringify(text);
|
|
85
|
+
out.push({ selector: `${scoped}::-p-text(${quoted})`, label: `Le texte « ${text} » à cet endroit` });
|
|
86
|
+
}
|
|
87
|
+
if (anchorSel)
|
|
88
|
+
out.push({ selector: scoped, label: 'Cet élément, dans son conteneur' });
|
|
89
|
+
if (anchorSel)
|
|
90
|
+
out.push({ selector: anchorSel, label: 'Son conteneur' });
|
|
91
|
+
out.push({ selector: targetSel, label: 'Cet élément, n\'importe où dans la page' });
|
|
92
|
+
return out.filter((c, i) => out.findIndex(o => o.selector === c.selector) === i);
|
|
93
|
+
}
|
|
94
|
+
/** Ouvre la page dans un contexte sans cookies, comme la verrait une session perdue. */
|
|
95
|
+
async function withAnonymousPage(pageUrl, fn) {
|
|
96
|
+
const context = await (await getBrowser()).createBrowserContext();
|
|
97
|
+
try {
|
|
98
|
+
const page = await context.newPage();
|
|
99
|
+
await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
|
|
100
|
+
return await fn(page);
|
|
101
|
+
}
|
|
102
|
+
finally {
|
|
103
|
+
await context.close().catch(() => undefined);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async function countMatches(page, selector) {
|
|
107
|
+
return (await page.$$(selector).catch(() => [])).length;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Propose les sélecteurs de l'élément cliqué, chacun compté sur la page vue
|
|
111
|
+
* connectée et sur la même page vue sans cookies. Un bon signe de connexion est
|
|
112
|
+
* trouvé dans le premier cas et absent dans le second ; ceux-là viennent en tête.
|
|
113
|
+
*/
|
|
114
|
+
export async function pickAt(x, y) {
|
|
115
|
+
const { page, pageUrl } = requirePicker();
|
|
116
|
+
const point = pointOf(page, x, y);
|
|
117
|
+
const proposals = await page.evaluate(candidatesAt, point.x, point.y);
|
|
118
|
+
if (!proposals.length)
|
|
119
|
+
return [];
|
|
120
|
+
const connected = await Promise.all(proposals.map(p => countMatches(page, p.selector)));
|
|
121
|
+
const anonymous = await withAnonymousPage(pageUrl, anon => Promise.all(proposals.map(p => countMatches(anon, p.selector))));
|
|
122
|
+
const results = proposals.map((p, i) => ({ ...p, connected: connected[i], anonymous: anonymous[i] }));
|
|
123
|
+
const good = (c) => c.connected > 0 && c.anonymous === 0;
|
|
124
|
+
return [...results.filter(good), ...results.filter(c => !good(c))];
|
|
125
|
+
}
|
|
126
|
+
/** Vérifie le signe sur la page vue connectée (attendu : trouvé) et sans cookies (attendu : absent). */
|
|
127
|
+
export async function testSignal(pageUrl, signal) {
|
|
128
|
+
const page = await (await getBrowser()).newPage();
|
|
129
|
+
let connected;
|
|
130
|
+
let connectedUrl;
|
|
131
|
+
try {
|
|
132
|
+
await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
|
|
133
|
+
connectedUrl = page.url();
|
|
134
|
+
connected = await pageShowsSignal(page, signal, SIGNAL_WAIT_MS);
|
|
135
|
+
}
|
|
136
|
+
finally {
|
|
137
|
+
await page.close().catch(() => undefined);
|
|
138
|
+
}
|
|
139
|
+
const anon = await withAnonymousPage(pageUrl, async (p) => ({
|
|
140
|
+
url: p.url(),
|
|
141
|
+
shown: signal.kind === 'url' ? urlMatchesSignal(p.url(), signal.value) : await pageShowsSignal(p, signal),
|
|
142
|
+
}));
|
|
143
|
+
return { connected, anonymous: anon.shown, connectedUrl, anonymousUrl: anon.url };
|
|
144
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@imenam/simple-scraper",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.29",
|
|
4
4
|
"description": "MCP server for web scraping and JavaScript execution using Puppeteer",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -20,7 +20,8 @@
|
|
|
20
20
|
"dev:server": "tsx src/index.ts",
|
|
21
21
|
"dev:gui": "cd public && npx vite",
|
|
22
22
|
"prepublishOnly": "npm run build",
|
|
23
|
-
"release": "npm version patch && npm run build && npm publish --access public && git push origin master --follow-tags"
|
|
23
|
+
"release": "npm version patch && npm run build && npm publish --access public && git push origin master --follow-tags",
|
|
24
|
+
"prebuild": "node scripts/clean-dist.mjs"
|
|
24
25
|
},
|
|
25
26
|
"keywords": [
|
|
26
27
|
"mcp",
|