@imenam/simple-scraper 1.0.23 → 1.0.27

package/dist/browser.js

@@ -1,5 +1,7 @@
 import puppeteer from 'puppeteer';
+import { getStore } from './site-store.js';
 let browser = null;
+let launching = null;
 export function getDefaultTimeout() {
     const val = process.env.PUPPETEER_TIMEOUT;
     return val ? parseInt(val, 10) : 30_000;
@@ -27,18 +29,36 @@ export function getDefaultViewport() {
         deviceScaleFactor: envNumber('PUPPETEER_DEVICE_SCALE_FACTOR', 1),
     };
 }
-export async function getBrowser() {
-    if (browser && browser.connected) {
-        return browser;
-    }
+/** Lance le navigateur partagé et y charge les cookies de tous les sites. */
+async function launchBrowser() {
+    const cookies = getStore()?.allCookies() ?? [];
     const headless = process.env.PUPPETEER_HEADLESS !== 'false';
-    browser = await puppeteer.launch({
+    const launched = await puppeteer.launch({
         headless,
         defaultViewport: getDefaultViewport(),
         args: ['--no-sandbox', '--disable-setuid-sandbox'],
     });
-    console.error('[Browser] Launched Puppeteer instance');
-    return browser;
+    if (cookies.length)
+        await launched.setCookie(...cookies);
+    console.error(`[Browser] Launched Puppeteer instance (${cookies.length} site cookie(s) loaded)`);
+    return launched;
+}
+export async function getBrowser() {
+    if (browser && browser.connected) {
+        return browser;
+    }
+    if (!launching) {
+        launching = launchBrowser()
+            .then(b => (browser = b))
+            .finally(() => {
+            launching = null;
+        });
+    }
+    return launching;
+}
+/** Navigateur partagé s'il est déjà lancé, sans le démarrer. */
+export function getRunningBrowser() {
+    return browser && browser.connected ? browser : null;
 }
 export async function closeBrowser() {
     if (browser) {
@@ -54,11 +74,13 @@ export async function closeBrowser() {
 }
 for (const sig of ['SIGTERM', 'SIGINT']) {
     process.on(sig, async () => {
-        // Sessions must be closed before the browser to avoid dangling page handles.
-        // closeAllSessions is imported dynamically to avoid a circular dependency
-        // (sessions.ts → browser.ts → sessions.ts).
+        // Sessions must be closed before the browser to avoid dangling page handles,
+        // and site cookies saved before it disappears. Both modules are imported
+        // dynamically: each of them imports this one.
         const { closeAllSessions } = await import('./sessions.js');
+        const { syncJarsFromBrowser } = await import('./cookie-sync.js');
         await closeAllSessions();
+        await syncJarsFromBrowser().catch(err => console.error(`[Cookies] Synchronisation échouée : ${err}`));
         await closeBrowser();
         process.exit(0);
     });

package/dist/cli/setup-mcp.js

@@ -33,7 +33,7 @@ function buildServerEntry(codeRoot) {
             PROXY_APP_PATH: `/simple-scraper-${folder}`,
             PUPPETEER_VIEWPORT_WIDTH: '1920',
             PUPPETEER_VIEWPORT_HEIGHT: '1080',
-            COOKIES_DIR: '',
+            SCRAPER_DATA_DIR: '',
         },
     };
 }
@@ -44,7 +44,7 @@ const GENERATED_VARS = [
     ['PROXY_APP_PATH', 'chemin de montage de la GUI derrière le proxy'],
     ['PUPPETEER_VIEWPORT_WIDTH', 'largeur du viewport en pixels CSS'],
     ['PUPPETEER_VIEWPORT_HEIGHT', 'hauteur du viewport en pixels CSS'],
-    ['COOKIES_DIR', 'dossier de fichiers cookies au format Netscape (.txt)'],
+    ['SCRAPER_DATA_DIR', 'dossier des sites : réglages, cookies, historique (alias : COOKIES_DIR)'],
 ];
 const OPTIONAL_VARS = [
     ['PUPPETEER_DEVICE_SCALE_FACTOR', 'pixels physiques par pixel CSS (2 = screenshots retina)'],

package/dist/cookie-sync.js

@@ -0,0 +1,67 @@
+import { getRunningBrowser } from './browser.js';
+import { cookieInScope, getStore, requireStore, toCookieData } from './site-store.js';
+const PERIODIC_SYNC_MS = 5 * 60 * 1000;
+let queue = Promise.resolve();
+/** Exécute les opérations sur les cookies une par une, dans l'ordre d'appel. */
+function runExclusive(fn) {
+    const next = queue.then(fn);
+    queue = next.catch(() => undefined);
+    return next;
+}
+/**
+ * Recopie dans les cookies de chaque site ceux que le navigateur partagé détient
+ * pour ses domaines : les cookies renouvelés par le site sont gardés, ceux qu'il
+ * a supprimés disparaissent.
+ */
+export function syncJarsFromBrowser() {
+    return runExclusive(async () => {
+        const browser = getRunningBrowser();
+        const store = getStore();
+        if (!browser || !store)
+            return;
+        const all = await browser.cookies();
+        for (const site of store.list()) {
+            const scope = store.cookieScope(site.id);
+            store.writeJar(site.id, all.filter(c => cookieInScope(c.domain, scope)));
+        }
+    });
+}
+/** Lance la synchronisation en arrière-plan ; une erreur est journalisée. */
+export function scheduleJarSync() {
+    syncJarsFromBrowser().catch(err => console.error(`[Cookies] Synchronisation échouée : ${err instanceof Error ? err.message : String(err)}`));
+}
+export function startPeriodicJarSync() {
+    setInterval(scheduleJarSync, PERIODIC_SYNC_MS).unref();
+}
+/** Retire du navigateur partagé les cookies couverts par les domaines donnés. */
+async function deleteBrowserCookies(scope) {
+    const browser = getRunningBrowser();
+    if (!browser)
+        return;
+    const existing = (await browser.cookies()).filter(c => cookieInScope(c.domain, scope));
+    if (existing.length)
+        await browser.deleteCookie(...existing);
+}
+/** Remplace les cookies du site, dans son dossier comme dans le navigateur partagé. */
+export function replaceSiteCookies(siteId, cookies) {
+    return runExclusive(async () => {
+        const store = requireStore();
+        const previousScope = store.cookieScope(siteId);
+        const prepared = cookies.map(toCookieData);
+        store.writeJar(siteId, prepared);
+        await deleteBrowserCookies(Array.from(new Set([...previousScope, ...store.cookieScope(siteId)])));
+        const browser = getRunningBrowser();
+        if (browser && prepared.length)
+            await browser.setCookie(...prepared);
+        return prepared.length;
+    });
+}
+/** Efface les cookies du site, dans son dossier comme dans le navigateur partagé. */
+export function clearSiteCookies(siteId) {
+    return runExclusive(async () => {
+        const store = requireStore();
+        const scope = store.cookieScope(siteId);
+        store.writeJar(siteId, []);
+        await deleteBrowserCookies(scope);
+    });
+}

package/dist/cookies.js

@@ -1,49 +1,37 @@
-import fs from 'fs/promises';
-import path from 'path';
+const HTTP_ONLY_PREFIX = '#HttpOnly_';
+/**
+ * Lit un export de cookies au format Netscape (une ligne par cookie, champs
+ * séparés par des tabulations). Le préfixe `#HttpOnly_` marque un cookie
+ * http-only ; une date d'expiration à 0 désigne un cookie de session.
+ */
 export function parseNetscapeCookies(content) {
-    return content
-        .split('\n')
-        .filter(line => line.trim() && !line.startsWith('#'))
-        .map(line => {
+    const cookies = [];
+    for (const rawLine of content.split(/\r?\n/)) {
+        let line = rawLine.trim();
+        let httpOnly = false;
+        if (line.startsWith(HTTP_ONLY_PREFIX)) {
+            httpOnly = true;
+            line = line.slice(HTTP_ONLY_PREFIX.length);
+        }
+        else if (!line || line.startsWith('#')) {
+            continue;
+        }
         const parts = line.split('\t');
         if (parts.length < 7)
-            return null;
-        const [domain, , cookiePath, secure, , name, value] = parts;
-        return {
+            continue;
+        const [domain, , cookiePath, secure, expires, name, ...valueParts] = parts;
+        const cookie = {
             domain: domain.trim(),
-            path: cookiePath.trim(),
+            path: cookiePath.trim() || '/',
             secure: secure.trim().toUpperCase() === 'TRUE',
+            httpOnly,
             name: name.trim(),
-            value: value.trim(),
+            value: valueParts.join('\t').trim(),
         };
-    })
-        .filter((c) => c !== null);
-}
-/**
- * Loads and merges all cookie files from the COOKIES_DIR directory.
- * The directory is created if it does not exist yet.
- * Silently returns an empty array if COOKIES_DIR is not set.
- */
-export async function loadAllCookies() {
-    const cookiesDir = process.env.COOKIES_DIR;
-    if (!cookiesDir)
-        return [];
-    let files;
-    try {
-        await fs.mkdir(cookiesDir, { recursive: true });
-        files = await fs.readdir(cookiesDir);
-    }
-    catch {
-        return [];
+        const expiresSec = Number(expires);
+        if (Number.isFinite(expiresSec) && expiresSec > 0)
+            cookie.expires = expiresSec;
+        cookies.push(cookie);
     }
-    const results = await Promise.all(files.map(async (file) => {
-        try {
-            const content = await fs.readFile(path.join(cookiesDir, file), 'utf-8');
-            return parseNetscapeCookies(content);
-        }
-        catch {
-            return [];
-        }
-    }));
-    return results.flat();
+    return cookies;
 }

package/dist/data-dir.js

@@ -0,0 +1,24 @@
+import fs from 'fs';
+import path from 'path';
+/**
+ * Dossier de données des sites (réglages, cookies, historique).
+ *
+ * SCRAPER_DATA_DIR et COOKIES_DIR sont deux noms du même réglage : l'un ou
+ * l'autre suffit. Définis tous les deux, ils doivent désigner le même dossier.
+ * Retourne null quand aucun des deux n'est renseigné : les sites sont alors
+ * inactifs. Le dossier est créé s'il n'existe pas.
+ */
+export function getDataDir() {
+    const dataDir = process.env.SCRAPER_DATA_DIR?.trim() ?? '';
+    const cookiesDir = process.env.COOKIES_DIR?.trim() ?? '';
+    if (dataDir && cookiesDir && path.resolve(dataDir) !== path.resolve(cookiesDir)) {
+        throw new Error(`SCRAPER_DATA_DIR (${dataDir}) et COOKIES_DIR (${cookiesDir}) désignent deux dossiers différents. ` +
+            'Ces deux variables nomment le même réglage : définir l\'une des deux, ou leur donner la même valeur.');
+    }
+    const dir = dataDir || cookiesDir;
+    if (!dir)
+        return null;
+    const resolved = path.resolve(dir);
+    fs.mkdirSync(resolved, { recursive: true });
+    return resolved;
+}

package/dist/gui.js

@@ -1,7 +1,6 @@
 import express from 'express';
 import bodyParser from 'body-parser';
 import path from 'path';
-import fs from 'fs';
 import { fileURLToPath } from 'url';
 import { setupLogging, ProxyClient, IpcHub, GUI_LISTEN_HOST, expressGuiGuard } from '@imenam/mcp-gui-interface';
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -13,6 +12,7 @@ async function callMaster(type, data, timeoutMs = 5000) {
     return resp.data;
 }
 const SESSION_ID_RE = /^sess_[a-z0-9]{6}$/;
+const SITE_ID_RE = /^site_[a-z0-9]{6}$/;
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const rootDir = path.resolve(__dirname, '..');
 setupLogging({ processLabel: 'GUI', logDir: process.env.MCP_LOG_DIR });
@@ -61,17 +61,9 @@ process.on('unhandledRejection', (reason) => {
     console.error(`[GUI] Unhandled Rejection: ${reason}`);
     process.exit(1);
 });
-/** Retourne COOKIES_DIR en le créant au besoin (null si non configuré). */
-const getCookiesDir = () => {
-    const cookiesDir = process.env.COOKIES_DIR ?? null;
-    if (!cookiesDir)
-        return null;
-    fs.mkdirSync(cookiesDir, { recursive: true });
-    return cookiesDir;
-};
 const app = express();
 app.use(expressGuiGuard());
-app.use(bodyParser.json());
+app.use(bodyParser.json({ limit: '5mb' }));
 app.use(express.static(path.join(rootDir, 'public/dist')));
 app.get(/^(?!\/api\/|\/proxy\/)/, (_req, res) => {
     res.sendFile(path.join(rootDir, 'public/dist/index.html'));
@@ -79,103 +71,101 @@ app.get(/^(?!\/api\/|\/proxy\/)/, (_req, res) => {
 app.get('/proxy/health', (_req, res) => {
     res.status(200).json({ status: 'healthy' });
 });
-// ─── GET /api/cookies — liste les fichiers du dossier ─────────────────────────
-app.get('/api/cookies', (_req, res) => {
-    const cookiesDir = getCookiesDir();
-    if (!cookiesDir) {
-        res.status(400).json({ success: false, error: 'COOKIES_DIR not configured' });
-        return;
-    }
+/** Transmet la requête au master et renvoie sa réponse en JSON, ou l'erreur en 500. */
+async function forward(res, type, data, timeoutMs) {
     try {
-        const files = fs.readdirSync(cookiesDir).filter(f => fs.statSync(path.join(cookiesDir, f)).isFile());
-        res.json({ success: true, files });
+        const result = await callMaster(type, data, timeoutMs);
+        res.json({ success: true, data: result ?? null });
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         res.status(500).json({ success: false, error: message });
     }
-});
-// ─── POST /api/cookies — ajoute un fichier ────────────────────────────────────
-app.post('/api/cookies', (req, res) => {
-    const cookiesDir = getCookiesDir();
-    if (!cookiesDir) {
-        res.status(400).json({ success: false, error: 'COOKIES_DIR not configured' });
-        return;
-    }
-    const { filename, content } = req.body;
-    if (!filename || typeof content !== 'string') {
-        res.status(400).json({ success: false, error: 'filename and content are required' });
-        return;
-    }
-    const safeName = path.basename(filename);
+}
+/** Renvoie la capture JPEG produite par le master. */
+async function sendJpeg(res, type, data) {
     try {
-        fs.writeFileSync(path.join(cookiesDir, safeName), content, 'utf-8');
-        res.json({ success: true });
+        const shot = await callMaster(type, data);
+        res.type('image/jpeg').send(Buffer.from(shot.base64, 'base64'));
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         res.status(500).json({ success: false, error: message });
     }
+}
+/** Refuse en 400 un identifiant mal formé ; retourne false dans ce cas. */
+function checkId(res, value, re, label) {
+    if (re.test(value))
+        return true;
+    res.status(400).json({ success: false, error: `Invalid ${label} ID` });
+    return false;
+}
+// ─── Statut général ───────────────────────────────────────────────────────────
+app.get('/api/status', (_req, res) => forward(res, 'STATUS'));
+// ─── Sites ────────────────────────────────────────────────────────────────────
+app.get('/api/sites', (_req, res) => forward(res, 'SITES_LIST'));
+app.post('/api/sites', (req, res) => forward(res, 'SITE_CREATE', { config: req.body }));
+app.get('/api/sites/:id', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site'))
+        void forward(res, 'SITE_GET', { siteId: req.params.id });
 });
-// ─── GET /api/cookies/:filename — lit le contenu d'un fichier ────────────────
-app.get('/api/cookies/:filename', (req, res) => {
-    const cookiesDir = getCookiesDir();
-    if (!cookiesDir) {
-        res.status(400).json({ success: false, error: 'COOKIES_DIR not configured' });
-        return;
-    }
-    const safeName = path.basename(req.params.filename);
-    try {
-        const content = fs.readFileSync(path.join(cookiesDir, safeName), 'utf-8');
-        res.json({ success: true, content });
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        res.status(500).json({ success: false, error: message });
+app.put('/api/sites/:id', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site')) {
+        void forward(res, 'SITE_UPDATE', { siteId: req.params.id, config: req.body });
     }
 });
-// ─── DELETE /api/cookies/:filename — supprime un fichier ──────────────────────
-app.delete('/api/cookies/:filename', (req, res) => {
-    const cookiesDir = getCookiesDir();
-    if (!cookiesDir) {
-        res.status(400).json({ success: false, error: 'COOKIES_DIR not configured' });
-        return;
-    }
-    const safeName = path.basename(req.params.filename);
-    try {
-        fs.unlinkSync(path.join(cookiesDir, safeName));
-        res.json({ success: true });
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        res.status(500).json({ success: false, error: message });
-    }
+app.delete('/api/sites/:id', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site'))
+        void forward(res, 'SITE_DELETE', { siteId: req.params.id });
 });
-// ─── GET /api/sessions — liste les sessions actives ──────────────────────────
-app.get('/api/sessions', async (_req, res) => {
-    try {
-        const sessions = await callMaster('SESSION_LIST');
-        res.json({ success: true, sessions });
+app.post('/api/sites/:id/check', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site'))
+        void forward(res, 'SITE_CHECK', { siteId: req.params.id });
+});
+app.post('/api/sites/:id/cookies', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site')) {
+        void forward(res, 'SITE_IMPORT_COOKIES', { siteId: req.params.id, content: req.body.content }, 15_000);
     }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        res.status(500).json({ success: false, error: message });
+});
+app.delete('/api/sites/:id/cookies', (req, res) => {
+    if (checkId(res, req.params.id, SITE_ID_RE, 'site')) {
+        void forward(res, 'SITE_CLEAR_COOKIES', { siteId: req.params.id }, 15_000);
     }
 });
-// ─── GET /api/sessions/:id/screenshot — capture JPEG de la session ───────────
-app.get('/api/sessions/:id/screenshot', async (req, res) => {
-    const { id } = req.params;
-    if (!SESSION_ID_RE.test(id)) {
-        res.status(400).json({ success: false, error: 'Invalid session ID' });
+// ─── Historique ───────────────────────────────────────────────────────────────
+app.get('/api/events', (req, res) => {
+    const siteId = typeof req.query.site === 'string' ? req.query.site : undefined;
+    if (siteId && !checkId(res, siteId, SITE_ID_RE, 'site'))
         return;
+    const limit = typeof req.query.limit === 'string' ? parseInt(req.query.limit, 10) : undefined;
+    void forward(res, 'EVENTS_LIST', { siteId, limit });
+});
+// ─── Connexion assistée ───────────────────────────────────────────────────────
+app.post('/api/login', (req, res) => {
+    const siteId = String(req.body.siteId ?? '');
+    if (checkId(res, siteId, SITE_ID_RE, 'site'))
+        void forward(res, 'LOGIN_START', { siteId }, 30_000);
+});
+app.get('/api/login', (_req, res) => forward(res, 'LOGIN_STATUS'));
+app.get('/api/login/screenshot', (_req, res) => sendJpeg(res, 'LOGIN_SCREENSHOT'));
+app.post('/api/login/confirm', (_req, res) => forward(res, 'LOGIN_CONFIRM', undefined, 15_000));
+app.post('/api/login/cancel', (_req, res) => forward(res, 'LOGIN_CANCEL', undefined, 15_000));
+// ─── Sessions ─────────────────────────────────────────────────────────────────
+app.get('/api/sessions', (_req, res) => forward(res, 'SESSION_LIST'));
+app.delete('/api/sessions', (_req, res) => forward(res, 'SESSION_CLOSE_ALL', undefined, 15_000));
+app.get('/api/sessions/:id/screenshot', (req, res) => {
+    if (checkId(res, req.params.id, SESSION_ID_RE, 'session')) {
+        void sendJpeg(res, 'SESSION_SCREENSHOT', { sessionId: req.params.id });
     }
-    try {
-        const data = await callMaster('SESSION_SCREENSHOT', { sessionId: id });
-        res.type('image/jpeg').send(Buffer.from(data.base64, 'base64'));
+});
+app.get('/api/sessions/:id/logs', (req, res) => {
+    if (checkId(res, req.params.id, SESSION_ID_RE, 'session')) {
+        void forward(res, 'SESSION_LOGS', { sessionId: req.params.id });
     }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        res.status(500).json({ success: false, error: message });
+});
+app.delete('/api/sessions/:id', (req, res) => {
+    if (checkId(res, req.params.id, SESSION_ID_RE, 'session')) {
+        void forward(res, 'SESSION_CLOSE', { sessionId: req.params.id });
     }
 });
 async function startServer() {