@imenam/simple-scraper 1.0.23 → 1.0.27

package/dist/sessions.js

@@ -1,18 +1,11 @@
 import { getBrowser, getDefaultViewport } from './browser.js';
-import { loadAllCookies } from './cookies.js';
+import { scheduleJarSync } from './cookie-sync.js';
 const MAX_SESSIONS = parseInt(process.env.SCRAPER_MAX_SESSIONS ?? '5', 10);
 const TTL_MS = parseInt(process.env.SCRAPER_SESSION_TTL_MS ?? String(10 * 60 * 1000), 10);
 const sessions = new Map();
 function generateSessionId() {
     return 'sess_' + Math.random().toString(36).slice(2, 8);
 }
-async function applyCookies(page, pageUrl) {
-    const cookies = await loadAllCookies();
-    if (!cookies.length)
-        return;
-    const prepared = cookies.map(c => (c.domain ? c : { ...c, url: pageUrl }));
-    await page.setCookie(...prepared);
-}
 /** Merge a partial override onto the env-configured default viewport. */
 export function resolveViewport(override) {
     const base = getDefaultViewport();
@@ -31,7 +24,13 @@ export async function createSession(url, opts = {}) {
     const sessionId = generateSessionId();
     const browser = await getBrowser();
     const page = await browser.newPage();
-    const entry = { page, createdAt: Date.now(), lastUsedAt: Date.now(), consoleLogs: [] };
+    const entry = {
+        page,
+        createdAt: Date.now(),
+        lastUsedAt: Date.now(),
+        lastAction: 'open_session',
+        consoleLogs: [],
+    };
     page.on('console', (msg) => {
         entry.consoleLogs.push({ type: msg.type(), text: msg.text(), timestamp: Date.now() });
     });
@@ -39,7 +38,6 @@ export async function createSession(url, opts = {}) {
         if (opts.viewport) {
             await page.setViewport(resolveViewport(opts.viewport));
         }
-        await applyCookies(page, url);
         await page.goto(url, { waitUntil: 'networkidle2', timeout: opts.timeout ?? 30_000 });
         if (opts.waitFor) {
             await page.waitForSelector(opts.waitFor, { timeout: opts.timeout ?? 30_000 });
@@ -53,7 +51,8 @@ export async function createSession(url, opts = {}) {
     console.error(`[Session] Created ${sessionId} → ${url}`);
     return sessionId;
 }
-export function getSession(sessionId) {
+/** Page of the session; `action` is recorded as the last action performed on it. */
+export function getSession(sessionId, action) {
     const entry = sessions.get(sessionId);
     if (!entry) {
         throw new Error(`Session "${sessionId}" not found or already closed.`);
@@ -63,6 +62,15 @@ export function getSession(sessionId) {
         throw new Error(`Session "${sessionId}" page was closed unexpectedly.`);
     }
     entry.lastUsedAt = Date.now();
+    entry.lastAction = action;
+    return entry.page;
+}
+/** Page of the session, without marking it as used. */
+export function peekSession(sessionId) {
+    const entry = sessions.get(sessionId);
+    if (!entry || entry.page.isClosed()) {
+        throw new Error(`Session "${sessionId}" not found or already closed.`);
+    }
     return entry.page;
 }
 export async function closeSession(sessionId) {
@@ -71,6 +79,7 @@ export async function closeSession(sessionId) {
         return;
     await entry.page.close().catch(() => undefined);
     sessions.delete(sessionId);
+    scheduleJarSync();
     console.error(`[Session] Closed ${sessionId}`);
 }
 export async function closeAllSessions() {
@@ -97,6 +106,9 @@ export function listSessions() {
         sessionId,
         createdAt: s.createdAt,
         lastUsedAt: s.lastUsedAt,
+        lastAction: s.lastAction,
+        url: s.page.isClosed() ? '' : s.page.url(),
+        errorCount: s.consoleLogs.filter(l => l.type === 'error').length,
     }));
 }
 // GC: close sessions inactive for more than TTL_MS

package/dist/site-check.js

@@ -0,0 +1,88 @@
+import { getBrowser, getDefaultTimeout } from './browser.js';
+import { syncJarsFromBrowser } from './cookie-sync.js';
+import { requireStore } from './site-store.js';
+const SELECTOR_WAIT_MS = 5_000;
+/** L'adresse, sans paramètres ni ancre, avec ou sans son port, contient la valeur attendue. */
+export function urlMatchesSignal(url, value) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return false;
+    }
+    const expected = value.replace(/^https?:\/\//, '');
+    return [`${parsed.host}${parsed.pathname}`, `${parsed.hostname}${parsed.pathname}`].some(bare => bare.includes(expected));
+}
+/** La page montre le signe de connexion du site. */
+export async function pageShowsSignal(page, signal, waitMs = 0) {
+    if (signal.kind === 'url')
+        return urlMatchesSignal(page.url(), signal.value);
+    if (waitMs > 0) {
+        return page
+            .waitForSelector(signal.value, { timeout: waitMs })
+            .then(el => el !== null)
+            .catch(() => false);
+    }
+    return (await page.$(signal.value)) !== null;
+}
+function describePath(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.pathname + parsed.search;
+    }
+    catch {
+        return url;
+    }
+}
+const running = new Set();
+export function isChecking(siteId) {
+    return running.has(siteId);
+}
+/**
+ * Visite la page de vérification du site avec ses cookies et met à jour son
+ * état : connecté si le signe de connexion apparaît, déconnecté sinon. Une
+ * page injoignable laisse l'état inchangé et enregistre l'erreur.
+ */
+export async function checkSite(siteId) {
+    const store = requireStore();
+    const site = store.get(siteId);
+    if (!site.keepAlive.pageUrl) {
+        throw new Error(`Le site "${site.name}" n'a pas de page de vérification.`);
+    }
+    if (running.has(siteId))
+        return site;
+    running.add(siteId);
+    let page = null;
+    try {
+        page = await (await getBrowser()).newPage();
+        await page.goto(site.keepAlive.pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
+        const connected = await pageShowsSignal(page, site.signal, SELECTOR_WAIT_MS);
+        const now = Date.now();
+        if (connected) {
+            store.setState(siteId, { status: 'connected', lostAt: null, lastCheckAt: now, lastCheckError: null });
+            store.addEvent(siteId, 'check-ok', `visite de ${describePath(site.keepAlive.pageUrl)}`);
+        }
+        else {
+            const wasConnected = site.state.status !== 'disconnected';
+            store.setState(siteId, {
+                status: 'disconnected',
+                lostAt: wasConnected ? now : site.state.lostAt,
+                lastCheckAt: now,
+                lastCheckError: null,
+            });
+            store.addEvent(siteId, 'check-lost', `redirigé vers ${describePath(page.url())}`);
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        store.setState(siteId, { lastCheckAt: Date.now(), lastCheckError: message });
+        store.addEvent(siteId, 'check-error', message);
+    }
+    finally {
+        await page?.close().catch(() => undefined);
+        running.delete(siteId);
+    }
+    await syncJarsFromBrowser();
+    return store.get(siteId);
+}

package/dist/site-store.js

@@ -0,0 +1,210 @@
+import fs from 'fs';
+import path from 'path';
+import { getDataDir } from './data-dir.js';
+const MAX_EVENTS = 1000;
+/** « 1 cookie enregistré », « 18 cookies enregistrés ». */
+export function cookieCountLabel(count, participle) {
+    return count > 1 ? `${count} cookies ${participle}s` : `${count} cookie ${participle}`;
+}
+const SITE_ID_RE = /^site_[a-z0-9]{6}$/;
+export function isSiteId(value) {
+    return SITE_ID_RE.test(value);
+}
+/** L'hôte appartient au domaine, ou à l'un de ses sous-domaines. */
+export function hostMatches(host, domain) {
+    return host === domain || host.endsWith('.' + domain);
+}
+/** Le cookie s'applique à l'un des domaines, ou l'un des domaines l'englobe. */
+export function cookieInScope(cookieDomain, scope) {
+    const d = cookieDomain.replace(/^\./, '').toLowerCase();
+    return scope.some(s => s === d || s.endsWith('.' + d) || d.endsWith('.' + s));
+}
+/** Cookie réduit aux champs acceptés par `setCookie`, sans date pour un cookie de session. */
+export function toCookieData(cookie) {
+    const data = {
+        name: cookie.name,
+        value: cookie.value,
+        domain: cookie.domain,
+        path: cookie.path ?? '/',
+        secure: cookie.secure ?? false,
+        httpOnly: cookie.httpOnly ?? false,
+    };
+    if (cookie.sameSite)
+        data.sameSite = cookie.sameSite;
+    if (typeof cookie.expires === 'number' && cookie.expires > 0)
+        data.expires = cookie.expires;
+    return data;
+}
+function isExpired(cookie, nowSec) {
+    return typeof cookie.expires === 'number' && cookie.expires > 0 && cookie.expires < nowSec;
+}
+function cookieKey(c) {
+    return `${c.domain}\t${c.path ?? '/'}\t${c.name}`;
+}
+function writeJson(file, value) {
+    const tmp = `${file}.tmp`;
+    fs.writeFileSync(tmp, JSON.stringify(value, null, 2), 'utf-8');
+    fs.renameSync(tmp, file);
+}
+function readJson(file, empty) {
+    if (!fs.existsSync(file))
+        return empty;
+    return JSON.parse(fs.readFileSync(file, 'utf-8'));
+}
+function generateSiteId() {
+    return 'site_' + Math.random().toString(36).slice(2, 8).padEnd(6, '0');
+}
+/**
+ * Sites d'un dossier de données : `sites.json` (réglages et état), `events.json`
+ * (historique borné) et `cookies/<id>.json` (cookies de chaque site). Le process
+ * principal en est le seul lecteur et le seul écrivain ; tout est gardé en mémoire
+ * et réécrit à chaque modification.
+ */
+export class SiteStore {
+    dir;
+    sitesFile;
+    eventsFile;
+    cookiesDir;
+    sites;
+    events;
+    jars = new Map();
+    jarUpdatedAt = new Map();
+    constructor(dir) {
+        this.dir = dir;
+        this.sitesFile = path.join(dir, 'sites.json');
+        this.eventsFile = path.join(dir, 'events.json');
+        this.cookiesDir = path.join(dir, 'cookies');
+        fs.mkdirSync(this.cookiesDir, { recursive: true });
+        this.sites = readJson(this.sitesFile, { sites: [] }).sites;
+        this.events = readJson(this.eventsFile, []);
+        for (const site of this.sites) {
+            const file = this.jarFile(site.id);
+            this.jars.set(site.id, readJson(file, []));
+            if (fs.existsSync(file))
+                this.jarUpdatedAt.set(site.id, fs.statSync(file).mtimeMs);
+        }
+    }
+    jarFile(id) {
+        return path.join(this.cookiesDir, `${id}.json`);
+    }
+    saveSites() {
+        writeJson(this.sitesFile, { sites: this.sites });
+    }
+    list() {
+        return this.sites;
+    }
+    get(id) {
+        const site = this.sites.find(s => s.id === id);
+        if (!site)
+            throw new Error(`Site "${id}" introuvable.`);
+        return site;
+    }
+    /** Premier site dont l'un des domaines couvre l'hôte de l'URL. */
+    siteForUrl(url) {
+        let host;
+        try {
+            host = new URL(url).hostname.toLowerCase();
+        }
+        catch {
+            return null;
+        }
+        return this.sites.find(s => s.domains.some(d => hostMatches(host, d))) ?? null;
+    }
+    create(config) {
+        let id = generateSiteId();
+        while (this.sites.some(s => s.id === id))
+            id = generateSiteId();
+        const site = {
+            ...config,
+            id,
+            state: { status: 'unknown', sessionOpenedAt: null, lostAt: null, lastCheckAt: null, lastCheckError: null },
+        };
+        this.sites.push(site);
+        this.jars.set(id, []);
+        this.saveSites();
+        return site;
+    }
+    update(id, config) {
+        const site = this.get(id);
+        Object.assign(site, config);
+        this.saveSites();
+        return site;
+    }
+    remove(id) {
+        this.get(id);
+        this.sites = this.sites.filter(s => s.id !== id);
+        this.jars.delete(id);
+        this.jarUpdatedAt.delete(id);
+        fs.rmSync(this.jarFile(id), { force: true });
+        this.events = this.events.filter(e => e.siteId !== id);
+        this.saveSites();
+        writeJson(this.eventsFile, this.events);
+    }
+    setState(id, patch) {
+        const site = this.get(id);
+        site.state = { ...site.state, ...patch };
+        this.saveSites();
+        return site;
+    }
+    addEvent(siteId, kind, detail) {
+        this.events.push({ siteId, at: Date.now(), kind, detail });
+        if (this.events.length > MAX_EVENTS)
+            this.events.splice(0, this.events.length - MAX_EVENTS);
+        writeJson(this.eventsFile, this.events);
+    }
+    /** Événements du plus récent au plus ancien, éventuellement limités à un site. */
+    listEvents(opts = {}) {
+        const filtered = opts.siteId ? this.events.filter(e => e.siteId === opts.siteId) : this.events;
+        return filtered.slice().reverse().slice(0, opts.limit ?? 200);
+    }
+    /** Cookies encore valides du site. */
+    getJar(id) {
+        this.get(id);
+        const now = Date.now() / 1000;
+        return (this.jars.get(id) ?? []).filter(c => !isExpired(c, now));
+    }
+    jarUpdated(id) {
+        return this.jarUpdatedAt.get(id) ?? null;
+    }
+    /** Cookies encore valides de tous les sites. */
+    allCookies() {
+        return this.sites.flatMap(s => this.getJar(s.id));
+    }
+    /** Domaines dont le site garde les cookies : les siens, plus ceux déjà présents dans ses cookies. */
+    cookieScope(id) {
+        const site = this.get(id);
+        const jarDomains = (this.jars.get(id) ?? []).map(c => c.domain.replace(/^\./, '').toLowerCase());
+        return Array.from(new Set([...site.domains, ...jarDomains]));
+    }
+    /** Remplace les cookies du site. Retourne false quand ils sont identiques à ceux déjà enregistrés. */
+    writeJar(id, cookies) {
+        this.get(id);
+        const next = cookies.map(toCookieData).sort((a, b) => cookieKey(a).localeCompare(cookieKey(b)));
+        const current = this.jars.get(id) ?? [];
+        if (JSON.stringify(current) === JSON.stringify(next))
+            return false;
+        this.jars.set(id, next);
+        writeJson(this.jarFile(id), next);
+        this.jarUpdatedAt.set(id, Date.now());
+        return true;
+    }
+}
+let store = null;
+/** Store du dossier de données courant, ou null si aucun dossier n'est configuré. */
+export function getStore() {
+    const dir = getDataDir();
+    if (!dir) {
+        store = null;
+        return null;
+    }
+    if (!store || store.dir !== dir)
+        store = new SiteStore(dir);
+    return store;
+}
+/** Store du dossier de données courant ; échoue si aucun dossier n'est configuré. */
+export function requireStore() {
+    const s = getStore();
+    if (!s)
+        throw new Error('SCRAPER_DATA_DIR (ou COOKIES_DIR) n\'est pas configuré : les sites sont désactivés.');
+    return s;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@imenam/simple-scraper",
-  "version": "1.0.23",
+  "version": "1.0.27",
   "description": "MCP server for web scraping and JavaScript execution using Puppeteer",
   "type": "module",
   "main": "dist/index.js",
@@ -52,6 +52,7 @@
     "@types/react-dom": "^19.2.3",
     "@vitejs/plugin-react": "^6.0.1",
     "concurrently": "^9.2.1",
+    "lucide-react": "^0.468.0",
     "react": "^19.2.4",
     "react-dom": "^19.2.4",
     "tsx": "^4.21.0",