@imenam/simple-scraper 1.0.24 → 1.0.28

package/dist/rpc.js

@@ -0,0 +1,184 @@
+import { z } from 'zod';
+import { clearSiteCookies, replaceSiteCookies } from './cookie-sync.js';
+import { parseNetscapeCookies } from './cookies.js';
+import { getDataDir } from './data-dir.js';
+import { nextKeepAliveAt, nextVisitAt } from './keep-alive.js';
+import { cancelLogin, confirmLogin, getLoginStatus, loginInput, loginScreenshot, startLogin } from './login.js';
+import { parseRemoteAction } from './remote-input.js';
+import { closePicker, openPicker, pickAt, pickerScreenshot, pickerScroll, testSignal } from './signal-picker.js';
+import { checkSite, isChecking } from './site-check.js';
+import { cookieCountLabel, getStore, requireStore } from './site-store.js';
+import { closeAllSessions, closeSession, getSessionLogs, listSessions, peekSession } from './sessions.js';
+const signalSchema = z.object({ kind: z.enum(['url', 'selector']), value: z.string().trim() });
+const siteConfigSchema = z.object({
+    name: z.string().trim().min(1, 'Le nom est requis.'),
+    domains: z
+        .array(z.string().trim().toLowerCase().regex(/^[a-z0-9.-]+$/, 'Domaine invalide.'))
+        .min(1, 'Au moins un domaine est requis.'),
+    loginUrl: z.union([z.literal(''), z.url('Page de connexion invalide.')]),
+    signal: signalSchema,
+    keepAlive: z.object({
+        enabled: z.boolean(),
+        intervalMin: z.number().int().min(1).max(24 * 60),
+        pageUrl: z.union([z.literal(''), z.url('Page visitée invalide.')]),
+    }),
+    flagWhenLost: z.boolean(),
+});
+/** Valide une donnée de la GUI ; l'erreur reprend les messages du schéma. */
+function parse(schema, data) {
+    const result = schema.safeParse(data);
+    if (!result.success)
+        throw new Error(result.error.issues.map(i => i.message).join(' '));
+    return result.data;
+}
+const pageUrlSchema = z.url('Page visitée invalide.');
+const ratioSchema = z.number().min(0).max(1);
+function parseSiteConfig(data) {
+    return parse(siteConfigSchema, data);
+}
+/** Site enrichi de ce que l'interface affiche à côté de ses réglages. */
+function siteView(site) {
+    const store = requireStore();
+    return {
+        ...site,
+        cookieCount: store.getJar(site.id).length,
+        jarUpdatedAt: store.jarUpdated(site.id),
+        checking: isChecking(site.id),
+        nextVisitAt: nextVisitAt(site),
+    };
+}
+function sessionViews() {
+    const store = getStore();
+    return listSessions().map(s => {
+        const site = store?.siteForUrl(s.url) ?? null;
+        return { ...s, siteId: site?.id ?? null, siteName: site?.name ?? null };
+    });
+}
+/** Lance la vérification sans l'attendre : l'interface suit son avancement par l'état du site. */
+function startCheck(siteId) {
+    checkSite(siteId).catch(err => console.error(`[Sites] Vérification de ${siteId} échouée : ${err instanceof Error ? err.message : String(err)}`));
+}
+function status() {
+    let dataDir = null;
+    let error = null;
+    try {
+        dataDir = getDataDir();
+    }
+    catch (err) {
+        error = err instanceof Error ? err.message : String(err);
+    }
+    const sites = dataDir ? requireStore().list() : [];
+    return {
+        dataDir,
+        error,
+        lostCount: sites.filter(s => s.state.status === 'disconnected' && s.flagWhenLost).length,
+        sessionCount: listSessions().length,
+        keepAlive: {
+            maintainedCount: sites.filter(s => nextVisitAt(s) !== null).length,
+            nextAt: dataDir ? nextKeepAliveAt() : null,
+        },
+    };
+}
+const handlers = {
+    STATUS: () => status(),
+    SITES_LIST: () => requireStore().list().map(siteView),
+    SITE_GET: ({ siteId }) => siteView(requireStore().get(siteId)),
+    SITE_CREATE: ({ config }) => {
+        const site = requireStore().create(parseSiteConfig(config));
+        if (site.keepAlive.pageUrl)
+            startCheck(site.id);
+        return siteView(site);
+    },
+    SITE_UPDATE: ({ siteId, config }) => siteView(requireStore().update(siteId, parseSiteConfig(config))),
+    SITE_DELETE: async ({ siteId }) => {
+        await clearSiteCookies(siteId);
+        requireStore().remove(siteId);
+        return null;
+    },
+    SITE_CHECK: ({ siteId }) => {
+        const site = requireStore().get(siteId);
+        if (!site.keepAlive.pageUrl)
+            throw new Error(`Le site "${site.name}" n'a pas de page de vérification.`);
+        startCheck(siteId);
+        return null;
+    },
+    SITE_IMPORT_COOKIES: async ({ siteId, content }) => {
+        const cookies = parseNetscapeCookies(String(content ?? ''));
+        if (!cookies.length)
+            throw new Error('Aucun cookie au format Netscape dans ce fichier.');
+        const count = await replaceSiteCookies(siteId, cookies);
+        const store = requireStore();
+        store.setState(siteId, { status: 'unknown', sessionOpenedAt: Date.now(), lostAt: null, lastCheckError: null });
+        store.addEvent(siteId, 'import', cookieCountLabel(count, 'importé'));
+        if (store.get(siteId).keepAlive.pageUrl)
+            startCheck(siteId);
+        return siteView(store.get(siteId));
+    },
+    SITE_CLEAR_COOKIES: async ({ siteId }) => {
+        await clearSiteCookies(siteId);
+        const store = requireStore();
+        store.setState(siteId, { status: 'disconnected', sessionOpenedAt: null, lostAt: Date.now() });
+        store.addEvent(siteId, 'cleared', 'session effacée');
+        return siteView(store.get(siteId));
+    },
+    EVENTS_LIST: ({ siteId, limit }) => requireStore().listEvents({ siteId, limit }),
+    LOGIN_START: ({ siteId }) => startLogin(siteId),
+    LOGIN_STATUS: () => getLoginStatus(),
+    LOGIN_SCREENSHOT: async () => ({ base64: await loginScreenshot() }),
+    LOGIN_INPUT: async ({ action }) => {
+        await loginInput(parseRemoteAction(action));
+        return getLoginStatus();
+    },
+    LOGIN_CONFIRM: () => confirmLogin(),
+    LOGIN_CANCEL: async () => {
+        await cancelLogin();
+        return null;
+    },
+    PICKER_OPEN: ({ pageUrl }) => openPicker(parse(pageUrlSchema, pageUrl)),
+    PICKER_SCREENSHOT: async () => ({ base64: await pickerScreenshot() }),
+    PICKER_SCROLL: async ({ dy }) => {
+        await pickerScroll(parse(z.number().min(-5000).max(5000), dy));
+        return null;
+    },
+    PICKER_PICK: ({ x, y }) => pickAt(parse(ratioSchema, x), parse(ratioSchema, y)),
+    PICKER_CLOSE: async () => {
+        await closePicker();
+        return null;
+    },
+    SIGNAL_TEST: ({ pageUrl, signal }) => testSignal(parse(pageUrlSchema, pageUrl), parse(signalSchema, signal)),
+    SESSION_LIST: () => sessionViews(),
+    SESSION_SCREENSHOT: async ({ sessionId }) => {
+        const buffer = await peekSession(sessionId).screenshot({ type: 'jpeg', quality: 60 });
+        return { base64: Buffer.from(buffer).toString('base64') };
+    },
+    SESSION_LOGS: ({ sessionId }) => getSessionLogs(sessionId),
+    SESSION_CLOSE: async ({ sessionId }) => {
+        await closeSession(sessionId);
+        return null;
+    },
+    SESSION_CLOSE_ALL: async () => {
+        await closeAllSessions();
+        return null;
+    },
+};
+/** Répond aux requêtes de la GUI, qui n'a accès ni au navigateur ni aux sites. */
+export function registerRpc(launcher) {
+    launcher.getIpcHub().onMessage(async (msg) => {
+        const handler = handlers[msg.type];
+        if (!handler)
+            return;
+        const reply = (data, error) => launcher.getIpcHub().send({
+            type: `${msg.type}_RESPONSE`,
+            correlationId: msg.correlationId,
+            data,
+            error,
+            timestamp: new Date().toISOString(),
+        });
+        try {
+            reply(await handler(msg.data ?? {}));
+        }
+        catch (err) {
+            reply(undefined, err instanceof Error ? err.message : String(err));
+        }
+    });
+}

package/dist/sessions.js

@@ -1,18 +1,11 @@
 import { getBrowser, getDefaultViewport } from './browser.js';
-import { loadAllCookies } from './cookies.js';
+import { scheduleJarSync } from './cookie-sync.js';
 const MAX_SESSIONS = parseInt(process.env.SCRAPER_MAX_SESSIONS ?? '5', 10);
 const TTL_MS = parseInt(process.env.SCRAPER_SESSION_TTL_MS ?? String(10 * 60 * 1000), 10);
 const sessions = new Map();
 function generateSessionId() {
     return 'sess_' + Math.random().toString(36).slice(2, 8);
 }
-async function applyCookies(page, pageUrl) {
-    const cookies = await loadAllCookies();
-    if (!cookies.length)
-        return;
-    const prepared = cookies.map(c => (c.domain ? c : { ...c, url: pageUrl }));
-    await page.setCookie(...prepared);
-}
 /** Merge a partial override onto the env-configured default viewport. */
 export function resolveViewport(override) {
     const base = getDefaultViewport();
@@ -31,7 +24,13 @@ export async function createSession(url, opts = {}) {
     const sessionId = generateSessionId();
     const browser = await getBrowser();
     const page = await browser.newPage();
-    const entry = { page, createdAt: Date.now(), lastUsedAt: Date.now(), consoleLogs: [] };
+    const entry = {
+        page,
+        createdAt: Date.now(),
+        lastUsedAt: Date.now(),
+        lastAction: 'open_session',
+        consoleLogs: [],
+    };
     page.on('console', (msg) => {
         entry.consoleLogs.push({ type: msg.type(), text: msg.text(), timestamp: Date.now() });
     });
@@ -39,7 +38,6 @@ export async function createSession(url, opts = {}) {
         if (opts.viewport) {
             await page.setViewport(resolveViewport(opts.viewport));
         }
-        await applyCookies(page, url);
         await page.goto(url, { waitUntil: 'networkidle2', timeout: opts.timeout ?? 30_000 });
         if (opts.waitFor) {
             await page.waitForSelector(opts.waitFor, { timeout: opts.timeout ?? 30_000 });
@@ -53,7 +51,8 @@ export async function createSession(url, opts = {}) {
     console.error(`[Session] Created ${sessionId} → ${url}`);
     return sessionId;
 }
-export function getSession(sessionId) {
+/** Page of the session; `action` is recorded as the last action performed on it. */
+export function getSession(sessionId, action) {
     const entry = sessions.get(sessionId);
     if (!entry) {
         throw new Error(`Session "${sessionId}" not found or already closed.`);
@@ -63,6 +62,15 @@ export function getSession(sessionId) {
         throw new Error(`Session "${sessionId}" page was closed unexpectedly.`);
     }
     entry.lastUsedAt = Date.now();
+    entry.lastAction = action;
+    return entry.page;
+}
+/** Page of the session, without marking it as used. */
+export function peekSession(sessionId) {
+    const entry = sessions.get(sessionId);
+    if (!entry || entry.page.isClosed()) {
+        throw new Error(`Session "${sessionId}" not found or already closed.`);
+    }
     return entry.page;
 }
 export async function closeSession(sessionId) {
@@ -71,6 +79,7 @@ export async function closeSession(sessionId) {
         return;
     await entry.page.close().catch(() => undefined);
     sessions.delete(sessionId);
+    scheduleJarSync();
     console.error(`[Session] Closed ${sessionId}`);
 }
 export async function closeAllSessions() {
@@ -97,6 +106,9 @@ export function listSessions() {
         sessionId,
         createdAt: s.createdAt,
         lastUsedAt: s.lastUsedAt,
+        lastAction: s.lastAction,
+        url: s.page.isClosed() ? '' : s.page.url(),
+        errorCount: s.consoleLogs.filter(l => l.type === 'error').length,
     }));
 }
 // GC: close sessions inactive for more than TTL_MS

package/dist/signal-picker.js

@@ -0,0 +1,144 @@
+import { getBrowser, getDefaultTimeout } from './browser.js';
+import { applyRemoteAction, pointOf } from './remote-input.js';
+import { pageShowsSignal, urlMatchesSignal } from './site-check.js';
+const SIGNAL_WAIT_MS = 3_000;
+let picker = null;
+function requirePicker() {
+    if (!picker || picker.page.isClosed())
+        throw new Error('Aucune page ouverte pour choisir le signe de connexion.');
+    return picker;
+}
+/** Ouvre la page visitée du site dans le navigateur partagé, donc avec les cookies des sites. */
+export async function openPicker(pageUrl) {
+    await closePicker();
+    const page = await (await getBrowser()).newPage();
+    picker = { page, pageUrl };
+    await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
+    return { url: page.url() };
+}
+export async function closePicker() {
+    const current = picker;
+    picker = null;
+    await current?.page.close().catch(() => undefined);
+}
+export async function pickerScreenshot() {
+    const buffer = await requirePicker().page.screenshot({ type: 'jpeg', quality: 70 });
+    return Buffer.from(buffer).toString('base64');
+}
+/** Fait défiler la page du sélecteur ; seul le défilement y est relayé. */
+export async function pickerScroll(dy) {
+    const action = { type: 'scroll', dy };
+    await applyRemoteAction(requirePicker().page, action);
+}
+/**
+ * Sélecteurs désignant l'élément situé au point donné, du plus précis au plus
+ * général : son texte dans son conteneur repérable, son chemin depuis ce
+ * conteneur, le conteneur seul, l'élément seul.
+ */
+function candidatesAt(x, y) {
+    const target = document.elementFromPoint(x, y);
+    if (!target)
+        return [];
+    const stable = (name) => /^[a-zA-Z_][\w-]*$/.test(name) &&
+        !/\d{3,}/.test(name) &&
+        !/^(active|open|show|hover|focus|focused|in|collapsed|selected|disabled|is-|has-|ng-)/.test(name);
+    const count = (selector) => document.querySelectorAll(selector).length;
+    /** Classe stable de l'élément la plus rare dans la page ; à égalité, la plus longue, souvent la plus parlante. */
+    const ownClass = (el) => Array.from(el.classList)
+        .filter(stable)
+        .sort((a, b) => count(`.${CSS.escape(a)}`) - count(`.${CSS.escape(b)}`) || b.length - a.length)[0];
+    const selfSel = (el) => {
+        if (el.id && stable(el.id))
+            return `#${CSS.escape(el.id)}`;
+        const cls = ownClass(el);
+        return cls ? `.${CSS.escape(cls)}` : null;
+    };
+    const part = (el) => {
+        if (el.id && stable(el.id))
+            return `#${CSS.escape(el.id)}`;
+        const cls = ownClass(el);
+        return el.tagName.toLowerCase() + (cls ? `.${CSS.escape(cls)}` : '');
+    };
+    // Conteneur : l'ancêtre repérable le plus proche qui soit unique dans la page,
+    // sinon le plus proche repérable.
+    let anchorSel = null;
+    let firstSel = null;
+    for (let el = target.parentElement, depth = 0; el && depth < 6; el = el.parentElement, depth++) {
+        if (el === document.body || el === document.documentElement)
+            break;
+        const sel = selfSel(el);
+        if (!sel)
+            continue;
+        firstSel ??= sel;
+        if (count(sel) === 1) {
+            anchorSel = sel;
+            break;
+        }
+    }
+    anchorSel ??= firstSel;
+    const targetSel = part(target);
+    const scoped = anchorSel ? `${anchorSel} ${targetSel}` : targetSel;
+    const out = [];
+    const text = (target.textContent ?? '').replace(/\s+/g, ' ').trim();
+    if (text && text.length <= 60 && target.children.length === 0) {
+        const quoted = JSON.stringify(text);
+        out.push({ selector: `${scoped}::-p-text(${quoted})`, label: `Le texte « ${text} » à cet endroit` });
+    }
+    if (anchorSel)
+        out.push({ selector: scoped, label: 'Cet élément, dans son conteneur' });
+    if (anchorSel)
+        out.push({ selector: anchorSel, label: 'Son conteneur' });
+    out.push({ selector: targetSel, label: 'Cet élément, n\'importe où dans la page' });
+    return out.filter((c, i) => out.findIndex(o => o.selector === c.selector) === i);
+}
+/** Ouvre la page dans un contexte sans cookies, comme la verrait une session perdue. */
+async function withAnonymousPage(pageUrl, fn) {
+    const context = await (await getBrowser()).createBrowserContext();
+    try {
+        const page = await context.newPage();
+        await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
+        return await fn(page);
+    }
+    finally {
+        await context.close().catch(() => undefined);
+    }
+}
+async function countMatches(page, selector) {
+    return (await page.$$(selector).catch(() => [])).length;
+}
+/**
+ * Propose les sélecteurs de l'élément cliqué, chacun compté sur la page vue
+ * connectée et sur la même page vue sans cookies. Un bon signe de connexion est
+ * trouvé dans le premier cas et absent dans le second ; ceux-là viennent en tête.
+ */
+export async function pickAt(x, y) {
+    const { page, pageUrl } = requirePicker();
+    const point = pointOf(page, x, y);
+    const proposals = await page.evaluate(candidatesAt, point.x, point.y);
+    if (!proposals.length)
+        return [];
+    const connected = await Promise.all(proposals.map(p => countMatches(page, p.selector)));
+    const anonymous = await withAnonymousPage(pageUrl, anon => Promise.all(proposals.map(p => countMatches(anon, p.selector))));
+    const results = proposals.map((p, i) => ({ ...p, connected: connected[i], anonymous: anonymous[i] }));
+    const good = (c) => c.connected > 0 && c.anonymous === 0;
+    return [...results.filter(good), ...results.filter(c => !good(c))];
+}
+/** Vérifie le signe sur la page vue connectée (attendu : trouvé) et sans cookies (attendu : absent). */
+export async function testSignal(pageUrl, signal) {
+    const page = await (await getBrowser()).newPage();
+    let connected;
+    let connectedUrl;
+    try {
+        await page.goto(pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
+        connectedUrl = page.url();
+        connected = await pageShowsSignal(page, signal, SIGNAL_WAIT_MS);
+    }
+    finally {
+        await page.close().catch(() => undefined);
+    }
+    const anon = await withAnonymousPage(pageUrl, async (p) => ({
+        url: p.url(),
+        shown: signal.kind === 'url' ? urlMatchesSignal(p.url(), signal.value) : await pageShowsSignal(p, signal),
+    }));
+    return { connected, anonymous: anon.shown, connectedUrl, anonymousUrl: anon.url };
+}

package/dist/site-check.js

@@ -0,0 +1,88 @@
+import { getBrowser, getDefaultTimeout } from './browser.js';
+import { syncJarsFromBrowser } from './cookie-sync.js';
+import { requireStore } from './site-store.js';
+const SELECTOR_WAIT_MS = 5_000;
+/** L'adresse, sans paramètres ni ancre, avec ou sans son port, contient la valeur attendue. */
+export function urlMatchesSignal(url, value) {
+    let parsed;
+    try {
+        parsed = new URL(url);
+    }
+    catch {
+        return false;
+    }
+    const expected = value.replace(/^https?:\/\//, '');
+    return [`${parsed.host}${parsed.pathname}`, `${parsed.hostname}${parsed.pathname}`].some(bare => bare.includes(expected));
+}
+/** La page montre le signe de connexion du site. */
+export async function pageShowsSignal(page, signal, waitMs = 0) {
+    if (signal.kind === 'url')
+        return urlMatchesSignal(page.url(), signal.value);
+    if (waitMs > 0) {
+        return page
+            .waitForSelector(signal.value, { timeout: waitMs })
+            .then(el => el !== null)
+            .catch(() => false);
+    }
+    return (await page.$(signal.value)) !== null;
+}
+function describePath(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.pathname + parsed.search;
+    }
+    catch {
+        return url;
+    }
+}
+const running = new Set();
+export function isChecking(siteId) {
+    return running.has(siteId);
+}
+/**
+ * Visite la page de vérification du site avec ses cookies et met à jour son
+ * état : connecté si le signe de connexion apparaît, déconnecté sinon. Une
+ * page injoignable laisse l'état inchangé et enregistre l'erreur.
+ */
+export async function checkSite(siteId) {
+    const store = requireStore();
+    const site = store.get(siteId);
+    if (!site.keepAlive.pageUrl) {
+        throw new Error(`Le site "${site.name}" n'a pas de page de vérification.`);
+    }
+    if (running.has(siteId))
+        return site;
+    running.add(siteId);
+    let page = null;
+    try {
+        page = await (await getBrowser()).newPage();
+        await page.goto(site.keepAlive.pageUrl, { waitUntil: 'networkidle2', timeout: getDefaultTimeout() });
+        const connected = await pageShowsSignal(page, site.signal, SELECTOR_WAIT_MS);
+        const now = Date.now();
+        if (connected) {
+            store.setState(siteId, { status: 'connected', lostAt: null, lastCheckAt: now, lastCheckError: null });
+            store.addEvent(siteId, 'check-ok', `visite de ${describePath(site.keepAlive.pageUrl)}`);
+        }
+        else {
+            const wasConnected = site.state.status !== 'disconnected';
+            store.setState(siteId, {
+                status: 'disconnected',
+                lostAt: wasConnected ? now : site.state.lostAt,
+                lastCheckAt: now,
+                lastCheckError: null,
+            });
+            store.addEvent(siteId, 'check-lost', `redirigé vers ${describePath(page.url())}`);
+        }
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        store.setState(siteId, { lastCheckAt: Date.now(), lastCheckError: message });
+        store.addEvent(siteId, 'check-error', message);
+    }
+    finally {
+        await page?.close().catch(() => undefined);
+        running.delete(siteId);
+    }
+    await syncJarsFromBrowser();
+    return store.get(siteId);
+}