@aiguru/google-web-operations 0.6.4

package/packages/connectors/odoo/src/index.js

@@ -0,0 +1,206 @@
+const SEO_FIELDS = ['website_meta_title', 'website_meta_description', 'website_meta_keywords', 'website_meta_og_img'];
+const PATCH_TO_FIELD = {
+    title: 'website_meta_title', description: 'website_meta_description', keywords: 'website_meta_keywords', ogImage: 'website_meta_og_img',
+};
+/** Minimal JSON-RPC 2.0 client for Odoo's /jsonrpc endpoint. */
+export class OdooClient {
+    uid = null;
+    id = 0;
+    c;
+    fetchImpl;
+    constructor(c, fetchImpl = fetch) { this.c = c; this.fetchImpl = fetchImpl; }
+    async rpc(service, method, args) {
+        const res = await this.fetchImpl(new URL('/jsonrpc', this.c.url).toString(), {
+            method: 'POST', headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ jsonrpc: '2.0', id: ++this.id, method: 'call', params: { service, method, args } }),
+        });
+        if (!res.ok)
+            throw new Error(`Odoo HTTP ${res.status} at ${this.c.url}`);
+        const body = (await res.json());
+        if (body.error) {
+            const d = body.error.data;
+            throw new Error(`Odoo: ${d?.message ?? body.error.message}${d?.name ? ` (${d.name})` : ''}`);
+        }
+        return body.result;
+    }
+    async authenticate() {
+        if (this.uid)
+            return this.uid;
+        const uid = await this.rpc('common', 'authenticate', [this.c.db, this.c.login, this.c.apiKey, {}]);
+        if (!uid)
+            throw new Error(`Odoo: authentication failed for ${this.c.login} on ${this.c.db}. Check the API key and that the user is active.`);
+        return (this.uid = uid);
+    }
+    async execute(model, method, args = [], kwargs = {}) {
+        const uid = await this.authenticate();
+        return this.rpc('object', 'execute_kw', [this.c.db, uid, this.c.apiKey, model, method, args, kwargs]);
+    }
+    searchRead(model, domain, fields, kwargs = {}) {
+        return this.execute(model, 'search_read', [domain], { fields, ...kwargs });
+    }
+    version() { return this.rpc('common', 'version', []); }
+}
+// ---------------------------------------------------------------- JSON-LD in HTML / QWeb arch (pure)
+/** Marker attribute on the script element this system manages. */
+export const SCHEMA_MARK = 'data-gwo="schema"';
+const MANAGED_RE = /<script\b[^>]*\bdata-gwo\s*=\s*["']schema["'][^>]*>([\s\S]*?)<\/script>\s*/gi;
+/**
+ * Serialise blocks for embedding in a QWeb arch, which is parsed as XML:
+ * the characters XML cares about are written as JSON escapes, which every
+ * JSON parser reads back unchanged.
+ */
+export function serialiseSchema(blocks) {
+    const json = JSON.stringify(blocks.length === 1 ? blocks[0] : blocks);
+    return json.replace(/&/g, '\\u0026').replace(/</g, '\\u003c').replace(/>/g, '\\u003e');
+}
+/** The managed JSON-LD blocks in an arch or HTML fragment. Unparseable managed blocks are skipped. */
+export function extractSchema(html) {
+    const out = [];
+    for (const m of html.matchAll(MANAGED_RE)) {
+        try {
+            const parsed = JSON.parse((m[1] ?? '').trim());
+            out.push(...(Array.isArray(parsed) ? parsed : [parsed]));
+        }
+        catch { /* leave it out; verification will report the mismatch */ }
+    }
+    return out;
+}
+/**
+ * Replace the managed block. Removes every existing managed script; with
+ * blocks, inserts one script:
+ *   - in a page arch, before `<div id="wrap">` (outside the editor-owned
+ *     structure) or, failing that, before the QWeb root's closing `</t>`;
+ *   - in an HTML fragment (blog content), appended at the end.
+ * Other markup is untouched.
+ */
+export function injectSchema(html, blocks) {
+    const stripped = html.replace(MANAGED_RE, '');
+    if (!blocks.length)
+        return stripped;
+    const script = `<script type="application/ld+json" ${SCHEMA_MARK}>${serialiseSchema(blocks)}</script>\n`;
+    const wrap = stripped.search(/<div\b[^>]*\bid\s*=\s*["']wrap["']/i);
+    if (wrap >= 0)
+        return stripped.slice(0, wrap) + script + stripped.slice(wrap);
+    const root = stripped.match(/<\/t>\s*$/);
+    if (root && root.index !== undefined)
+        return stripped.slice(0, root.index) + script + stripped.slice(root.index);
+    return stripped.replace(/\s*$/, '\n') + script;
+}
+const str = (v) => (v ? String(v) : null);
+export class OdooAdapter {
+    kind = 'odoo';
+    client;
+    c;
+    siteId;
+    constructor(client, c, siteId) { this.client = client; this.c = c; this.siteId = siteId; }
+    /** Build from a site record: connection from write.odoo, credentials from the secrets file. */
+    static fromSite(site, fetchImpl) {
+        const cfg = site.write.odoo ?? {};
+        const sec = site._secrets?.['odoo'] ?? {};
+        const missing = [
+            !cfg.url && 'write.odoo.url', !cfg.db && 'write.odoo.db',
+            !sec.login && 'odoo.login (secrets)', !sec.api_key && 'odoo.api_key (secrets)',
+        ].filter(Boolean);
+        if (missing.length)
+            throw new Error(`Site ${site.site_id}: Odoo adapter needs ${missing.join(', ')}.`);
+        const conn = { url: cfg.url, db: cfg.db, login: sec.login, apiKey: sec.api_key, websiteId: cfg.website_id ?? null };
+        return new OdooAdapter(new OdooClient(conn, fetchImpl), conn, site.site_id);
+    }
+    describe() { return `Odoo ${this.c.url} db=${this.c.db} user=${this.c.login}${this.c.websiteId ? ` website_id=${this.c.websiteId}` : ''}`; }
+    async ping() {
+        const v = await this.client.version();
+        const uid = await this.client.authenticate();
+        return { ok: true, detail: `Odoo ${v.server_version ?? '?'}, authenticated as uid ${uid}` };
+    }
+    websiteDomain() {
+        return this.c.websiteId ? [['website_id', 'in', [this.c.websiteId, false]]] : [];
+    }
+    async listPages() {
+        const pages = await this.client.searchRead('website.page', this.websiteDomain(), ['url', 'name', 'is_published'], { limit: 500, order: 'url' });
+        const posts = await this.client.searchRead('blog.post', [], ['name', 'website_url', 'is_published'], { limit: 500 }).catch(() => []);
+        // Products exist only with the eCommerce app; a missing model is not an error.
+        const products = await this.client.searchRead('product.template', [['website_published', '=', true]], ['name', 'website_url', 'is_published'], { limit: 500 }).catch(() => []);
+        return [
+            ...pages.map((p) => ({ path: p.url, ref: `website.page:${p.id}`, name: p.name, published: p.is_published ?? null })),
+            ...posts.map((p) => ({ path: p.website_url, ref: `blog.post:${p.id}`, name: p.name ?? null, published: p.is_published ?? null })),
+            ...products.map((p) => ({ path: p.website_url, ref: `product.template:${p.id}`, name: p.name ?? null, published: p.is_published ?? null })),
+        ];
+    }
+    /** Resolve a path to (model, id). Blog and shop URLs end in "-<id>"; pages match website.page.url exactly. */
+    async locate(path) {
+        const clean = path.replace(/[?#].*$/, '').replace(/\/+$/, '') || '/';
+        const blog = clean.match(/^\/blog\/.*-(\d+)$/);
+        if (blog)
+            return { model: 'blog.post', id: Number(blog[1]) };
+        const product = clean.match(/^\/shop\/(?!category\/).*-(\d+)$/);
+        if (product)
+            return { model: 'product.template', id: Number(product[1]) };
+        const hits = await this.client.searchRead('website.page', [['url', '=', clean], ...this.websiteDomain()], ['id'], { limit: 1 });
+        return hits[0] ? { model: 'website.page', id: hits[0].id } : null;
+    }
+    async readPageSeo(path) {
+        const loc = await this.locate(path);
+        if (!loc)
+            return null;
+        const extra = loc.model === 'website.page' ? ['url', 'name', 'is_published', 'website_indexed'] : ['website_url', 'name', 'is_published'];
+        const [r] = await this.client.execute(loc.model, 'read', [[loc.id], [...SEO_FIELDS, ...extra]]);
+        if (!r)
+            return null;
+        return {
+            path: r.url ?? r.website_url ?? path, ref: `${loc.model}:${loc.id}`, name: r.name ?? null,
+            title: str(r.website_meta_title), description: str(r.website_meta_description),
+            keywords: str(r.website_meta_keywords), ogImage: str(r.website_meta_og_img),
+            indexed: r.website_indexed ?? null, published: r.is_published ?? null,
+        };
+    }
+    async writePageSeo(path, patch) {
+        const loc = await this.locate(path);
+        if (!loc)
+            throw new Error(`Odoo: no page at ${path}`);
+        const vals = {};
+        for (const [k, field] of Object.entries(PATCH_TO_FIELD)) {
+            if (patch[k] !== undefined)
+                vals[field] = patch[k] === null ? false : patch[k];
+        }
+        if (!Object.keys(vals).length)
+            return;
+        await this.client.execute(loc.model, 'write', [[loc.id], vals]);
+    }
+    /**
+     * Where a page's markup lives: the view arch for website.page, the HTML
+     * content for blog.post. Returns the model, record id, field and markup.
+     */
+    async markup(path) {
+        const loc = await this.locate(path);
+        if (!loc)
+            return null;
+        if (loc.model === 'blog.post') {
+            const [r] = await this.client.execute('blog.post', 'read', [[loc.id], ['content']]);
+            return r ? { model: 'blog.post', id: loc.id, field: 'content', html: r.content || '' } : null;
+        }
+        if (loc.model === 'product.template') {
+            // The product's website description is the HTML block the shop page renders; JSON-LD rides in it like in a blog post.
+            const [r] = await this.client.execute('product.template', 'read', [[loc.id], ['website_description']]);
+            return r ? { model: 'product.template', id: loc.id, field: 'website_description', html: r.website_description || '' } : null;
+        }
+        const [p] = await this.client.execute('website.page', 'read', [[loc.id], ['view_id']]);
+        const viewId = p?.view_id ? p.view_id[0] : null;
+        if (!viewId)
+            throw new Error(`Odoo: page ${path} has no view (website.page:${loc.id}.view_id is empty).`);
+        const [v] = await this.client.execute('ir.ui.view', 'read', [[viewId], ['arch']]);
+        return v ? { model: 'ir.ui.view', id: viewId, field: 'arch', html: v.arch || '' } : null;
+    }
+    async readPageSchema(path) {
+        const m = await this.markup(path);
+        return m ? extractSchema(m.html) : null;
+    }
+    async writePageSchema(path, blocks) {
+        const m = await this.markup(path);
+        if (!m)
+            throw new Error(`Odoo: no page at ${path}`);
+        const next = injectSchema(m.html, blocks);
+        if (next === m.html)
+            return;
+        await this.client.execute(m.model, 'write', [[m.id], { [m.field]: next }]);
+    }
+}

package/packages/connectors/repo/src/index.js

@@ -0,0 +1,94 @@
+/**
+ * Repository write adapter for sites AI Guru develops itself.
+ *
+ * Convention: the site keeps its SEO metadata in `gwo.seo.json` at the
+ * repository root, keyed by path, and its build reads that file (for Next.js,
+ * from generateMetadata). The adapter reads and writes that file in a local
+ * checkout declared as `write.repo.local_path`. Committing and opening a pull
+ * request is the GitHub connector's job; this adapter only changes the
+ * working tree, and the write loop verifies by reading the file back.
+ *
+ * Structured data: an entry's `schema` array holds the JSON-LD blocks the
+ * page should emit. The site's layout renders each as a
+ * <script type="application/ld+json"> element.
+ *
+ * Sites without the manifest are still `full` tier, but changes to them are
+ * proposals for a developer until the manifest is adopted.
+ */
+import { existsSync, readFileSync, writeFileSync, statSync } from 'node:fs';
+import path from 'node:path';
+export const MANIFEST = 'gwo.seo.json';
+export class RepoAdapter {
+    kind = 'repo';
+    file;
+    localPath;
+    siteId;
+    constructor(localPath, siteId) {
+        this.localPath = localPath;
+        this.siteId = siteId;
+        this.file = path.join(localPath, MANIFEST);
+    }
+    static fromSite(site) {
+        const lp = site.write?.repo?.local_path;
+        if (!lp)
+            throw new Error(`Site ${site.site_id}: repo adapter needs write.repo.local_path (a local checkout).`);
+        return new RepoAdapter(lp, site.site_id);
+    }
+    describe() { return `repository checkout ${this.localPath} (${MANIFEST})`; }
+    async ping() {
+        if (!existsSync(this.localPath) || !statSync(this.localPath).isDirectory())
+            throw new Error(`Repo path does not exist: ${this.localPath}`);
+        if (!existsSync(this.file)) {
+            throw new Error(`${MANIFEST} not found in ${this.localPath}. Adopt the manifest convention (see docs/SITE-ADAPTERS.md) or treat changes as developer proposals.`);
+        }
+        return { ok: true, detail: `${MANIFEST} with ${Object.keys(this.read()).length} entries` };
+    }
+    read() {
+        return existsSync(this.file) ? JSON.parse(readFileSync(this.file, 'utf8')) : {};
+    }
+    write(m) {
+        // Stable key order keeps diffs reviewable.
+        const sorted = Object.fromEntries(Object.keys(m).sort().map((k) => [k, m[k]]));
+        writeFileSync(this.file, JSON.stringify(sorted, null, 2) + '\n');
+    }
+    entry(p) {
+        const e = this.read()[p];
+        if (!e)
+            throw new Error(`${MANIFEST} has no entry for ${p}; add the page to the manifest first.`);
+        return e;
+    }
+    async listPages() {
+        return Object.entries(this.read()).map(([p, e]) => ({ path: p, ref: MANIFEST, name: e.name ?? null, published: e.published ?? null }));
+    }
+    async readPageSeo(p) {
+        const e = this.read()[p];
+        if (!e)
+            return null;
+        return { path: p, ref: MANIFEST, name: e.name ?? null, title: e.title ?? null, description: e.description ?? null,
+            keywords: e.keywords ?? null, ogImage: e.ogImage ?? null, indexed: e.indexed ?? null, published: e.published ?? null };
+    }
+    async writePageSeo(p, patch) {
+        const m = this.read();
+        const e = m[p];
+        if (!e)
+            throw new Error(`${MANIFEST} has no entry for ${p}; add the page to the manifest first.`);
+        for (const k of ['title', 'description', 'keywords', 'ogImage']) {
+            if (patch[k] !== undefined)
+                e[k] = patch[k];
+        }
+        this.write(m);
+    }
+    async readPageSchema(p) {
+        const e = this.read()[p];
+        return e ? (e.schema ?? []) : null;
+    }
+    async writePageSchema(p, blocks) {
+        const m = this.read();
+        const e = m[p] ?? this.entry(p);
+        if (blocks.length)
+            e.schema = blocks;
+        else
+            delete e.schema;
+        this.write(m);
+    }
+}

package/packages/connectors/resend/src/index.js

@@ -0,0 +1,30 @@
+/**
+ * Resend: delivery of run summaries by email.
+ * Configuration by environment only: RESEND_API_KEY, GWO_REPORT_FROM, GWO_REPORT_TO
+ * (comma-separated). Without a key, isConfigured() is false and callers print instead.
+ */
+export function isConfigured() {
+    return !!(process.env['RESEND_API_KEY']?.trim() && process.env['GWO_REPORT_FROM']?.trim() && process.env['GWO_REPORT_TO']?.trim());
+}
+export function describe() {
+    if (!isConfigured())
+        return 'Resend not configured (RESEND_API_KEY, GWO_REPORT_FROM, GWO_REPORT_TO)';
+    return `Resend from ${process.env['GWO_REPORT_FROM']} to ${process.env['GWO_REPORT_TO']}`;
+}
+/** Send one email. Returns the Resend message id. */
+export async function send(mail, fetchImpl = fetch) {
+    const key = process.env['RESEND_API_KEY']?.trim();
+    const from = process.env['GWO_REPORT_FROM']?.trim();
+    const to = mail.to ?? (process.env['GWO_REPORT_TO'] ?? '').split(',').map((s) => s.trim()).filter(Boolean);
+    if (!key || !from || !to.length)
+        throw new Error(describe());
+    const res = await fetchImpl('https://api.resend.com/emails', {
+        method: 'POST',
+        headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
+        body: JSON.stringify({ from, to, subject: mail.subject, text: mail.text, ...(mail.html ? { html: mail.html } : {}) }),
+    });
+    const body = await res.text();
+    if (!res.ok)
+        throw new Error(`Resend ${res.status}: ${body.slice(0, 400)}`);
+    return JSON.parse(body);
+}

package/packages/connectors/web-crawler/src/a11y.js

@@ -0,0 +1,91 @@
+/**
+ * Accessibility signals from static HTML, for the IS 5568 rule area.
+ *
+ * IS 5568 is anchored to WCAG 2.0 AA and adds Israeli requirements: an
+ * accessibility statement, Hebrew with RTL declared, a contact route. What a
+ * crawler can judge from markup alone is the structural half: labels, names,
+ * headings, landmarks, zoom, focus order, the statement link. Contrast and
+ * keyboard behaviour need a rendered page and stay out of scope here.
+ *
+ * Regex-based like the rest of the parser; counts are conservative (an
+ * element is only counted as a fault when no accessible-name mechanism is
+ * present at all).
+ */
+const attr = (tag, name) => {
+    const m = tag.match(new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'));
+    return m ? (m[2] ?? m[3] ?? m[4] ?? '') : null;
+};
+const text = (s) => s.replace(/<[^>]+>/g, ' ').replace(/&nbsp;/g, ' ').replace(/\s+/g, ' ').trim();
+const has = (tag, name) => attr(tag, name) !== null && (attr(tag, name) ?? '').trim() !== '';
+/** Link texts that say nothing about the destination. */
+const GENERIC = /^(click here|read more|more|here|learn more|link|לחץ כאן|לחצו כאן|קרא עוד|קראו עוד|עוד|כאן|לפרטים|המשך)$/i;
+export function parseA11y(html) {
+    const h = html || '';
+    const body = (h.match(/<body\b[^>]*>([\s\S]*)<\/body>/i)?.[1]) ?? h;
+    // Headings in order.
+    const headingOrder = [...body.matchAll(/<h([1-6])\b[^>]*>/gi)].map((m) => Number(m[1]));
+    let skippedLevels = 0;
+    for (let i = 1; i < headingOrder.length; i++)
+        if (headingOrder[i] - headingOrder[i - 1] > 1)
+            skippedLevels++;
+    // Form controls: needs a <label for=id>, a wrapping <label>, aria-label, aria-labelledby or title.
+    const labelFor = new Set([...body.matchAll(/<label\b[^>]*\bfor\s*=\s*["']?([^"'\s>]+)/gi)].map((m) => m[1].toLowerCase()));
+    const wrapped = new Set();
+    for (const m of body.matchAll(/<label\b[^>]*>([\s\S]*?)<\/label>/gi))
+        for (const c of (m[1] ?? '').matchAll(/<(?:input|select|textarea)\b[^>]*>/gi))
+            wrapped.add(c[0]);
+    const controls = [...body.matchAll(/<(?:input|select|textarea)\b[^>]*>/gi)].map((m) => m[0])
+        .filter((t) => !/\btype\s*=\s*["']?(hidden|submit|button|reset|image)\b/i.test(t));
+    const unlabelled = controls.filter((t) => {
+        if (wrapped.has(t))
+            return false;
+        const id = attr(t, 'id');
+        if (id && labelFor.has(id.toLowerCase()))
+            return false;
+        return !has(t, 'aria-label') && !has(t, 'aria-labelledby') && !has(t, 'title');
+    }).length;
+    // Links: an accessible name from text, aria-label, title, or an image alt inside.
+    const anchors = [...body.matchAll(/<a\b[^>]*>([\s\S]*?)<\/a>/gi)];
+    let empty = 0, generic = 0, skipLink = false, statement = false;
+    for (const m of anchors) {
+        const tag = m[0].match(/^<a\b[^>]*>/i)?.[0] ?? '';
+        const inner = m[1] ?? '';
+        const href = attr(tag, 'href') ?? '';
+        const imgAlt = [...inner.matchAll(/<img\b[^>]*>/gi)].map((x) => attr(x[0], 'alt') ?? '').find((a) => a.trim());
+        const name = (text(inner) || attr(tag, 'aria-label') || attr(tag, 'title') || imgAlt || '').trim();
+        if (!name) {
+            if (href && !/^(#|javascript:)/i.test(href))
+                empty++;
+            continue;
+        }
+        if (GENERIC.test(name))
+            generic++;
+        if (/^#./.test(href) && /דלג|skip/i.test(name))
+            skipLink = true;
+        if (/נגישות|accessibility/i.test(name) || /accessib|negishut|nagishut/i.test(href))
+            statement = true;
+    }
+    const buttons = [...body.matchAll(/<button\b[^>]*>([\s\S]*?)<\/button>/gi)];
+    const buttonsWithoutName = buttons.filter((m) => {
+        const tag = m[0].match(/^<button\b[^>]*>/i)?.[0] ?? '';
+        const inner = m[1] ?? '';
+        const imgAlt = [...inner.matchAll(/<img\b[^>]*>/gi)].some((x) => (attr(x[0], 'alt') ?? '').trim());
+        return !text(inner) && !has(tag, 'aria-label') && !has(tag, 'aria-labelledby') && !has(tag, 'title') && !imgAlt;
+    }).length;
+    const iframesWithoutTitle = [...body.matchAll(/<iframe\b[^>]*>/gi)].filter((m) => !has(m[0], 'title') && !has(m[0], 'aria-label')).length;
+    const viewport = [...h.matchAll(/<meta\b[^>]*\bname\s*=\s*["']viewport["'][^>]*>/gi)].map((m) => attr(m[0], 'content') ?? '').join(';').toLowerCase();
+    const maxScale = viewport.match(/maximum-scale\s*=\s*([\d.]+)/)?.[1];
+    const zoomBlocked = /user-scalable\s*=\s*(no|0)\b/.test(viewport) || (maxScale !== undefined && Number(maxScale) < 2);
+    const positiveTabindex = [...body.matchAll(/\btabindex\s*=\s*["']?(\d+)/gi)].filter((m) => Number(m[1]) > 0).length;
+    const ids = [...body.matchAll(/\bid\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1]);
+    const duplicateIds = ids.length - new Set(ids).size;
+    const autoplayMedia = [...body.matchAll(/<(?:video|audio)\b[^>]*\bautoplay\b[^>]*>/gi)].length;
+    const mainLandmark = /<main\b/i.test(body) || /\brole\s*=\s*["']main["']/i.test(body);
+    return {
+        headingOrder, skippedLevels,
+        inputs: { total: controls.length, unlabelled },
+        links: { total: anchors.length, empty, generic },
+        buttonsWithoutName, iframesWithoutTitle, zoomBlocked, positiveTabindex, duplicateIds, autoplayMedia,
+        mainLandmark, skipLink, accessibilityStatementLink: statement,
+    };
+}

package/packages/connectors/web-crawler/src/fetch.js

@@ -0,0 +1,165 @@
+const UA_DEFAULT = 'Mozilla/5.0 (compatible; GWebOps/0.2; +https://aiguru.co.il/) AppleWebKit/537.36 Chrome/124 Safari/537.36';
+/** AI answer-engine crawlers we probe robots.txt for. */
+export const AI_AGENTS = [
+    'GPTBot', 'OAI-SearchBot', 'ChatGPT-User',
+    'ClaudeBot', 'Claude-Web', 'anthropic-ai',
+    'PerplexityBot', 'Perplexity-User',
+    'Google-Extended', 'Applebot-Extended',
+    'CCBot', 'Bytespider', 'Amazonbot', 'meta-externalagent',
+];
+export const SEARCH_AGENTS = ['Googlebot', 'Bingbot'];
+/** One request, no redirect following. */
+async function hop(url, ua, timeout) {
+    const ctrl = new AbortController();
+    const timer = setTimeout(() => ctrl.abort(), timeout);
+    const t0 = Date.now();
+    try {
+        const res = await fetch(url, {
+            redirect: 'manual',
+            signal: ctrl.signal,
+            headers: { 'user-agent': ua, accept: 'text/html,application/xhtml+xml,*/*;q=0.8' },
+        });
+        const headers = {};
+        res.headers.forEach((v, k) => { headers[k] = v; });
+        return { res, status: res.status, headers, location: headers['location'] ?? null, ms: Date.now() - t0 };
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+/** Fetch a URL, following up to maxHops redirects and recording each. */
+export async function get(url, opts = {}) {
+    const { ua = UA_DEFAULT, timeout = 30_000, maxHops = 10, body = true } = opts;
+    const chain = [];
+    let current = url;
+    let totalMs = 0;
+    for (let i = 0; i <= maxHops; i++) {
+        let h;
+        try {
+            h = await hop(current, ua, timeout);
+        }
+        catch (e) {
+            const err = e;
+            return { ok: false, finalUrl: current, status: 0, chain, headers: {}, body: null, ms: totalMs, bytes: 0,
+                error: err.name === 'AbortError' ? 'timeout' : String(err.message ?? err) };
+        }
+        totalMs += h.ms;
+        chain.push({ url: current, status: h.status, location: h.location, ms: h.ms });
+        const redirecting = h.status >= 300 && h.status < 400 && h.location;
+        if (!redirecting) {
+            let text = null;
+            let bytes = 0;
+            if (body) {
+                try {
+                    text = await h.res.text();
+                    bytes = Buffer.byteLength(text, 'utf8');
+                }
+                catch { /* unreadable body */ }
+            }
+            return { ok: h.status >= 200 && h.status < 300, finalUrl: current, status: h.status, chain,
+                headers: h.headers, body: text, ms: totalMs, bytes, error: null };
+        }
+        current = new URL(h.location, current).toString();
+    }
+    return { ok: false, finalUrl: current, status: 0, chain, headers: {}, body: null, ms: totalMs, bytes: 0,
+        error: `redirect loop or more than ${maxHops} hops` };
+}
+// ---------------------------------------------------------------- robots.txt
+/** Fetch and parse robots.txt into per-agent rule groups. */
+export async function robots(origin, opts = {}) {
+    const url = new URL('/robots.txt', origin).toString();
+    const r = await get(url, opts);
+    const out = { ok: r.ok, status: r.status, url, raw: r.body ?? '', sitemaps: [], groups: [] };
+    if (!r.ok || !r.body)
+        return out;
+    let group = null;
+    let lastWasAgent = false;
+    for (const rawLine of r.body.split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, '').trim();
+        if (!line)
+            continue;
+        const idx = line.indexOf(':');
+        if (idx < 0)
+            continue;
+        const field = line.slice(0, idx).trim().toLowerCase();
+        const value = line.slice(idx + 1).trim();
+        if (field === 'sitemap') {
+            out.sitemaps.push(value);
+            continue;
+        }
+        if (field === 'user-agent') {
+            if (!group || !lastWasAgent) {
+                group = { agents: [], allow: [], disallow: [], crawlDelay: null };
+                out.groups.push(group);
+            }
+            group.agents.push(value);
+            lastWasAgent = true;
+            continue;
+        }
+        lastWasAgent = false;
+        if (!group) {
+            group = { agents: ['*'], allow: [], disallow: [], crawlDelay: null };
+            out.groups.push(group);
+        }
+        if (field === 'allow')
+            group.allow.push(value);
+        else if (field === 'disallow')
+            group.disallow.push(value);
+        else if (field === 'crawl-delay')
+            group.crawlDelay = Number(value) || null;
+    }
+    return out;
+}
+/** Longest-match robots evaluation for one agent and path (RFC 9309 semantics). */
+export function robotsAllows(parsed, agent, pathname = '/') {
+    if (!parsed.groups.length)
+        return { allowed: true, rule: null, matchedAgent: null };
+    const lower = agent.toLowerCase();
+    const chosen = parsed.groups.find((g) => g.agents.some((a) => a.toLowerCase() === lower)) ??
+        parsed.groups.find((g) => g.agents.includes('*')) ?? null;
+    if (!chosen)
+        return { allowed: true, rule: null, matchedAgent: null };
+    const longest = (patterns) => {
+        let best = null;
+        for (const p of patterns) {
+            if (p === '')
+                continue;
+            const re = new RegExp('^' + p.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\$$/, '$'));
+            if (re.test(pathname) && (!best || p.length > best.length))
+                best = p;
+        }
+        return best;
+    };
+    const a = longest(chosen.allow);
+    const d = longest(chosen.disallow);
+    const agentLabel = chosen.agents[0] ?? '*';
+    if (chosen.disallow.includes('') && !d)
+        return { allowed: true, rule: 'Disallow: (empty)', matchedAgent: agentLabel };
+    if (d && (!a || d.length > a.length))
+        return { allowed: false, rule: `Disallow: ${d}`, matchedAgent: agentLabel };
+    return { allowed: true, rule: a ? `Allow: ${a}` : null, matchedAgent: agentLabel };
+}
+// ---------------------------------------------------------------- sitemap
+/** Fetch a sitemap or sitemap index (recursing two levels into indexes). */
+export async function sitemap(url, opts = {}, depth = 0) {
+    const r = await get(url, opts);
+    const out = { url, ok: r.ok, status: r.status, urls: [], indexes: [], error: r.error };
+    if (!r.ok || !r.body)
+        return out;
+    const isIndex = /<sitemapindex[\s>]/i.test(r.body);
+    const locs = [...r.body.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map((m) => m[1] ?? '');
+    const lastmods = [...r.body.matchAll(/<lastmod>\s*([^<]+?)\s*<\/lastmod>/gi)].map((m) => m[1] ?? null);
+    if (isIndex) {
+        out.indexes = locs;
+        if (depth < 2) {
+            for (const child of locs.slice(0, 50)) {
+                const sub = await sitemap(child, opts, depth + 1);
+                out.urls.push(...sub.urls);
+            }
+        }
+    }
+    else {
+        out.urls = locs.map((loc, i) => ({ loc, lastmod: lastmods[i] ?? null }));
+    }
+    return out;
+}

package/packages/connectors/web-crawler/src/index.js

@@ -0,0 +1,69 @@
+/**
+ * Web crawler connector: fetch pages and site-level resources for a site.
+ * Pure I/O and parsing. No rules live here; those are in core/scan.
+ */
+import * as F from "./fetch.js";
+import { parseHtml } from "./parse.js";
+export { AI_AGENTS, SEARCH_AGENTS, get, robots, robotsAllows, sitemap } from "./fetch.js";
+export { parseHtml, safeAbs, safeOrigin } from "./parse.js";
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+/** Fetch and parse one page. */
+export async function fetchPage(url, opts = {}) {
+    const r = await F.get(url, { timeout: 30_000, ...opts });
+    const page = {
+        url, finalUrl: r.finalUrl, status: r.status, chain: r.chain,
+        ms: r.ms, bytes: r.bytes, error: r.error, signals: null,
+    };
+    const ct = r.headers['content-type'] ?? 'text/html';
+    if (r.ok && r.body && /text\/html|application\/xhtml/i.test(ct)) {
+        page.signals = parseHtml(r.body, r.finalUrl || url);
+    }
+    return page;
+}
+/** Fetch many pages with bounded concurrency and a politeness delay. */
+export async function fetchPages(urls, { concurrency = 4, delayMs = 250 } = {}) {
+    const out = new Array(urls.length);
+    let next = 0;
+    async function worker() {
+        while (next < urls.length) {
+            const i = next++;
+            if (i)
+                await sleep(delayMs);
+            out[i] = await fetchPage(urls[i]);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(concurrency, urls.length) }, worker));
+    return out;
+}
+/** Robots verdicts for search engines and AI crawlers at the site root. */
+export async function checkAgents(robotsFile) {
+    const out = [];
+    for (const agent of F.SEARCH_AGENTS)
+        out.push({ kind: 'search', agent, ...F.robotsAllows(robotsFile, agent, '/') });
+    for (const agent of F.AI_AGENTS)
+        out.push({ kind: 'ai', agent, ...F.robotsAllows(robotsFile, agent, '/') });
+    return out;
+}
+/**
+ * Probe host variants with a single hop each: does https://www, https://,
+ * and http:// answer 200 (a split) or redirect (correct)?
+ */
+export async function probeHosts(site) {
+    const origin = new URL(site.production_url).origin;
+    const u = new URL(origin);
+    const candidates = new Set([origin]);
+    for (const h of site.alt_hosts) {
+        try {
+            candidates.add(new URL(h).origin);
+        }
+        catch { /* skip malformed */ }
+    }
+    candidates.add(`http://${u.host}`);
+    candidates.add(u.host.startsWith('www.') ? `https://${u.host.slice(4)}` : `https://www.${u.host}`);
+    const out = [];
+    for (const h of candidates) {
+        const r = await F.get(h + '/', { maxHops: 0, body: false, timeout: 15_000 });
+        out.push({ url: h + '/', status: r.status, location: r.chain[0]?.location ?? null, error: r.error });
+    }
+    return out;
+}