@aiguru/google-web-operations 0.6.4

package/packages/connectors/web-crawler/src/parse.js

@@ -0,0 +1,157 @@
+import { parseA11y } from "./a11y.js";
+const strip = (s) => s == null ? null : s.replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+const decode = (s) => s == null ? null : s
+    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"').replace(/&#0?39;|&apos;/g, "'").replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)));
+/** Read one attribute from a raw tag string. */
+function attr(tag, name) {
+    const m = tag.match(new RegExp(`\\b${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'));
+    if (!m)
+        return null;
+    return decode(m[2] ?? m[3] ?? m[4] ?? null);
+}
+function metaContent(html, key, which = 'name') {
+    const re = new RegExp(`<meta\\b[^>]*\\b${which}\\s*=\\s*["']?${key}["']?[^>]*>`, 'gi');
+    const out = [];
+    for (const m of html.matchAll(re)) {
+        const v = attr(m[0], 'content');
+        if (v !== null)
+            out.push(v);
+    }
+    return out;
+}
+function collectTypes(node, acc = []) {
+    if (!node)
+        return acc;
+    if (Array.isArray(node)) {
+        node.forEach((n) => collectTypes(n, acc));
+        return acc;
+    }
+    if (typeof node !== 'object')
+        return acc;
+    const obj = node;
+    const t = obj['@type'];
+    if (typeof t === 'string')
+        acc.push(t);
+    else if (Array.isArray(t))
+        acc.push(...t.filter((x) => typeof x === 'string'));
+    for (const k of Object.keys(obj))
+        if (k !== '@type')
+            collectTypes(obj[k], acc);
+    return acc;
+}
+export function safeAbs(href, base) {
+    if (!href)
+        return null;
+    try {
+        return new URL(href, base).toString();
+    }
+    catch {
+        return null;
+    }
+}
+export function safeOrigin(u) {
+    if (!u)
+        return null;
+    try {
+        return new URL(u).origin;
+    }
+    catch {
+        return null;
+    }
+}
+/** Parse a document into a flat signal record. */
+export function parseHtml(html, url) {
+    const h = html || '';
+    const head = (h.match(/<head\b[^>]*>([\s\S]*?)<\/head>/i)?.[1]) ?? h;
+    const htmlTag = h.match(/<html\b[^>]*>/i)?.[0] ?? null;
+    const lang = htmlTag ? attr(htmlTag, 'lang') : null;
+    const dir = htmlTag ? attr(htmlTag, 'dir') : null;
+    const title = decode(strip(h.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? null));
+    const descriptions = metaContent(head, 'description');
+    const description = descriptions[0] ?? null;
+    const robotsMeta = [...metaContent(head, 'robots'), ...metaContent(head, 'googlebot')];
+    const robotsTokens = robotsMeta.flatMap((v) => v.split(',').map((t) => t.trim().toLowerCase()));
+    const links = [...head.matchAll(/<link\b[^>]*>/gi)].map((m) => m[0]);
+    const canonicalTags = links.filter((t) => (attr(t, 'rel') ?? '').toLowerCase().split(/\s+/).includes('canonical'));
+    const canonical = canonicalTags[0] ? attr(canonicalTags[0], 'href') : null;
+    const alternates = links
+        .filter((t) => attr(t, 'hreflang'))
+        .map((t) => ({ hreflang: attr(t, 'hreflang'), href: safeAbs(attr(t, 'href'), url) }));
+    const og = {};
+    for (const m of head.matchAll(/<meta\b[^>]*property\s*=\s*["']og:([^"']+)["'][^>]*>/gi)) {
+        const v = attr(m[0], 'content');
+        const key = m[1];
+        if (v !== null && key && og[key] === undefined)
+            og[key] = v;
+    }
+    const twitter = {};
+    for (const m of head.matchAll(/<meta\b[^>]*name\s*=\s*["']twitter:([^"']+)["'][^>]*>/gi)) {
+        const v = attr(m[0], 'content');
+        const key = m[1];
+        if (v !== null && key && twitter[key] === undefined)
+            twitter[key] = v;
+    }
+    const jsonld = [];
+    for (const m of h.matchAll(/<script\b[^>]*type\s*=\s*["']application\/ld\+json["'][^>]*>([\s\S]*?)<\/script>/gi)) {
+        const raw = (m[1] ?? '').trim();
+        try {
+            jsonld.push({ ok: true, types: collectTypes(JSON.parse(raw)), bytes: raw.length });
+        }
+        catch (e) {
+            jsonld.push({ ok: false, types: [], bytes: raw.length, error: String(e.message ?? e) });
+        }
+    }
+    const microdata = [...h.matchAll(/itemtype\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1] ?? '');
+    const headings = { h1: [], h2: [], h3: [], h4: [] };
+    for (const lvl of ['h1', 'h2', 'h3', 'h4']) {
+        headings[lvl] = [...h.matchAll(new RegExp(`<${lvl}\\b[^>]*>([\\s\\S]*?)<\\/${lvl}>`, 'gi'))]
+            .map((m) => decode(strip(m[1] ?? '')))
+            .filter((x) => !!x);
+    }
+    const imgTags = [...h.matchAll(/<img\b[^>]*>/gi)].map((m) => m[0]);
+    const images = {
+        total: imgTags.length,
+        missingAlt: imgTags.filter((t) => attr(t, 'alt') === null).length,
+        emptyAlt: imgTags.filter((t) => attr(t, 'alt') === '').length,
+        lazy: imgTags.filter((t) => (attr(t, 'loading') ?? '').toLowerCase() === 'lazy').length,
+    };
+    const origin = safeOrigin(url);
+    let internal = 0, external = 0, nofollow = 0;
+    for (const a of [...h.matchAll(/<a\b[^>]*>/gi)].map((m) => m[0])) {
+        const href = attr(a, 'href');
+        if (!href || /^(#|mailto:|tel:|javascript:)/i.test(href))
+            continue;
+        const abs = safeAbs(href, url);
+        if (!abs)
+            continue;
+        if (safeOrigin(abs) === origin)
+            internal++;
+        else
+            external++;
+        if ((attr(a, 'rel') ?? '').toLowerCase().includes('nofollow'))
+            nofollow++;
+    }
+    // Text that exists without JavaScript: the only text most AI crawlers see.
+    const bodyText = decode(strip(h.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' '))) ?? '';
+    const uniq = (re) => [...new Set([...h.matchAll(re)].map((m) => m[0]))];
+    return {
+        url, lang, dir,
+        title, titleLength: title?.length ?? 0,
+        description, descriptionLength: description?.length ?? 0, descriptionCount: descriptions.length,
+        robotsTokens, noindex: robotsTokens.includes('noindex'), nofollowPage: robotsTokens.includes('nofollow'),
+        canonical, canonicalAbs: canonical ? safeAbs(canonical, url) : null, canonicalCount: canonicalTags.length,
+        alternates, og, twitter,
+        jsonld, jsonldTypes: [...new Set(jsonld.flatMap((j) => j.types))], microdata,
+        headings, images,
+        links: { internal, external, nofollow, total: internal + external },
+        textLength: bodyText.length, wordCount: bodyText ? bodyText.split(/\s+/).length : 0,
+        analytics: {
+            ga4: uniq(/\bG-[A-Z0-9]{8,12}\b/g),
+            gtm: uniq(/\bGTM-[A-Z0-9]{5,9}\b/g),
+            ua: uniq(/\bUA-\d{4,}-\d+\b/g),
+        },
+        a11y: parseA11y(h),
+    };
+}

package/packages/core/src/adapters.js

@@ -0,0 +1,27 @@
+export class NoAdapter {
+    kind = 'none';
+    siteId;
+    constructor(siteId) { this.siteId = siteId; }
+    describe() { return `no write adapter configured for ${this.siteId}`; }
+    async ping() { throw new Error(this.describe()); }
+    async listPages() { throw new Error(this.describe()); }
+    async readPageSeo() { throw new Error(this.describe()); }
+    async writePageSeo() { throw new Error(this.describe()); }
+    async readPageSchema() { throw new Error(this.describe()); }
+    async writePageSchema() { throw new Error(this.describe()); }
+}
+/** Build the adapter declared in the site file. Lazy imports keep crawler-only paths light. */
+export async function adapterFor(site) {
+    switch (site.write?.adapter) {
+        case 'odoo': {
+            const { OdooAdapter } = await import("../../connectors/odoo/src/index.js");
+            return OdooAdapter.fromSite(site);
+        }
+        case 'repo': {
+            const { RepoAdapter } = await import("../../connectors/repo/src/index.js");
+            return RepoAdapter.fromSite(site);
+        }
+        default:
+            return new NoAdapter(site.site_id);
+    }
+}

package/packages/core/src/admin.js

@@ -0,0 +1,232 @@
+/**
+ * Administration from the chat: setup status, Google authorisation, property
+ * discovery, and registry edits (add a site, set a field, record a site
+ * decision). Nothing here touches a client site; it changes this system's
+ * own configuration, and every change is recorded in the audit log under
+ * the site it concerns (or "_registry" for global setup).
+ *
+ * Secrets are never accepted as values. The OAuth client is installed from
+ * a file path the operator downloaded; site credentials go into
+ * sites/<host>.secrets.yaml by hand, and setupStatus() says when they are
+ * missing.
+ */
+import { copyFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { parseDocument } from 'yaml';
+import { enforce } from "../../policy/src/index.js";
+import * as storage from "../../storage/src/index.js";
+import { discover, login, makeOauthProvider, setTokenProvider, SCOPES_FOR, SECRETS_DIR, CLIENT_FILE } from "../../connectors/google/src/index.js";
+import { SITES_DIR } from "./paths.js";
+import { getSite, listSites } from "./registry.js";
+const REGISTRY = { site_id: '_registry', policy: { profile: 'standard-client-site' } };
+const record = (siteId, operation, summary, data) => storage.appendAudit({ siteId, operation, status: 'success', risk: 'auto', summary, ...(data !== undefined ? { data } : {}) });
+function siteSetup(s) {
+    const missing = [];
+    const gsc = s.google.search_console?.property ?? null, ga4 = s.google.ga4?.property_id ?? null, gtm = s.google.tag_manager?.public_id ?? null;
+    if (!gsc)
+        missing.push('Search Console property (grant access, then google_discover_apply)');
+    if (!ga4)
+        missing.push('GA4 property id (grant access, then google_discover_apply)');
+    if (!gtm)
+        missing.push('Tag Manager container (none registered; fine if the site has no container)');
+    const adapter = s.write?.adapter ?? 'none';
+    let adapterConfigured = false;
+    const secretsFile = existsSync(path.join(SITES_DIR, s._file.replace(/\.yaml$/, '.secrets.yaml')));
+    if (adapter === 'odoo') {
+        adapterConfigured = !!(s.write.odoo?.url && s.write.odoo?.db);
+        if (!adapterConfigured)
+            missing.push('write.odoo.url and write.odoo.db');
+        if (!secretsFile)
+            missing.push(`sites/${s._file.replace(/\.yaml$/, '.secrets.yaml')} with odoo.login and odoo.api_key (by hand, never through chat)`);
+    }
+    else if (adapter === 'repo') {
+        adapterConfigured = !!s.write.repo?.local_path;
+        if (!adapterConfigured)
+            missing.push('write.repo.local_path (a local checkout with gwo.seo.json)');
+    }
+    else if (s.tier === 'full')
+        missing.push('write adapter (odoo or repo) for a full-tier site');
+    return {
+        id: s.site_id, name: s.name, host: s.production_url, enabled: s.enabled, platform: s.platform, tier: s.tier,
+        searchConsole: gsc, ga4PropertyId: ga4, gtm, adapter, adapterConfigured, secretsFile,
+        suppressions: s.findings?.suppress?.length ?? 0, missing,
+    };
+}
+/** Everything an operator needs to know to finish setting the system up, in one read. */
+export function setupStatus() {
+    const o = makeOauthProvider().status();
+    const has = (s) => o.scopes.some((x) => x === s || x.endsWith(s));
+    const writeScopes = { ga4_write: has(SCOPES_FOR.ga4_write), gtm_edit: has(SCOPES_FOR.gtm_edit), gtm_publish: has(SCOPES_FOR.gtm_publish) };
+    const env = Object.fromEntries(['RESEND_API_KEY', 'GWO_REPORT_FROM', 'GWO_REPORT_TO', 'GITHUB_TOKEN', 'GWO_DOCX_TEMPLATE', 'GOOGLE_PAGESPEED_API_KEY']
+        .map((k) => [k, !!process.env[k]?.trim()]));
+    const sites = listSites({ includeDisabled: true }).map(siteSetup);
+    const next = [];
+    if (!o.clientFile)
+        next.push('Create a "Desktop app" OAuth client in Google Cloud (project "Google Web Operations"), download its JSON, then google_setup_oauth_client with the downloaded file path.');
+    else if (!o.authorised)
+        next.push('google_auth_login: the browser opens for consent with the operations account.');
+    else {
+        if (!writeScopes.ga4_write)
+            next.push('Optional: google_auth_login with scopes ["ga4_write"] before creating key events.');
+        if (!writeScopes.gtm_edit || !writeScopes.gtm_publish)
+            next.push('Optional: google_auth_login with scopes ["gtm_edit", "gtm_publish"] before staging Tag Manager changes.');
+    }
+    if (!sites.length)
+        next.push('ops_add_site for the first site.');
+    if (o.authorised && sites.some((s) => !s.searchConsole || !s.ga4PropertyId))
+        next.push('google_discover_apply once the client has added the operations account to Search Console and GA4.');
+    for (const s of sites)
+        for (const m of s.missing)
+            if (!/discover_apply|none registered/.test(m))
+                next.push(`${s.id}: ${m}`);
+    if (!env.RESEND_API_KEY)
+        next.push('Optional: RESEND_API_KEY, GWO_REPORT_FROM, GWO_REPORT_TO in .env for emailed worker reports.');
+    if (!env.GWO_DOCX_TEMPLATE)
+        next.push('Optional: GWO_DOCX_TEMPLATE in .env for client reports in the company template.');
+    return { google: { clientFile: o.clientFile, authorised: o.authorised, email: o.email, scopes: o.scopes, writeScopes }, env, sites, nextSteps: next };
+}
+// ---------------------------------------------------------------- Google
+/** Copy a downloaded OAuth client JSON into secrets/. Validates the shape; never returns the secret. */
+export function installOauthClient(fromPath) {
+    enforce(REGISTRY, 'google.auth.setup');
+    if (!existsSync(fromPath))
+        throw new Error(`File not found: ${fromPath}`);
+    const raw = JSON.parse(readFileSync(fromPath, 'utf8'));
+    const c = raw.installed;
+    if (!c?.client_id || !c.token_uri) {
+        throw new Error(raw.web ? 'This is a "Web application" client. Create a "Desktop app" client instead; the loopback flow needs it.' : 'Not a Google OAuth client JSON (expected an "installed" section with client_id and token_uri).');
+    }
+    mkdirSync(SECRETS_DIR, { recursive: true });
+    copyFileSync(fromPath, CLIENT_FILE);
+    record('_registry', 'google.auth.setup', `Installed OAuth client ${c.client_id.slice(0, 12)}… from ${fromPath}`);
+    return { clientId: c.client_id, installedAt: CLIENT_FILE };
+}
+/**
+ * Run the consent flow from a chat host. Opens the browser and waits up to
+ * `waitMs`; if consent takes longer the flow keeps running in the background
+ * and the token is stored when it completes, so the caller can poll
+ * setupStatus(). Scope names are the short ones from SCOPES_FOR or full URLs.
+ */
+export async function authLogin(scopes = [], { waitMs = 90_000 } = {}) {
+    enforce(REGISTRY, 'google.auth.login');
+    const resolved = scopes.map((s) => SCOPES_FOR[s] ?? s);
+    const pending = login({ scopes: resolved, open: true, log: () => { } }).then((r) => {
+        setTokenProvider(makeOauthProvider()); // drop the "no authorisation" the process may have cached
+        record('_registry', 'google.auth.login', `Google authorisation stored for ${r.email ?? 'unknown account'} with ${r.scopes.length} scopes`, { scopes: r.scopes });
+        return r;
+    });
+    const timeout = new Promise((res) => setTimeout(() => res(null), waitMs).unref());
+    const r = await Promise.race([pending, timeout]);
+    if (!r) {
+        pending.catch(() => { });
+        return { status: 'waiting', message: `The browser is open for consent. Finish it, then call ops_setup_status; the token is stored automatically.` };
+    }
+    return { status: 'authorised', email: r.email, scopes: r.scopes, message: `Authorised as ${r.email ?? 'unknown account'}; token stored (${r.store}).` };
+}
+/** Discover properties and write the ids of confident matches into the site files. */
+export async function discoverApply() {
+    enforce(REGISTRY, 'registry.site.update');
+    const r = await discover({ apply: true });
+    for (const f of r.applied)
+        record('_registry', 'registry.site.update', `Discovery wrote Google ids into ${f}`);
+    return { applied: r.applied, matches: r.matches, inventory: { searchConsole: r.inventory.gsc.length, ga4: r.inventory.ga4.length, tagManager: r.inventory.gtm.length } };
+}
+const q = (s) => JSON.stringify(s);
+const list = (xs) => `[${xs.map(q).join(', ')}]`;
+/** Host without www; site id is its first label. "doritgrinberg.co.il" -> "doritgrinberg". */
+export function siteIdFor(url) {
+    const host = new URL(url).hostname.replace(/^www\./, '');
+    return { host, id: host.split('.')[0] };
+}
+/** Render a new site file with the same comments as sites/schema.yaml. Pure. */
+export function renderSiteFile(n) {
+    const u = new URL(n.url);
+    const { host, id } = siteIdFor(n.url);
+    const production = `${u.protocol}//${host}`;
+    const alt = n.altHosts ?? [`${u.protocol}//www.${host}`];
+    const locales = n.locales?.length ? n.locales : ['he'];
+    const adapter = n.adapter ?? (n.platform === 'odoo' ? 'odoo' : n.platform === 'nextjs' ? 'repo' : 'none');
+    const write = adapter === 'odoo'
+        ? `write:\n  adapter: odoo\n  odoo:                             # login + api_key go in ${host}.secrets.yaml\n    url: ${n.odoo?.url ?? production}\n    db: ${n.odoo?.db ?? 'CHANGE_ME'}\n    website_id: ${n.odoo?.websiteId ?? 'null'}\n`
+        : adapter === 'repo'
+            ? `write:\n  adapter: repo\n  repo:                             # the checkout must contain gwo.seo.json\n    url: ${n.repo?.url ?? 'null'}\n    branch: ${n.repo?.branch ?? 'main'}\n    local_path: ${n.repo?.localPath ? q(n.repo.localPath) : 'null'}\n`
+            : 'write:\n  adapter: none                     # advisory: every change is a proposal\n';
+    const text = `# ${n.name}. Registered ${new Date().toISOString().slice(0, 10)} through ops_add_site.\n` +
+        `# Credentials go in sites/${host}.secrets.yaml (gitignored), never here.\n\n` +
+        `site_id: ${id}\nname: ${q(n.name)}\nenabled: true\nproduction_url: ${production}\nalt_hosts:\n${alt.map((a) => `  - ${a}`).join('\n')}\n` +
+        `locales: ${list(locales)}\ndefault_locale: ${n.defaultLocale ?? locales[0]}\ntimezone: Asia/Jerusalem\nplatform: ${n.platform ?? 'custom'}\ntier: ${n.tier ?? 'full'}\n\n` +
+        `seo:\n  sitemap: /sitemap.xml\n  robots: /robots.txt\n\n` +
+        `google:                             # filled by google_discover_apply once access is granted\n  search_console:\n    property: null\n  ga4:\n    measurement_id: null\n    property_id: null\n  tag_manager:\n    public_id: null\n\n` +
+        write + `\nconversions:\n  primary_event: ${n.primaryEvent ?? 'generate_lead'}\n\n` +
+        `findings:                           # site decisions; each needs a reason (ops_add_suppression)\n  suppress: []\n\n` +
+        `policy:\n  profile: standard-client-site\n\ncadence:\n  daily: true\n  weekly: true\n  monthly: true\n\n` +
+        `contacts:\n  owner: ${q(n.owner ?? '')}\n  report_to: []\n` +
+        (n.notes ? `\nnotes: ${q(n.notes)}\n` : '');
+    return { file: `${host}.yaml`, text, id };
+}
+/** Create sites/<host>.yaml. Refuses to overwrite. Returns the loaded site. */
+export function addSite(n) {
+    enforce(REGISTRY, 'registry.site.create');
+    const { file, text, id } = renderSiteFile(n);
+    const full = path.join(SITES_DIR, file);
+    if (existsSync(full))
+        throw new Error(`${file} already exists. Use ops_update_site to change it.`);
+    const clash = listSites({ includeDisabled: true }).find((s) => s.site_id === id);
+    if (clash)
+        throw new Error(`site id "${id}" is already used by ${clash._file}.`);
+    mkdirSync(SITES_DIR, { recursive: true });
+    writeFileSync(full, text);
+    const site = getSite(id);
+    record(id, 'registry.site.create', `Registered ${site.production_url} as ${id} (${site.platform}, ${site.tier}, adapter ${site.write.adapter})`);
+    return { file, site };
+}
+const PROTECTED_PATHS = /^(site_id|policy)(\.|$)/;
+/**
+ * Set one field in a site file by dotted path, keeping comments. Values are
+ * parsed as JSON when they look like it ("123", "true", "[\"he\",\"en\"]"),
+ * otherwise kept as strings. site_id and policy are not editable here.
+ */
+export function updateSite(siteIdOrHost, dotted, value) {
+    enforce(REGISTRY, 'registry.site.update');
+    if (PROTECTED_PATHS.test(dotted))
+        throw new Error(`${dotted} is not editable from chat. Edit the file by hand; policy can only be tightened.`);
+    if (/secret|api_key|password|token/i.test(dotted))
+        throw new Error('Credentials never go into a site file. Put them in sites/<host>.secrets.yaml by hand.');
+    const site = getSite(siteIdOrHost);
+    const file = path.join(SITES_DIR, site._file);
+    const doc = parseDocument(readFileSync(file, 'utf8'));
+    const p = dotted.split('.');
+    const before = doc.getIn(p, true) instanceof Object ? JSON.parse(JSON.stringify(doc.getIn(p))) : doc.getIn(p);
+    let v = value;
+    if (typeof value === 'string') {
+        try {
+            v = JSON.parse(value);
+        }
+        catch {
+            v = value;
+        }
+    }
+    doc.setIn(p, v);
+    writeFileSync(file, doc.toString());
+    record(site.site_id, 'registry.site.update', `Set ${dotted}: ${JSON.stringify(before ?? null)} -> ${JSON.stringify(v)}`);
+    return { file: site._file, path: dotted, before: before ?? null, after: v };
+}
+/** Record a site decision: a finding that is intentional here. */
+export function addSuppression(siteIdOrHost, s) {
+    enforce(REGISTRY, 'registry.site.update');
+    if (!s.rule.trim() || !s.reason.trim())
+        throw new Error('A suppression needs a rule id and a reason.');
+    const site = getSite(siteIdOrHost);
+    const file = path.join(SITES_DIR, site._file);
+    const doc = parseDocument(readFileSync(file, 'utf8'));
+    const current = (doc.getIn(['findings', 'suppress']) ? JSON.parse(JSON.stringify(doc.getIn(['findings', 'suppress']))) : []);
+    const entry = { rule: s.rule.trim(), ...(s.path ? { path: s.path.trim() } : {}), reason: s.reason.trim() };
+    const dup = current.find((x) => x.rule === entry.rule && (x.path ?? '') === (entry.path ?? ''));
+    if (dup)
+        throw new Error(`Already suppressed: ${entry.rule}${entry.path ? ` on ${entry.path}` : ''} ("${dup.reason}").`);
+    const next = [...current, entry];
+    doc.setIn(['findings', 'suppress'], next);
+    writeFileSync(file, doc.toString());
+    record(site.site_id, 'registry.site.update', `Suppressed ${entry.rule}${entry.path ? ` on ${entry.path}` : ''}: ${entry.reason}`);
+    return { file: site._file, suppressions: next };
+}

package/packages/core/src/content.js

@@ -0,0 +1,213 @@
+/**
+ * Content writes: meta descriptions, titles and structured data.
+ *
+ * The agent writes the text (see the gwo-content-brief playbook). This
+ * module validates it, runs it through performWrite() against the site's
+ * adapter, and verifies by reading the stored value back. Live-page
+ * confirmation is a separate step because CDNs and build pipelines can lag.
+ */
+import { performWrite } from "./writes.js";
+import { adapterFor } from "./adapters.js";
+import { getSite } from "./registry.js";
+import { DESC_BOUNDS, isHebrew } from "./scan/rules.js";
+/** Validator bounds for titles. The brief aims at 50 to 60; these are the hard limits. */
+export const TITLE_BOUNDS = { min: 15, max: 65 };
+/** Hard ceiling on one page's managed JSON-LD, serialised. Larger blocks slow every page load. */
+export const SCHEMA_MAX_BYTES = 20_000;
+function commonTextProblems(text, problems) {
+    const t = text.trim();
+    if (t !== text)
+        problems.push({ code: 'whitespace', message: 'Leading or trailing whitespace.' });
+    if (/[\r\n\t]/.test(t))
+        problems.push({ code: 'linebreak', message: 'Line breaks or tabs are not allowed.' });
+    if (/<[^>]+>/.test(t))
+        problems.push({ code: 'html', message: 'HTML tags are not allowed.' });
+    const latinLetters = t.replace(/[^A-Za-z]/g, '');
+    if (latinLetters.length >= 20 && latinLetters === latinLetters.toUpperCase())
+        problems.push({ code: 'caps', message: 'All-caps text.' });
+    return t;
+}
+/**
+ * Validate a meta description against the content brief.
+ * Length by script, no stuffing, no shouting, no line breaks, no leading/trailing junk.
+ */
+export function validateMetaDescription(text) {
+    const problems = [];
+    if (!text.trim()) {
+        problems.push({ code: 'empty', message: 'Description is empty.' });
+        return problems;
+    }
+    const t = commonTextProblems(text, problems);
+    const b = isHebrew(t) ? DESC_BOUNDS.hebrew : DESC_BOUNDS.latin;
+    if (t.length < b.min)
+        problems.push({ code: 'short', message: `${t.length} characters; minimum ${b.min} for ${isHebrew(t) ? 'Hebrew' : 'Latin'} text.` });
+    if (t.length > b.max)
+        problems.push({ code: 'long', message: `${t.length} characters; maximum ${b.max} for ${isHebrew(t) ? 'Hebrew' : 'Latin'} text.` });
+    // Keyword stuffing: any token of 4+ letters repeated more than twice.
+    const counts = new Map();
+    for (const w of t.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((x) => x.length >= 4))
+        counts.set(w, (counts.get(w) ?? 0) + 1);
+    const stuffed = [...counts].filter(([, n]) => n > 2).map(([w]) => w);
+    if (stuffed.length)
+        problems.push({ code: 'stuffing', message: `Repeated terms: ${stuffed.join(', ')}. Keyword stuffing lowers visibility.` });
+    return problems;
+}
+/**
+ * Validate a page title against the content brief: 15 to 65 characters, one
+ * line, no HTML, no shouting, at most one brand separator, no term repeated.
+ */
+export function validateTitle(text) {
+    const problems = [];
+    if (!text.trim()) {
+        problems.push({ code: 'empty', message: 'Title is empty.' });
+        return problems;
+    }
+    const t = commonTextProblems(text, problems);
+    if (t.length < TITLE_BOUNDS.min)
+        problems.push({ code: 'short', message: `${t.length} characters; minimum ${TITLE_BOUNDS.min}.` });
+    if (t.length > TITLE_BOUNDS.max)
+        problems.push({ code: 'long', message: `${t.length} characters; maximum ${TITLE_BOUNDS.max}. Search shows about 60.` });
+    const separators = (t.match(/\s[|–—-]\s/g) ?? []).length;
+    if (separators > 1)
+        problems.push({ code: 'separators', message: 'More than one separator. One "primary | brand" split at most.' });
+    const counts = new Map();
+    for (const w of t.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((x) => x.length >= 4))
+        counts.set(w, (counts.get(w) ?? 0) + 1);
+    const repeated = [...counts].filter(([, n]) => n > 1).map(([w]) => w);
+    if (repeated.length)
+        problems.push({ code: 'stuffing', message: `Repeated terms: ${repeated.join(', ')}. A title says each thing once.` });
+    return problems;
+}
+/**
+ * Validate JSON-LD blocks: a non-empty array of objects, each with a
+ * schema.org @context and a @type, no script-closing sequence, bounded size.
+ * Checks the shape, not the vocabulary; the brief owns what to say.
+ */
+export function validateJsonLd(blocks) {
+    const problems = [];
+    if (!Array.isArray(blocks))
+        return [{ code: 'shape', message: 'Expected an array of JSON-LD objects.' }];
+    if (!blocks.length)
+        return [{ code: 'empty', message: 'No blocks. To remove structured data, call with an explicit empty list and say so.' }];
+    blocks.forEach((b, i) => {
+        if (!b || typeof b !== 'object' || Array.isArray(b)) {
+            problems.push({ code: 'shape', message: `Block ${i} is not an object.` });
+            return;
+        }
+        const o = b;
+        const ctx = o['@context'];
+        const ctxOk = typeof ctx === 'string' ? /schema\.org\/?$/i.test(ctx) : !!ctx && typeof ctx === 'object';
+        if (!ctxOk)
+            problems.push({ code: 'context', message: `Block ${i}: @context must be "https://schema.org".` });
+        const type = o['@type'];
+        if (!(typeof type === 'string' && type.trim()) && !(Array.isArray(type) && type.length && type.every((x) => typeof x === 'string'))) {
+            problems.push({ code: 'type', message: `Block ${i}: @type is missing.` });
+        }
+        if (type === 'FAQPage') {
+            const main = o['mainEntity'];
+            if (!Array.isArray(main) || !main.length)
+                problems.push({ code: 'faq', message: `Block ${i}: FAQPage needs a non-empty mainEntity array of Question items.` });
+        }
+    });
+    const text = JSON.stringify(blocks);
+    if (/<\/script/i.test(text))
+        problems.push({ code: 'script', message: 'A value contains "</script"; it would end the script element.' });
+    if (text.length > SCHEMA_MAX_BYTES)
+        problems.push({ code: 'long', message: `${text.length} bytes serialised; maximum ${SCHEMA_MAX_BYTES}.` });
+    return problems;
+}
+async function resolve(siteIdOrHost, opts) {
+    const site = getSite(siteIdOrHost);
+    return { site, adapter: opts.adapter ?? (await adapterFor(site)) };
+}
+const seoSummary = (s) => (s ? { ref: s.ref, title: s.title, description: s.description } : null);
+/**
+ * Set a page's meta description.
+ * Empty field  -> policy action content.meta_description.fill_empty (auto).
+ * Existing text -> content.meta_description.update (approval required).
+ * The class is chosen from the current stored value, never from the caller.
+ */
+export async function setMetaDescription(siteIdOrHost, pagePath, text, opts = {}) {
+    const problems = validateMetaDescription(text);
+    if (problems.length)
+        return { status: 'invalid', problems };
+    const { site, adapter } = await resolve(siteIdOrHost, opts);
+    const current = await adapter.readPageSeo(pagePath);
+    if (!current)
+        throw new Error(`${adapter.kind} adapter knows no page at ${pagePath} for ${site.site_id}.`);
+    const isEmpty = !current.description || !current.description.trim();
+    const action = isEmpty ? 'content.meta_description.fill_empty' : 'content.meta_description.update';
+    return performWrite(site, {
+        action,
+        operation: 'content.set_meta_description',
+        summary: `${isEmpty ? 'Filled empty' : 'Updated'} meta description on ${pagePath} (${current.ref})`,
+        ...(opts.approval ? { approval: opts.approval } : {}),
+        before: async () => current,
+        execute: () => adapter.writePageSeo(pagePath, { description: text.trim() }),
+        after: () => adapter.readPageSeo(pagePath),
+        verify: (_b, after) => (after?.description?.trim() === text.trim() ? null : `stored value is ${JSON.stringify(after?.description ?? null)}`),
+        summarise: seoSummary,
+    });
+}
+/**
+ * Set a page's title. Policy action content.title.update, approval required
+ * always: a worse title on a ranking page is the most expensive edit there is.
+ */
+export async function setTitle(siteIdOrHost, pagePath, text, opts = {}) {
+    const problems = validateTitle(text);
+    if (problems.length)
+        return { status: 'invalid', problems };
+    const { site, adapter } = await resolve(siteIdOrHost, opts);
+    const current = await adapter.readPageSeo(pagePath);
+    if (!current)
+        throw new Error(`${adapter.kind} adapter knows no page at ${pagePath} for ${site.site_id}.`);
+    return performWrite(site, {
+        action: 'content.title.update',
+        operation: 'content.set_title',
+        summary: `Set title on ${pagePath} (${current.ref}): ${JSON.stringify(current.title)} -> ${JSON.stringify(text.trim())}`,
+        ...(opts.approval ? { approval: opts.approval } : {}),
+        before: async () => current,
+        execute: () => adapter.writePageSeo(pagePath, { title: text.trim() }),
+        after: () => adapter.readPageSeo(pagePath),
+        verify: (_b, after) => (after?.title?.trim() === text.trim() ? null : `stored value is ${JSON.stringify(after?.title ?? null)}`),
+        summarise: seoSummary,
+    });
+}
+/** Order-insensitive, key-sorted serialisation so verification ignores formatting. */
+export function canonicalJson(v) {
+    if (Array.isArray(v))
+        return `[${v.map(canonicalJson).join(',')}]`;
+    if (v && typeof v === 'object') {
+        const o = v;
+        return `{${Object.keys(o).sort().map((k) => `${JSON.stringify(k)}:${canonicalJson(o[k])}`).join(',')}}`;
+    }
+    return JSON.stringify(v);
+}
+/**
+ * Replace the managed JSON-LD blocks on a page. Policy action content.schema.update
+ * (approval required on every site). Pass an empty list to remove them; that
+ * skips validation deliberately, since removing is a shape-free operation.
+ */
+export async function setSchema(siteIdOrHost, pagePath, blocks, opts = {}) {
+    if (blocks.length) {
+        const problems = validateJsonLd(blocks);
+        if (problems.length)
+            return { status: 'invalid', problems };
+    }
+    const { site, adapter } = await resolve(siteIdOrHost, opts);
+    const current = await adapter.readPageSchema(pagePath);
+    if (current === null)
+        throw new Error(`${adapter.kind} adapter knows no page at ${pagePath} for ${site.site_id}.`);
+    const types = (bs) => bs.map((b) => String(b['@type'] ?? '?')).join(', ') || 'none';
+    return performWrite(site, {
+        action: 'content.schema.update',
+        operation: 'content.set_schema',
+        summary: `${blocks.length ? 'Set' : 'Removed'} structured data on ${pagePath}: ${types(current)} -> ${types(blocks)}`,
+        ...(opts.approval ? { approval: opts.approval } : {}),
+        before: async () => current,
+        execute: () => adapter.writePageSchema(pagePath, blocks),
+        after: () => adapter.readPageSchema(pagePath),
+        verify: (_b, after) => (after && canonicalJson(after) === canonicalJson(blocks) ? null : `stored blocks are ${after ? types(after) : 'unreadable'}`),
+        summarise: (bs) => (bs ? bs.map((b) => ({ type: b['@type'] ?? null, bytes: JSON.stringify(b).length })) : null),
+    });
+}