@commercelayer/agent-cli 0.8.1 → 1.0.0-dev.1

package/dist/web.js

@@ -1,316 +0,0 @@
-// Reading a brand's public website, politely: catalog pull (source website) and storefront
-// inspect. robots.txt is honoured for every path, one request at a time and at most 2 a second,
-// 15 s per request and 60 s in all, under a User-Agent that names c11r. Nothing is downloaded to
-// keep: pages are read, and images stay on the brand's servers (their URLs are hot-linked).
-import { fail } from './output.js';
-import { VERSION } from './version.js';
-export const WEB_USER_AGENT = `c11r/${VERSION} (+https://commercelayer.ai)`;
-const ROBOTS_TOKEN = 'c11r';
-/** Pages larger than this are not read: no store page or feed is this big. */
-const MAX_BYTES = 8 * 1024 * 1024;
-const num = (name, fallback) => {
-    const v = Number(process.env[name]);
-    return Number.isFinite(v) && v >= 0 ? v : fallback;
-};
-export function parseRobots(text) {
-    const groups = [];
-    const sitemaps = [];
-    let current = null;
-    let lastWasAgent = false;
-    for (const raw of text.split(/\r?\n/)) {
-        const line = raw.replace(/#.*$/, '').trim();
-        const m = line.match(/^([A-Za-z-]+)\s*:\s*(.*)$/);
-        if (!m)
-            continue;
-        const key = m[1].toLowerCase();
-        const value = m[2].trim();
-        if (key === 'sitemap') {
-            if (value)
-                sitemaps.push(value);
-            continue;
-        }
-        if (key === 'user-agent') {
-            if (!lastWasAgent || !current)
-                groups.push((current = { agents: [], rules: [] }));
-            current.agents.push(value.toLowerCase());
-            lastWasAgent = true;
-            continue;
-        }
-        lastWasAgent = false;
-        if (!current || (key !== 'allow' && key !== 'disallow'))
-            continue;
-        // "Disallow:" with nothing after it allows everything: no rule.
-        if (value)
-            current.rules.push({ allow: key === 'allow', pattern: value });
-    }
-    const mine = groups.filter((g) => g.agents.some((a) => a !== '*' && ROBOTS_TOKEN.startsWith(a)));
-    const chosen = mine.length ? mine : groups.filter((g) => g.agents.includes('*'));
-    return { rules: chosen.flatMap((g) => g.rules), sitemaps };
-}
-const patternRegex = (pattern) => {
-    const anchored = pattern.endsWith('$');
-    const body = (anchored ? pattern.slice(0, -1) : pattern)
-        .split('*')
-        .map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))
-        .join('.*');
-    return new RegExp(`^${body}${anchored ? '$' : ''}`);
-};
-/** Whether robots.txt lets c11r read a path (with its query): the longest matching rule wins, allow on a tie. */
-export function robotsAllows(robots, pathAndQuery) {
-    if (robots.unavailable)
-        return false;
-    let best = null;
-    for (const r of robots.rules) {
-        if (!patternRegex(r.pattern).test(pathAndQuery))
-            continue;
-        const length = r.pattern.length;
-        if (!best || length > best.length || (length === best.length && r.allow))
-            best = { allow: r.allow, length };
-    }
-    return best ? best.allow : true;
-}
-export class BudgetExceeded extends Error {
-}
-/** The one way these commands read the web: robots.txt, pacing, timeouts and the total budget. */
-export class Web {
-    robots = new Map();
-    last = 0;
-    started = Date.now();
-    interval = num('C11R_WEB_INTERVAL_MS', 500);
-    timeout = num('C11R_WEB_TIMEOUT_MS', 15_000);
-    budget = num('C11R_WEB_BUDGET_MS', 60_000);
-    requests = 0;
-    robotsSkipped = 0;
-    budgetExhausted = false;
-    remaining() {
-        return this.budget - (Date.now() - this.started);
-    }
-    async raw(url, accept) {
-        if (this.remaining() <= 0) {
-            this.budgetExhausted = true;
-            throw new BudgetExceeded('budget');
-        }
-        const wait = this.last + this.interval - Date.now();
-        if (wait > 0)
-            await new Promise((r) => setTimeout(r, wait));
-        this.last = Date.now();
-        this.requests++;
-        const ms = Math.max(1, Math.min(this.timeout, this.remaining()));
-        try {
-            const res = await fetch(url, { headers: { 'User-Agent': WEB_USER_AGENT, Accept: accept }, redirect: 'follow', signal: AbortSignal.timeout(ms) });
-            const length = Number(res.headers.get('content-length') ?? 0);
-            const text = length > MAX_BYTES ? '' : await res.text();
-            this.last = Date.now();
-            return { url: res.url || url, status: res.status, ok: res.ok && length <= MAX_BYTES, type: res.headers.get('content-type') ?? '', text: text.length > MAX_BYTES ? '' : text, headers: res.headers };
-        }
-        catch (e) {
-            this.last = Date.now();
-            if (e.name === 'TimeoutError') {
-                if (this.remaining() <= 0) {
-                    this.budgetExhausted = true;
-                    throw new BudgetExceeded('budget');
-                }
-                return { url, status: 0, ok: false, type: '', text: '', headers: new Headers() };
-            }
-            throw e;
-        }
-    }
-    robotsFor(origin) {
-        let r = this.robots.get(origin);
-        if (!r) {
-            r = this.raw(`${origin}/robots.txt`, 'text/plain').then((p) => (p.ok ? parseRobots(p.text) : p.status >= 500 ? { rules: [], sitemaps: [], unavailable: true } : { rules: [], sitemaps: [] }), (e) => {
-                if (e instanceof BudgetExceeded)
-                    throw e;
-                return { rules: [], sitemaps: [] };
-            });
-            this.robots.set(origin, r);
-        }
-        return r;
-    }
-    async allowed(url) {
-        const u = new URL(url);
-        return robotsAllows(await this.robotsFor(u.origin), u.pathname + u.search);
-    }
-    /**
-     * GET a URL robots.txt allows: null when it does not (counted in robotsSkipped), the page
-     * otherwise, whatever its status. A redirect to a path robots.txt disallows is null too.
-     */
-    async get(url, accept = 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8') {
-        if (!(await this.allowed(url))) {
-            this.robotsSkipped++;
-            return null;
-        }
-        const page = await this.raw(url, accept);
-        if (page.url !== url && !(await this.allowed(page.url))) {
-            this.robotsSkipped++;
-            return null;
-        }
-        return page;
-    }
-    async json(url) {
-        const page = await this.get(url, 'application/json');
-        if (!page)
-            return null;
-        let body = null;
-        try {
-            body = page.ok ? JSON.parse(page.text) : null;
-        }
-        catch {
-            body = null;
-        }
-        return { page, body };
-    }
-}
-/** The site's origin from what the human gave: a bare host is https. */
-export function siteOrigin(raw, where) {
-    const withScheme = /^[a-z][a-z0-9+.-]*:\/\//i.test(raw) ? raw : `https://${raw}`;
-    let u;
-    try {
-        u = new URL(withScheme);
-    }
-    catch {
-        fail('invalid_argument', `${where} is "${raw}", not a website address.`, 'Use the site\'s address, such as https://brand.example.');
-    }
-    if (u.protocol !== 'https:' && u.protocol !== 'http:')
-        fail('invalid_argument', `${where} is "${raw}", not a website address.`, 'Use an http(s) address, such as https://brand.example.');
-    return u.origin;
-}
-/** The first page could not be read at all. */
-export function unreachable(url, e) {
-    const cause = e.cause;
-    fail('site_unreachable', `Could not reach ${url}: ${cause?.code ?? cause?.message ?? e.message}.`, 'Check the address with the human (open it in a browser), and that this machine is online.');
-}
-const NAMED = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', hellip: '…', ndash: '–', mdash: '—', rsquo: '’', lsquo: '‘', rdquo: '”', ldquo: '“', laquo: '«', raquo: '»', copy: '©', reg: '®', trade: '™', eacute: 'é', egrave: 'è', agrave: 'à', ograve: 'ò', ugrave: 'ù', igrave: 'ì', euro: '€', pound: '£', deg: '°', times: '×' };
-export function decodeEntities(s) {
-    return s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (m, e) => {
-        if (e[0] === '#') {
-            const code = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : Number(e.slice(1));
-            return Number.isFinite(code) && code > 0 && code < 0x110000 ? String.fromCodePoint(code) : m;
-        }
-        return NAMED[e.toLowerCase()] ?? m;
-    });
-}
-/** HTML as plain text, paragraphs kept as blank lines. */
-export function htmlToText(html) {
-    return decodeEntities(html
-        .replace(/<(script|style|head|template|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, '')
-        .replace(/<br\s*\/?>/gi, '\n')
-        .replace(/<li\b[^>]*>/gi, '\n- ')
-        .replace(/<\/li>/gi, '')
-        .replace(/<\/(p|div|tr|ul|ol|h[1-6]|table|section|blockquote)>/gi, '\n\n')
-        .replace(/<[^>]+>/g, ''))
-        .replace(/ /g, ' ')
-        .replace(/[ \t]+/g, ' ')
-        .replace(/ *\n */g, '\n')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-}
-/** Text on one line, trimmed: names. */
-export const oneLine = (s) => decodeEntities(String(s ?? '').replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
-/** At most max characters, cut at a word with an ellipsis. */
-export function clip(s, max) {
-    if (s.length <= max)
-        return s;
-    const cut = s.slice(0, max - 1);
-    const at = cut.search(/\s\S*$/);
-    return (at > max * 0.6 ? cut.slice(0, at) : cut).trimEnd() + '…';
-}
-const TRACKING = /^(utm_[a-z_]+|fbclid|gclid|gclsrc|dclid|gbraid|wbraid|msclkid|mc_cid|mc_eid|_ga|_gl|igshid|yclid|_hsenc|_hsmi|mkt_tok|ref|ref_src|srsltid)$/i;
-/** An absolute http(s) URL from a link on a page, tracking parameters removed; null for anything else. */
-export function absoluteUrl(href, base) {
-    if (typeof href !== 'string' || !href.trim() || /^(data|javascript|blob):/i.test(href.trim()))
-        return null;
-    let u;
-    try {
-        u = new URL(decodeEntities(href.trim()), base);
-    }
-    catch {
-        return null;
-    }
-    if (u.protocol !== 'https:' && u.protocol !== 'http:')
-        return null;
-    for (const k of [...u.searchParams.keys()])
-        if (TRACKING.test(k))
-            u.searchParams.delete(k);
-    u.hash = '';
-    return u.toString();
-}
-/** A sku from a handle or slug: upper case, words joined by underscores. */
-export const skuFromHandle = (handle) => handle
-    .normalize('NFKD')
-    .replace(/[̀-ͯ]/g, '')
-    .toUpperCase()
-    .replace(/[^A-Z0-9]+/g, '_')
-    .replace(/^_+|_+$/g, '');
-/** The attributes of one tag, names in lower case. */
-export function attributes(tag) {
-    const out = {};
-    const inner = tag.replace(/^<[\w:-]+/, '').replace(/\/?>$/, '');
-    for (const m of inner.matchAll(/([^\s=/>"']+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+)))?/g)) {
-        const k = m[1].toLowerCase();
-        if (!(k in out))
-            out[k] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? '');
-    }
-    return out;
-}
-/** Every opening tag of a name, with its attributes and where it starts. */
-export function tags(html, name) {
-    return [...html.matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => ({ attrs: attributes(m[0]), index: m.index, raw: m[0] }));
-}
-/** The JSON-LD blocks of a page, flattened: arrays and @graph opened, unparsable blocks left out. */
-export function jsonLd(html) {
-    const out = [];
-    const walk = (v) => {
-        if (Array.isArray(v))
-            v.forEach(walk);
-        else if (v && typeof v === 'object') {
-            const o = v;
-            out.push(o);
-            if (o['@graph'])
-                walk(o['@graph']);
-        }
-    };
-    for (const m of html.matchAll(/<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi)) {
-        try {
-            walk(JSON.parse(m[1].trim().replace(/^<!--|-->$/g, '')));
-        }
-        catch {
-            // A broken block on a page is common: the others still count.
-        }
-    }
-    return out;
-}
-/** Whether a JSON-LD node is of a type: "@type" is a string or a list. */
-export const isType = (node, type) => {
-    const t = node?.['@type'];
-    const has = (x) => typeof x === 'string' && (x === type || x.endsWith(`/${type}`) || x.endsWith(`:${type}`));
-    return Array.isArray(t) ? t.some(has) : has(t);
-};
-/** <meta name|property=...> content, by name in lower case; the first one wins. */
-export function metas(html) {
-    const out = {};
-    for (const { attrs } of tags(html, 'meta')) {
-        const key = (attrs.property ?? attrs.name ?? attrs.itemprop ?? '').toLowerCase();
-        if (key && attrs.content !== undefined && !(key in out))
-            out[key] = attrs.content.trim();
-    }
-    return out;
-}
-/** <link rel=alternate hreflang=...>: language, region, URL. */
-export function hreflangs(html, base) {
-    return tags(html, 'link')
-        .filter((t) => /\balternate\b/i.test(t.attrs.rel ?? '') && t.attrs.hreflang)
-        .map((t) => {
-        const h = t.attrs.hreflang.trim();
-        const m = h.match(/^([a-z]{2,3})(?:[-_]([a-z]{4}))?(?:[-_]([a-z]{2}|\d{3}))?$/i);
-        return { hreflang: h, language: m ? m[1].toLowerCase() : null, region: m?.[3] ? m[3].toUpperCase() : null, url: absoluteUrl(t.attrs.href, base) };
-    });
-}
-/** The platform a page runs on, from what Shopify and WooCommerce always leave in it. */
-export function platformOf(html, headers) {
-    if (headers?.get('x-shopid') || headers?.get('x-shopify-stage') || /cdn\.shopify\.com|Shopify\.shop\s*=|shopify-section|window\.Shopify\b/.test(html))
-        return 'shopify';
-    if (/wp-content\/plugins\/woocommerce|\bwoocommerce\b|wc-block-|wc\/store\/v1/i.test(html))
-        return 'woocommerce';
-    return 'unknown';
-}