@commercelayer/agent-cli 0.8.0 → 1.0.0-dev.0

package/dist/website.js

@@ -1,390 +0,0 @@
-// catalog pull, source website: the products of a brand's own site, for a demo store. Three ways
-// in, tried in order, and the first that yields products wins: Shopify's public products.json,
-// WooCommerce's Store API, then the sitemap's product pages and their JSON-LD (schema.org
-// Product). Images are hot-linked, never downloaded: they stay on the brand's servers.
-import { fail } from './output.js';
-import { absoluteUrl, BudgetExceeded, clip, htmlToText, hreflangs, isType, jsonLd, oneLine, platformOf, skuFromHandle, unreachable, Web } from './web.js';
-export const WEBSITE_LIMIT = 100;
-export const WEBSITE_MAX = 500;
-const DESCRIPTION_MAX = 2000;
-const SHOPIFY_PAGE = 250;
-const WOO_PAGE = 100;
-/** Child sitemaps read from a sitemap index, product ones first. */
-const MAX_SITEMAPS = 10;
-const PRODUCT_PATH = /\/(products?|p|shop|item)\/[^/?#]+/i;
-/** What a strategy gathered, and what it left out. */
-class Gathered {
-    limit;
-    products = [];
-    skus = new Set();
-    variantsSkipped = 0;
-    noPriceSkipped = 0;
-    duplicatesSkipped = 0;
-    seen = null;
-    currency = null;
-    constructor(limit) {
-        this.limit = limit;
-    }
-    full() {
-        return this.products.length >= this.limit;
-    }
-    add(f) {
-        if (this.full() || !f.name)
-            return;
-        if (f.price === null || !Number.isFinite(f.price) || f.price < 0) {
-            this.noPriceSkipped++;
-            return;
-        }
-        if (!f.sku || this.skus.has(f.sku)) {
-            this.duplicatesSkipped++;
-            return;
-        }
-        this.skus.add(f.sku);
-        this.products.push(f);
-    }
-}
-const text = (html) => {
-    const t = typeof html === 'string' ? clip(htmlToText(html), DESCRIPTION_MAX) : '';
-    return t || undefined;
-};
-/** A category path from site names: "Coffee > Beans"; empty levels and stray > dropped. */
-const category = (...levels) => {
-    const parts = levels
-        .flatMap((l) => String(l ?? '').split('>'))
-        .map((l) => oneLine(l))
-        .filter(Boolean);
-    return parts.length ? parts.join(' > ') : undefined;
-};
-const currencyCode = (v) => (typeof v === 'string' && /^[A-Za-z]{3}$/.test(v.trim()) ? v.trim().toUpperCase() : null);
-async function shopify(web, origin, g) {
-    let base = `${origin}/products.json`;
-    let collection = null;
-    let first = await web.json(`${base}?limit=${SHOPIFY_PAGE}&page=1`);
-    if (!Array.isArray(first?.body?.products)) {
-        const fallback = await web.json(`${origin}/collections/all/products.json?limit=${SHOPIFY_PAGE}&page=1`);
-        if (Array.isArray(fallback?.body?.products)) {
-            base = `${origin}/collections/all/products.json`;
-            collection = 'all';
-            first = fallback;
-        }
-        else
-            return { result: first === null && fallback === null ? 'disallowed' : 'not_found' };
-    }
-    const meta = await web.json(`${origin}/meta.json`).catch((e) => {
-        if (e instanceof BudgetExceeded)
-            throw e;
-        return null;
-    });
-    g.currency = currencyCode(meta?.body?.currency);
-    if (typeof meta?.body?.published_products_count === 'number')
-        g.seen = meta.body.published_products_count;
-    let listed = 0;
-    let page = 1;
-    let body = first.body;
-    for (;;) {
-        const products = body.products;
-        listed += products.length;
-        for (const p of products) {
-            if (g.full())
-                break;
-            const variants = Array.isArray(p.variants) ? p.variants : [];
-            const v = variants.find((x) => x?.available !== false) ?? variants[0];
-            if (variants.length > 1)
-                g.variantsSkipped += variants.length - 1;
-            const tags = Array.isArray(p.tags) ? p.tags : typeof p.tags === 'string' ? p.tags.split(',') : [];
-            const tag = tags.map((t) => t.trim()).find((t) => t && !t.startsWith('_') && !t.includes(':'));
-            g.add({
-                sku: String(v?.sku ?? '').trim() || skuFromHandle(String(p.handle ?? '')),
-                name: oneLine(p.title),
-                description: text(p.body_html),
-                image_url: absoluteUrl(p.images?.[0]?.src, origin) ?? undefined,
-                category: category(p.product_type) ?? category(tag) ?? (collection && collection !== 'all' ? category(collection) : undefined),
-                price: v && v.price !== undefined && v.price !== null && v.price !== '' ? Number(v.price) : null,
-                currency: currencyCode(v?.price_currency ?? v?.price?.currency_code),
-            });
-        }
-        if (g.full() || products.length < SHOPIFY_PAGE)
-            break;
-        page++;
-        const next = await web.json(`${base}?limit=${SHOPIFY_PAGE}&page=${page}`);
-        if (!Array.isArray(next?.body?.products) || next.body.products.length === 0)
-            break;
-        body = next.body;
-    }
-    g.seen ??= listed;
-    g.currency ??= g.products.find((p) => p.currency)?.currency ?? null;
-    return g.products.length ? { result: 'products', gathered: g } : { result: 'no_products' };
-}
-async function woocommerce(web, origin, g) {
-    const url = (page) => `${origin}/wp-json/wc/store/v1/products?per_page=${WOO_PAGE}&page=${page}`;
-    let r = await web.json(url(1));
-    if (r === null)
-        return { result: 'disallowed' };
-    if (!Array.isArray(r.body))
-        return { result: 'not_found' };
-    const total = Number(r.page.headers.get('x-wp-total'));
-    if (Number.isFinite(total) && total > 0)
-        g.seen = total;
-    let listed = 0;
-    for (let page = 1;;) {
-        const products = r.body;
-        listed += products.length;
-        for (const p of products) {
-            if (g.full())
-                break;
-            const prices = p.prices ?? {};
-            const minor = Number(prices.currency_minor_unit ?? 2);
-            const raw = prices.price;
-            const price = raw !== undefined && raw !== null && raw !== '' && Number.isFinite(Number(raw)) ? Number(raw) / 10 ** (Number.isFinite(minor) ? minor : 2) : null;
-            if (Array.isArray(p.variations) && p.variations.length > 1)
-                g.variantsSkipped += p.variations.length - 1;
-            g.add({
-                sku: String(p.sku ?? '').trim() || skuFromHandle(String(p.slug ?? p.id ?? '')),
-                name: oneLine(p.name),
-                description: text(p.description) ?? text(p.short_description),
-                image_url: absoluteUrl(p.images?.[0]?.src, origin) ?? undefined,
-                category: category(p.categories?.[0]?.name),
-                price,
-                currency: currencyCode(prices.currency_code),
-            });
-        }
-        if (g.full() || products.length < WOO_PAGE)
-            break;
-        page++;
-        const next = await web.json(url(page));
-        if (!Array.isArray(next?.body) || next.body.length === 0)
-            break;
-        r = next;
-    }
-    g.seen ??= listed;
-    g.currency = g.products.find((p) => p.currency)?.currency ?? null;
-    return g.products.length ? { result: 'products', gathered: g } : { result: 'no_products' };
-}
-const locs = (xml) => [...xml.matchAll(/<loc>\s*(?:<!\[CDATA\[)?\s*([^<\]]+?)\s*(?:\]\]>)?\s*<\/loc>/gi)].map((m) => m[1].replace(/&amp;/g, '&'));
-/** The product pages a site's sitemaps list: robots.txt's Sitemap lines, else /sitemap.xml; an index is opened. */
-async function productUrls(web, origin) {
-    const robots = await web.robotsFor(origin);
-    const roots = robots.sitemaps.length ? robots.sitemaps.slice(0, 3) : [`${origin}/sitemap.xml`];
-    const urls = new Set();
-    let disallowed = true;
-    const queue = [...roots];
-    let read = 0;
-    while (queue.length && read < MAX_SITEMAPS) {
-        const url = queue.shift();
-        if (/\.gz($|\?)/i.test(url))
-            continue;
-        read++;
-        const page = await web.get(url, 'application/xml,text/xml;q=0.9,*/*;q=0.5');
-        if (page === null)
-            continue;
-        disallowed = false;
-        if (!page.ok)
-            continue;
-        if (/<sitemapindex\b/i.test(page.text)) {
-            // Product sitemaps first (Shopify's sitemap_products_1.xml, WordPress's product-sitemap.xml).
-            const children = locs(page.text).sort((a, b) => Number(/product/i.test(b)) - Number(/product/i.test(a)));
-            queue.push(...children);
-            continue;
-        }
-        for (const loc of locs(page.text)) {
-            const u = absoluteUrl(loc, origin);
-            if (u && new URL(u).origin === origin && PRODUCT_PATH.test(new URL(u).pathname))
-                urls.add(u);
-        }
-    }
-    return { urls: [...urls], disallowed: disallowed && read > 0 };
-}
-const first = (v) => (Array.isArray(v) ? v[0] : v);
-const imageOf = (v, base) => {
-    const i = first(v);
-    const url = typeof i === 'string' ? i : (i?.url ?? i?.contentUrl);
-    return absoluteUrl(url, base) ?? undefined;
-};
-/** "1.234,50", "1,234.50", "14,50", "14.50": the last separator before 1 or 2 digits is the decimal one. */
-export function amount(raw) {
-    const s = raw.replace(/[^\d.,-]/g, '');
-    const m = s.match(/^(-?[\d.,]*?)[.,](\d{1,2})$/);
-    return m ? Number(`${m[1].replace(/[.,]/g, '') || '0'}.${m[2]}`) : Number(s.replace(/[.,]/g, '') || NaN);
-}
-/** The price of a schema.org offer: Offer, AggregateOffer (lowPrice), or a priceSpecification. */
-function offerPrice(offers) {
-    const list = (Array.isArray(offers) ? offers : offers ? [offers] : []);
-    for (const o of list) {
-        const spec = first(o?.priceSpecification);
-        const raw = o?.price ?? o?.lowPrice ?? spec?.price;
-        const n = typeof raw === 'number' ? raw : amount(String(raw ?? ''));
-        if (raw !== undefined && raw !== null && raw !== '' && Number.isFinite(n))
-            return { price: n, currency: currencyCode(o?.priceCurrency ?? spec?.priceCurrency) };
-        if (o?.offers) {
-            const inner = offerPrice(o.offers);
-            if (inner.price !== null)
-                return inner;
-        }
-    }
-    return { price: null, currency: null };
-}
-/** A product page's schema.org Product (a ProductGroup's first variant), with the breadcrumb trail as category. */
-export function productFromPage(html, url) {
-    const nodes = jsonLd(html);
-    const group = nodes.find((n) => isType(n, 'ProductGroup'));
-    const product = nodes.find((n) => isType(n, 'Product')) ?? group;
-    if (!product)
-        return null;
-    const variant = group ? first(group.hasVariant) : undefined;
-    const offers = offerPrice(product.offers ?? variant?.offers);
-    const name = oneLine(product.name ?? group?.name);
-    const crumbs = nodes.find((n) => isType(n, 'BreadcrumbList'));
-    let trail = [];
-    if (Array.isArray(crumbs?.itemListElement)) {
-        trail = [...crumbs.itemListElement]
-            .sort((a, b) => Number(a?.position ?? 0) - Number(b?.position ?? 0))
-            .map((e) => oneLine(e?.name ?? e?.item?.name))
-            .filter(Boolean);
-        if (trail.length && /^(home|homepage|start|inicio|accueil|startseite)$/i.test(trail[0]))
-            trail.shift();
-        if (trail.length && trail[trail.length - 1] === name)
-            trail.pop();
-    }
-    const slug = new URL(url).pathname.split('/').filter(Boolean).pop() ?? '';
-    return {
-        sku: String(product.sku ?? variant?.sku ?? product.productID ?? product.mpn ?? '').trim() || skuFromHandle(decodeURIComponent(slug).replace(/\.html?$/i, '')),
-        name,
-        description: text(product.description),
-        image_url: imageOf(product.image ?? variant?.image, url),
-        category: category(...trail) ?? (typeof product.category === 'string' ? category(product.category) : undefined),
-        price: offers.price,
-        currency: offers.currency,
-    };
-}
-async function structuredData(web, origin, g) {
-    const { urls, disallowed } = await productUrls(web, origin);
-    if (!urls.length)
-        return { result: disallowed ? 'disallowed' : 'not_found' };
-    g.seen = urls.length;
-    const currencies = new Map();
-    for (const url of urls) {
-        if (g.full())
-            break;
-        let page;
-        try {
-            page = await web.get(url);
-        }
-        catch (e) {
-            if (e instanceof BudgetExceeded)
-                break;
-            continue;
-        }
-        if (!page?.ok)
-            continue;
-        const found = productFromPage(page.text, page.url);
-        if (!found)
-            continue;
-        if (found.currency)
-            currencies.set(found.currency, (currencies.get(found.currency) ?? 0) + 1);
-        g.add(found);
-    }
-    g.currency = [...currencies.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
-    if (g.products.length)
-        return { result: 'products', gathered: g };
-    return { result: web.budgetExhausted ? 'budget_exhausted' : 'no_products' };
-}
-const STRATEGIES = [
-    ['shopify', shopify],
-    ['woocommerce', woocommerce],
-    ['structured_data', structuredData],
-];
-/** The site's products as catalog products in its currency, and how they were found. */
-export async function fromWebsite(c, marketCurrencies, currencyFlag) {
-    const web = new Web();
-    let origin = c.url;
-    // The home page: the site's real origin (after a redirect to www), its title, platform and languages.
-    let home = null;
-    try {
-        home = await web.get(`${origin}/`);
-    }
-    catch (e) {
-        if (e instanceof BudgetExceeded)
-            throw e;
-        unreachable(origin, e);
-    }
-    if (home && home.status === 0)
-        fail('site_unreachable', `${origin} did not answer within ${Math.round(web.timeout / 1000)} s.`, 'Check the address with the human (open it in a browser), and try again later.');
-    if (home?.ok)
-        origin = new URL(home.url).origin;
-    const html = home?.ok ? home.text : '';
-    const title = oneLine(html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)?.[1] ?? '') || null;
-    const langs = [html.match(/<html\b[^>]*\blang\s*=\s*["']?([a-zA-Z]{2,3})/)?.[1]?.toLowerCase(), ...hreflangs(html, origin).map((h) => h.language)].filter((l) => !!l);
-    const tried = [];
-    let won = null;
-    for (const [strategy, run] of STRATEGIES) {
-        const g = new Gathered(c.limit);
-        let outcome;
-        try {
-            outcome = await run(web, origin, g);
-        }
-        catch (e) {
-            if (!(e instanceof BudgetExceeded))
-                throw e;
-            outcome = g.products.length ? { result: 'products', gathered: g } : { result: 'budget_exhausted' };
-        }
-        tried.push({ strategy, result: outcome.result });
-        if (outcome.result === 'products') {
-            won = { strategy, gathered: outcome.gathered };
-            break;
-        }
-        if (web.budgetExhausted)
-            break;
-    }
-    if (!won) {
-        const all = tried.length === STRATEGIES.length && tried.every((t) => t.result === 'disallowed');
-        const some = tried.filter((t) => t.result === 'disallowed').map((t) => t.strategy);
-        fail('no_products_found', `Found no products on ${origin}: ${tried.map((t) => `${t.strategy} ${t.result.replace('_', ' ')}`).join(', ')}.`, `${all ? "The site's robots.txt does not let c11r read its products, and c11r does not go around it. " : some.length ? `robots.txt disallows ${some.join(' and ')}. ` : ''}${web.budgetExhausted ? 'The site answered too slowly to finish in 60 s. ' : ''}Ask the human for a product feed or a sheet instead (catalog source merchant or sheets, then c11r catalog pull).`);
-    }
-    const g = won.gathered;
-    const flag = currencyFlag?.trim().toUpperCase();
-    if (flag !== undefined && !/^[A-Z]{3}$/.test(flag))
-        fail('invalid_argument', `--currency is "${currencyFlag}".`, 'Use the ISO 4217 code of the prices on the site, such as EUR.');
-    if (flag && g.currency && flag !== g.currency)
-        fail('currency_mismatch', `The site says its prices are in ${g.currency}, not ${flag}.`, `Leave out --currency: the site's own currency is ${g.currency}.`);
-    const currency = g.currency ?? flag;
-    if (!currency)
-        fail('currency_unknown', `${origin} does not say which currency its prices are in.`, `Ask the human, then run c11r catalog pull --currency <code> (such as EUR). The prices are taken as they are on the site, never converted.`);
-    const others = marketCurrencies.filter((m) => m !== currency);
-    if (marketCurrencies.length && others.length) {
-        fail('currency_mismatch', `The site's prices are in ${currency}; the store's markets sell in ${marketCurrencies.join(', ')}.`, marketCurrencies.includes(currency)
-            ? `A website has one currency: the products would have no ${others.join(', ')} price. Keep only ${currency} markets in store.yml for the demo (ask the human), or add those prices from another source.`
-            : `c11r never converts prices. Ask the human whether the demo store sells in ${currency}: if so, set the market's currency to ${currency} in store.yml, then pull again.`);
-    }
-    let otherCurrency = 0;
-    const products = [];
-    for (const f of g.products) {
-        if (f.currency && f.currency !== currency) {
-            otherCurrency++;
-            continue;
-        }
-        const p = { sku: f.sku, name: f.name, prices: { [currency]: Math.round(f.price * 100) / 100 } };
-        if (f.description)
-            p.description = f.description;
-        if (f.image_url)
-            p.image_url = f.image_url;
-        if (f.category)
-            p.category = f.category;
-        products.push(p);
-    }
-    return {
-        products,
-        report: {
-            strategy: won.strategy,
-            source_site: { url: origin, title, platform: won.strategy === 'structured_data' ? platformOf(html, home?.headers) : won.strategy, product_count_seen: g.seen, currency, languages: [...new Set(langs)] },
-            limit: c.limit,
-            variants_skipped: g.variantsSkipped,
-            robots_skipped: web.robotsSkipped,
-            no_price_skipped: g.noPriceSkipped,
-            duplicates_skipped: g.duplicatesSkipped,
-            other_currency_skipped: otherCurrency,
-            tried,
-            requests: web.requests,
-            budget_exhausted: web.budgetExhausted,
-        },
-    };
-}