@commercelayer/agent-cli 0.5.2 → 0.8.0

package/dist/catalog.js

@@ -1,6 +1,7 @@
 // c11r catalog pull: the products live in a tool the human edits too (a Supabase
-// table, a Google Sheet, a Google Merchant Center feed). Pull reads them and
-// writes .commercelayer/products/catalog.csv, so plan and apply work as always.
+// table, a Google Sheet, a Google Merchant Center feed), or on a brand's website (a demo
+// store). Pull reads them and writes .commercelayer/products/catalog.csv, so plan and apply
+// work as always.
 // Every source maps onto the same fixed schema: sku, name, description,
 // image_url, category and one price column per currency.
 import fs from 'node:fs';
@@ -11,28 +12,32 @@ import { CliError, fail } from './output.js';
 import { parseCsv, readProductFile, toCsv } from './products.js';
 import { ISO_639_1, setPlain, setTranslated, translatedColumn, TRANSLATED } from './languages.js';
 import { canonical, checkProducts, readStoreFile, storeLanguage } from './store.js';
-export const SOURCES = ['supabase', 'sheets', 'merchant'];
+import { siteOrigin } from './web.js';
+import { fromWebsite, WEBSITE_LIMIT, WEBSITE_MAX } from './website.js';
+export const SOURCES = ['supabase', 'sheets', 'merchant', 'website'];
 export const catalogFile = path.join(paths.products, 'catalog.csv');
 const FIELDS = ['sku', 'name', 'description', 'image_url', 'category'];
 /** Columns tools add on their own (Supabase's defaults): pull skips them. */
 const IGNORED = ['id', 'created_at', 'updated_at'];
 /** Supabase rows per request; pull pages until a page comes back empty. */
 const PAGE = 1000;
-const EXAMPLE = 'For example `catalog: { source: sheets, url: <the sheet\'s CSV link> }`. Sources are supabase, sheets and merchant; see the comments in store.yml.';
+const EXAMPLE = 'For example `catalog: { source: sheets, url: <the sheet\'s CSV link> }`. Sources are supabase, sheets, merchant and website; see the comments in store.yml.';
 function readConfig(doc) {
     const c = doc.catalog;
     if (!c || typeof c !== 'object')
         fail('no_catalog_source', 'store.yml has no catalog source.', `Add a \`catalog:\` entry to ${paths.store}. ${EXAMPLE}`);
     if (!SOURCES.includes(c.source))
-        fail('invalid_catalog', `catalog.source in store.yml is "${c.source ?? ''}".`, `Use supabase, sheets or merchant. ${EXAMPLE}`);
+        fail('invalid_catalog', `catalog.source in store.yml is "${c.source ?? ''}".`, `Use supabase, sheets, merchant or website. ${EXAMPLE}`);
     const table = String(c.table ?? 'products');
     if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(table))
         fail('invalid_catalog', `catalog.table in store.yml is "${table}".`, 'Use the name of the Supabase table, such as products.');
     const url = String(c.url ?? '');
     if (c.source !== 'supabase' && !/^https?:\/\//.test(url)) {
-        fail('invalid_catalog', url ? `catalog.url in store.yml is "${url}", not a URL.` : 'catalog.url is missing in store.yml.', c.source === 'sheets'
-            ? 'Set url to the Google Sheet\'s CSV link (File > Share > Publish to web > Comma-separated values), or to the sheet\'s own link if it is shared with anyone who has the link.'
-            : 'Set url to the Merchant Center product feed (RSS/XML or TSV), as Merchant Center fetches it.');
+        fail('invalid_catalog', url ? `catalog.url in store.yml is "${url}", not a URL.` : 'catalog.url is missing in store.yml.', c.source === 'website'
+            ? "Set url to the brand's website, such as https://brand.example."
+            : c.source === 'sheets'
+                ? 'Set url to the Google Sheet\'s CSV link (File > Share > Publish to web > Comma-separated values), or to the sheet\'s own link if it is shared with anyone who has the link.'
+                : 'Set url to the Merchant Center product feed (RSS/XML or TSV), as Merchant Center fetches it.');
     }
     const translations = {};
     if (c.translations !== undefined && c.translations !== null) {
@@ -48,7 +53,15 @@ function readConfig(doc) {
             translations[ll] = String(u);
         }
     }
-    return { source: c.source, url, table, translations };
+    let limit = WEBSITE_LIMIT;
+    if (c.limit !== undefined && c.limit !== null) {
+        if (c.source !== 'website')
+            fail('invalid_catalog', `catalog.limit is for a website; this source is ${c.source}.`, 'Pull reads every product of a sheet, a table or a feed: remove limit.');
+        limit = Number(c.limit);
+        if (!Number.isInteger(limit) || limit < 1 || limit > WEBSITE_MAX)
+            fail('invalid_catalog', `catalog.limit in store.yml is "${c.limit}".`, `Use a whole number from 1 to ${WEBSITE_MAX} (default ${WEBSITE_LIMIT}).`);
+    }
+    return { source: c.source, url: c.source === 'website' ? siteOrigin(url, 'catalog.url in store.yml') : url, table, translations, limit };
 }
 async function get(url, what, headers = {}) {
     try {
@@ -271,7 +284,7 @@ async function fromMerchant(c) {
     }
     return out;
 }
-export async function pull() {
+export async function pull(values = {}) {
     if (!fs.existsSync(paths.store))
         fail('no_store', 'There is no store in this repo.', 'Run `c11r init` in the repo root first.');
     let doc;
@@ -285,16 +298,27 @@ export async function pull() {
     const config = readConfig(doc);
     const markets = Array.isArray(doc.markets) ? doc.markets : [];
     const currencies = [...new Set(markets.map((m) => m?.currency).filter(Boolean).map(String))];
-    const sourced = config.source === 'supabase' ? await fromSupabase(config, currencies) : config.source === 'sheets' ? await fromSheet(config, currencies) : await fromMerchant(config);
+    if (values.currency !== undefined && config.source !== 'website')
+        fail('invalid_argument', `--currency is for a website catalog; this source is ${config.source}.`, 'A sheet, a table or a feed says its currencies in its columns: leave out --currency.');
+    let site = null;
+    let sourced;
+    if (config.source === 'website') {
+        const r = await fromWebsite(config, currencies, values.currency);
+        site = r.report;
+        sourced = r.products.map((product) => ({ product, source: `${site.source_site.url}, product ${product.sku}` }));
+    }
+    else
+        sourced = config.source === 'supabase' ? await fromSupabase(config, currencies) : config.source === 'sheets' ? await fromSheet(config, currencies) : await fromMerchant(config);
+    const file = catalogFile;
     if (sourced.length === 0) {
         const rls = config.source === 'supabase' ? ' If the table has rows, row level security hides them from the anon key: ask the human to add a policy that allows select.' : '';
         fail('empty_catalog', `The ${config.source} catalog has no products.`, `Ask the human to add products to it, then pull again.${rls}`);
     }
     const products = checkProducts(sourced, currencies, storeLanguage(doc.language));
     let before = new Map();
-    if (fs.existsSync(catalogFile)) {
+    if (fs.existsSync(file)) {
         try {
-            before = new Map(readProductFile(catalogFile).map((s) => [String(s.product.sku), s.product]));
+            before = new Map(readProductFile(file).map((s) => [String(s.product.sku), s.product]));
         }
         catch (e) {
             if (!(e instanceof CliError))
@@ -303,22 +327,40 @@ export async function pull() {
     }
     const after = new Map(products.map((p) => [p.sku, p]));
     const csv = toCsv(products);
-    const changed = !fs.existsSync(catalogFile) || fs.readFileSync(catalogFile, 'utf8') !== csv;
+    const changed = !fs.existsSync(file) || fs.readFileSync(file, 'utf8') !== csv;
     if (changed) {
         fs.mkdirSync(paths.products, { recursive: true });
-        fs.writeFileSync(catalogFile, csv);
+        fs.writeFileSync(file, csv);
     }
     const removed = [...before.keys()].filter((sku) => !after.has(sku));
     return {
         status: changed ? 'pulled' : 'up_to_date',
         source: config.source,
-        file: catalogFile,
+        ...(site ? { strategy: site.strategy } : {}),
+        file,
         products: products.length,
         added: [...after.keys()].filter((sku) => !before.has(sku)),
         changed: [...after.keys()].filter((sku) => before.has(sku) && canonical(before.get(sku)) !== canonical(after.get(sku))),
         removed,
-        next: changed
-            ? `Run c11r plan to see what apply will change in the store.${removed.length ? ' Removed products are deleted from the store only by c11r apply --allow-delete, once the human agrees.' : ''} ${catalogFile} is rewritten by every pull: change products in the ${config.source} source, not in the file.`
-            : `${catalogFile} already matches the ${config.source} source.`,
+        ...(site ? websiteOutput(site, products.length) : {}),
+        next: site
+            ? websiteNext(site, products.length, changed, removed.length, file)
+            : changed
+                ? `Run c11r plan to see what apply will change in the store.${removed.length ? ' Removed products are deleted from the store only by c11r apply --allow-delete, once the human agrees.' : ''} ${file} is rewritten by every pull: change products in the ${config.source} source, not in the file.`
+                : `${file} already matches the ${config.source} source.`,
     };
 }
+function websiteOutput(r, pulled) {
+    const { strategy: _, ...rest } = r;
+    return { ...rest, pulled };
+}
+function websiteNext(r, pulled, changed, removed, file) {
+    const seen = r.source_site.product_count_seen;
+    const found = seen !== null && seen > pulled ? `I found ${seen} products on the site and pulled ${pulled} (catalog.limit is ${r.limit})` : `I pulled ${pulled} products from the site`;
+    const left = [r.no_price_skipped && `${r.no_price_skipped} without a price`, r.duplicates_skipped && `${r.duplicates_skipped} with a sku already taken`, r.robots_skipped && `${r.robots_skipped} pages robots.txt disallows`, r.other_currency_skipped && `${r.other_currency_skipped} in another currency`].filter(Boolean);
+    return [
+        `Tell the human: "${found}, in ${r.source_site.currency}${r.variants_skipped ? `, one variant each (${r.variants_skipped} other variants left out)` : ''}." ${left.length ? `Left out: ${left.join(', ')}. ` : ''}${r.budget_exhausted ? 'The site was slow: pull stopped at 60 s, so this is what it read by then. ' : ''}`,
+        'Images and the logo stay on the brand\'s servers: image_url links them, nothing is copied. ',
+        changed ? `Run c11r plan to see what apply will change in the store.${removed ? ' Removed products are deleted from the store only by c11r apply --allow-delete, once the human agrees.' : ''} ${file} is rewritten by every pull: never edit it; products to add go in another file in ${paths.products}.` : `${file} already matches the site.`,
+    ].join('');
+}