@opencommerceprotocol/cli 1.0.0 → 1.1.0

package/dist/cli.js

@@ -895,6 +895,184 @@ async function fetchSitemap(url, client, maxDepth = 3) {
   return result.urls;
 }
 
+// src/utils/product-discovery.ts
+var SAMPLE_BLOCKLIST = [
+  /\/cart\b/i,
+  /\/checkout\b/i,
+  /\/search\b/i,
+  /\/account\b/i,
+  /\/login\b/i,
+  /\/register\b/i,
+  /\/contact\b/i,
+  /\/about\b/i,
+  /\/blog\b/i,
+  /\/news\b/i,
+  /\/page\//i,
+  /\/pages\//i,
+  /\/collections?\//i,
+  /\/category\/?$/i,
+  /\/categories\/?$/i,
+  /\/sitemap/i,
+  /\/robots\.txt$/i,
+  /\/feed\b/i,
+  /\/rss\b/i
+];
+function selectSampleUrls(urls, sampleSize) {
+  if (sampleSize <= 0 || urls.length === 0) return [];
+  const candidates = urls.filter((u) => {
+    try {
+      const pathname = new URL(u.loc).pathname;
+      return !SAMPLE_BLOCKLIST.some((re) => re.test(pathname));
+    } catch {
+      return false;
+    }
+  });
+  if (candidates.length <= sampleSize) return candidates;
+  const selected = [];
+  for (let i = 0; i < sampleSize; i++) {
+    const idx = Math.floor(i * candidates.length / sampleSize);
+    selected.push(candidates[idx]);
+  }
+  return selected;
+}
+function pathSegments(pathname) {
+  return pathname.split("/").filter((s) => s.length > 0);
+}
+function pathExtension(pathname) {
+  const last = pathname.split("/").pop() ?? "";
+  const dotIdx = last.lastIndexOf(".");
+  if (dotIdx <= 0) return void 0;
+  const ext = last.slice(dotIdx).toLowerCase();
+  if (!/^\.[a-z][a-z0-9]{0,5}$/.test(ext)) return void 0;
+  return ext;
+}
+function escapeRegex(s) {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function validateCandidate(pattern, positives, negatives) {
+  if (positives.length === 0) return { ok: false, confidence: 0 };
+  const posHits = positives.filter((p) => pattern.test(p)).length;
+  if (posHits !== positives.length) return { ok: false, confidence: 0 };
+  const negHits = negatives.filter((n) => pattern.test(n)).length;
+  const negLeak = negatives.length > 0 ? negHits / negatives.length : 0;
+  if (negLeak > 0.1) return { ok: false, confidence: 0 };
+  return { ok: true, confidence: 1 - negLeak };
+}
+function tryStrategySharedSegment(positives, negatives) {
+  const segments = positives.map(pathSegments);
+  const minDepth = Math.min(...segments.map((s) => s.length));
+  if (minDepth === 0) return null;
+  for (let d = 0; d < minDepth; d++) {
+    const candidate = segments[0][d];
+    if (!candidate) continue;
+    const allMatch = segments.every((s) => s[d] === candidate);
+    if (!allMatch) continue;
+    const pattern = new RegExp(`\\/${escapeRegex(candidate)}\\/`, "i");
+    const validation = validateCandidate(pattern, positives, negatives);
+    if (validation.ok) {
+      return {
+        pattern,
+        strategy: "shared-segment",
+        positives,
+        confidence: validation.confidence
+      };
+    }
+  }
+  return null;
+}
+function tryStrategyExtensionSuffix(positives, negatives) {
+  const exts = positives.map(pathExtension);
+  const firstExt = exts[0];
+  if (!firstExt || !exts.every((e) => e === firstExt)) return null;
+  const pattern = new RegExp(`${escapeRegex(firstExt)}$`, "i");
+  const validation = validateCandidate(pattern, positives, negatives);
+  if (!validation.ok) return null;
+  return {
+    pattern,
+    strategy: "extension-suffix",
+    positives,
+    confidence: validation.confidence
+  };
+}
+function tryStrategyExtensionDepth(positives, negatives) {
+  const exts = positives.map(pathExtension);
+  const firstExt = exts[0];
+  if (!firstExt || !exts.every((e) => e === firstExt)) return null;
+  const depths = positives.map((p) => pathSegments(p).length);
+  const minDepth = Math.min(...depths);
+  if (minDepth < 2) return null;
+  const pattern = new RegExp(`^(?:\\/[^\\/]+){${minDepth},}${escapeRegex(firstExt)}$`, "i");
+  const validation = validateCandidate(pattern, positives, negatives);
+  if (!validation.ok) return null;
+  return {
+    pattern,
+    strategy: "extension-depth",
+    positives,
+    confidence: validation.confidence
+  };
+}
+function tryStrategyExtensionSegment(positives, negatives) {
+  const segments = positives.map(pathSegments);
+  if (segments.some((s) => s.length === 0)) return null;
+  const firstSeg = segments[0][0];
+  if (!segments.every((s) => s[0] === firstSeg)) return null;
+  const exts = positives.map(pathExtension);
+  const firstExt = exts[0];
+  if (!firstExt || !exts.every((e) => e === firstExt)) return null;
+  const pattern = new RegExp(
+    `^\\/${escapeRegex(firstSeg)}\\/.*${escapeRegex(firstExt)}$`,
+    "i"
+  );
+  const validation = validateCandidate(pattern, positives, negatives);
+  if (!validation.ok) return null;
+  return {
+    pattern,
+    strategy: "extension-segment",
+    positives,
+    confidence: validation.confidence
+  };
+}
+function inferProductPattern(positives, negatives) {
+  if (positives.length === 0) return null;
+  return tryStrategySharedSegment(positives, negatives) ?? tryStrategyExtensionDepth(positives, negatives) ?? tryStrategyExtensionSegment(positives, negatives) ?? tryStrategyExtensionSuffix(positives, negatives) ?? null;
+}
+async function discoverProductPattern(urls, fetchAndExtract, options = {}) {
+  const sampleSize = options.sampleSize ?? 25;
+  const minPositives = options.minPositives ?? 3;
+  const concurrency = Math.max(1, options.concurrency ?? 5);
+  const onProgress = options.onProgress;
+  const samples = selectSampleUrls(urls, sampleSize);
+  if (samples.length === 0) return null;
+  const positives = [];
+  const negatives = [];
+  let done = 0;
+  for (let i = 0; i < samples.length; i += concurrency) {
+    const batch = samples.slice(i, i + concurrency);
+    const results = await Promise.allSettled(
+      batch.map(async (u) => ({ url: u.loc, isProduct: await fetchAndExtract(u.loc) }))
+    );
+    for (const result of results) {
+      done++;
+      if (result.status !== "fulfilled") continue;
+      const { url, isProduct } = result.value;
+      let pathname;
+      try {
+        pathname = new URL(url).pathname;
+      } catch {
+        continue;
+      }
+      if (isProduct) {
+        positives.push(pathname);
+      } else {
+        negatives.push(pathname);
+      }
+    }
+    if (onProgress) onProgress(done, samples.length, positives.length);
+  }
+  if (positives.length < minPositives) return null;
+  return inferProductPattern(positives, negatives);
+}
+
 // src/utils/jsonld-extractor.ts
 function extractJsonLdBlocks(html) {
   const blocks = [];
@@ -1524,8 +1702,9 @@ async function runCrawl(url, options) {
   sitemapSpinner.succeed(`Found ${allSitemapUrls.length} URLs in sitemap(s)`);
   const productPattern = options.productPattern ? new RegExp(options.productPattern, "i") : void 0;
   let productUrls = filterProductUrls(allSitemapUrls, productPattern);
-  if (robots) {
-    productUrls = productUrls.filter((u) => {
+  const applyRobotsFilter = (candidates) => {
+    if (!robots) return candidates;
+    return candidates.filter((u) => {
       try {
         const urlPath = new URL(u.loc).pathname;
         return isPathAllowed(robots, urlPath, "OCPBot");
@@ -1533,6 +1712,41 @@ async function runCrawl(url, options) {
         return true;
       }
     });
+  };
+  productUrls = applyRobotsFilter(productUrls);
+  const SAMPLING_MIN_MATCHES = 20;
+  if (productUrls.length < SAMPLING_MIN_MATCHES && !options.productPattern && allSitemapUrls.length > 0) {
+    const sampleSpinner = ora(
+      `Only ${productUrls.length} URLs matched default product patterns \u2014 sampling to infer site layout...`
+    ).start();
+    const inference = await discoverProductPattern(
+      allSitemapUrls,
+      async (pageUrl) => {
+        const resp = await client.get(pageUrl);
+        if (!resp.ok) throw new Error(`fetch failed: ${resp.status}`);
+        return extractProductsFromHtml(resp.text, pageUrl).length > 0;
+      },
+      {
+        sampleSize: 25,
+        minPositives: 5,
+        concurrency: options.concurrency,
+        onProgress: (done, total, found) => {
+          sampleSpinner.text = `Sampling pages to find products... ${done}/${total} (${found} products found)`;
+        }
+      }
+    );
+    if (inference) {
+      sampleSpinner.succeed(
+        `Inferred product pattern: ${inference.pattern} (strategy: ${inference.strategy}, ${inference.positives.length} positives)`
+      );
+      productUrls = applyRobotsFilter(filterProductUrls(allSitemapUrls, inference.pattern));
+    } else {
+      sampleSpinner.fail("Could not infer a confident product URL pattern.");
+      console.log(import_chalk5.default.yellow("\n  Re-run with --product-pattern '<regex>' to specify the pattern manually."));
+      console.log(import_chalk5.default.dim("  Example: --product-pattern '\\.html$'"));
+      console.log(import_chalk5.default.dim("  If your product pages require JavaScript to render, the crawler cannot extract their data."));
+      return;
+    }
   }
   if (options.maxProducts) {
     productUrls = productUrls.slice(0, options.maxProducts);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opencommerceprotocol/cli",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Open Commerce Protocol — CLI tool for generating and validating OCP files",
   "bin": {
     "ocp": "./dist/cli.js"
@@ -18,7 +18,7 @@
   ],
   "scripts": {
     "build": "tsup",
-    "test": "vitest run",
+    "test": "vitest run --root ../.. packages/cli",
     "lint": "eslint src",
     "typecheck": "tsc --noEmit",
     "prepublishOnly": "npm run build"
@@ -36,17 +36,16 @@
     "@types/papaparse": "^5.3.14",
     "tsup": "^8.0.0",
     "typescript": "^5.5.0",
-    "vitest": "^2.0.0",
-    "@types/node": "^20.0.0",
-    "jsdom": "^28.1.0"
+    "vitest": "^2.0.0"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/OpenCommerceProtocol/cli.git"
+    "url": "https://github.com/opencommerceprotocol/ocp",
+    "directory": "packages/cli"
   },
   "homepage": "https://opencommerceprotocol.org",
   "bugs": {
-    "url": "https://github.com/OpenCommerceProtocol/cli/issues"
+    "url": "https://github.com/opencommerceprotocol/ocp/issues"
   },
   "keywords": [
     "ocp",