mallmaverick-store-scraper 0.1.0

package/src/browser.js

@@ -0,0 +1,234 @@
+'use strict';
+
+const puppeteer = require('puppeteer');
+const path = require('path');
+const fs = require('fs');
+
+const DEFAULT_UA =
+  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 ' +
+  '(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36';
+
+const BROWSER_LAUNCH_OPTS = {
+  args: [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    '--disable-dev-shm-usage',
+    '--disable-accelerated-2d-canvas',
+    '--no-first-run',
+    '--no-zygote',
+    '--disable-gpu',
+    '--window-size=1440,900',
+  ],
+};
+
+const XHR_CAPTURE_PATTERNS = [
+  /stores?\.json/i,
+  /\/api\/(stores?|listings?|tenants?|directory|merchants?|retailers?)/i,
+  /\/wp-json\/.*\/(stores?|listings?)/i,
+  /directory\.json/i,
+];
+
+async function launchBrowser({ headless = true } = {}) {
+  return puppeteer.launch({
+    headless: headless ? 'new' : false,
+    ...BROWSER_LAUNCH_OPTS,
+  });
+}
+
+async function newPage(browser) {
+  const page = await browser.newPage();
+  await page.setUserAgent(DEFAULT_UA);
+  await page.setViewport({ width: 1440, height: 900 });
+  await page.setExtraHTTPHeaders({
+    'Accept-Language': 'en-US,en;q=0.9',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
+  });
+  page.setDefaultNavigationTimeout(60000);
+  page.setDefaultTimeout(60000);
+  return page;
+}
+
+/**
+ * Attach XHR interception. Captures JSON responses matching either:
+ *   - directoryMode=true → only XHR_CAPTURE_PATTERNS (directory listing APIs)
+ *   - directoryMode=false → any JSON response 50B-200KB (store detail XHRs)
+ *
+ * Returns { interceptedJson: [] } — the array fills as responses arrive.
+ */
+async function attachXhrInterceptor(page, { directoryMode = false } = {}) {
+  const interceptedJson = [];
+  const interceptedRaw = [];
+  await page.setRequestInterception(true);
+  page.on('request', (req) => {
+    try { req.continue(); } catch (_) {}
+  });
+  page.on('response', async (response) => {
+    try {
+      const url = response.url();
+      const ct = response.headers()['content-type'] || '';
+      if (!ct.includes('application/json')) return;
+      if (directoryMode) {
+        if (!XHR_CAPTURE_PATTERNS.some(p => p.test(url))) return;
+        const text = await response.text();
+        interceptedRaw.push({ url, text });
+        interceptedJson.push(text);
+      } else {
+        const text = await response.text();
+        if (text.length > 50 && text.length < 200000) {
+          interceptedRaw.push({ url, text });
+          interceptedJson.push(text);
+        }
+      }
+    } catch (_) {}
+  });
+  return { interceptedJson, interceptedRaw };
+}
+
+/**
+ * Load a URL with attempt-dependent strategy.
+ *   attempt 1: domcontentloaded + single scroll + 1s wait
+ *   attempt 2: domcontentloaded + scroll + clickExpandables + 2s wait
+ *   attempt 3: networkidle2 + scroll + clickExpandables + 3s wait
+ */
+async function loadPageWithStrategy(page, url, { attempt = 1, timeout = 45000 } = {}) {
+  const waitUntil = attempt >= 3 ? 'networkidle2' : 'domcontentloaded';
+  await page.goto(url, { waitUntil, timeout });
+
+  if (attempt === 1) {
+    await autoScroll(page);
+    await sleep(1000);
+  } else if (attempt === 2) {
+    await autoScroll(page, { distance: 300, delay: 200 });
+    await clickExpandables(page);
+    await sleep(2000);
+  } else {
+    await autoScroll(page, { distance: 200, delay: 300 });
+    await clickExpandables(page);
+    await sleep(3000);
+  }
+
+  return collectPageData(page);
+}
+
+/**
+ * Standard load (no attempt escalation). Used by mall context, external follow, etc.
+ */
+async function loadPage(page, url, { waitUntil = 'domcontentloaded', timeout = 45000 } = {}) {
+  await page.goto(url, { waitUntil, timeout });
+  await autoScroll(page);
+  await sleep(500);
+  return collectPageData(page);
+}
+
+async function collectPageData(page) {
+  const html = await page.content();
+  const text = await page.evaluate(() => document.body ? document.body.innerText : '');
+  const title = await page.title();
+  const h1 = await extractH1(page);
+  const jsonLd = await extractJsonLd(page);
+  const metaTags = await extractMetaTags(page);
+  const finalUrl = page.url();
+  return { html, text, title, h1, jsonLd, metaTags, finalUrl };
+}
+
+async function autoScroll(page, opts = {}) {
+  const { distance = 400, delay = 150 } = opts;
+  try {
+    await page.evaluate(async ({ distance, delay }) => {
+      await new Promise(resolve => {
+        let total = 0;
+        const t = setInterval(() => {
+          const h = document.body.scrollHeight;
+          window.scrollBy(0, distance);
+          total += distance;
+          if (total >= h) {
+            clearInterval(t);
+            resolve();
+          }
+        }, delay);
+        setTimeout(() => { clearInterval(t); resolve(); }, 15000);
+      });
+    }, { distance, delay });
+  } catch (_) {}
+}
+
+async function clickExpandables(page) {
+  try {
+    await page.evaluate(() => {
+      const selectors = [
+        '[data-toggle]', '[aria-expanded="false"]',
+        '.accordion-button', '.expand-btn', '.show-more',
+        '[class*="expand"]', '[class*="toggle"]', '[class*="load-more"]',
+        'button[class*="more"]', 'a[class*="more"]',
+      ];
+      for (const sel of selectors) {
+        document.querySelectorAll(sel).forEach(el => { try { el.click(); } catch (_) {} });
+      }
+    });
+    await sleep(500);
+  } catch (_) {}
+}
+
+async function extractH1(page) {
+  try {
+    return await page.evaluate(() => {
+      const h1s = Array.from(document.querySelectorAll('h1'));
+      for (const h of h1s) {
+        const t = (h.innerText || h.textContent || '').trim();
+        if (t.length > 0 && t.length < 200) return t;
+      }
+      return '';
+    });
+  } catch (_) { return ''; }
+}
+
+async function extractJsonLd(page) {
+  try {
+    return await page.evaluate(() => {
+      const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
+      const out = [];
+      for (const s of scripts) {
+        try {
+          const parsed = JSON.parse(s.textContent);
+          if (Array.isArray(parsed)) out.push(...parsed);
+          else out.push(parsed);
+        } catch (_) {}
+      }
+      return out;
+    });
+  } catch (_) { return []; }
+}
+
+async function extractMetaTags(page) {
+  try {
+    return await page.evaluate(() => {
+      const m = {};
+      document.querySelectorAll('meta').forEach(el => {
+        const k = el.getAttribute('name') || el.getAttribute('property') || el.getAttribute('itemprop');
+        const v = el.getAttribute('content');
+        if (k && v) m[k] = v;
+      });
+      return m;
+    });
+  } catch (_) { return {}; }
+}
+
+async function captureScreenshot(page, storeUrl, screenshotDir, attempt = 1) {
+  if (!screenshotDir) return null;
+  try {
+    fs.mkdirSync(screenshotDir, { recursive: true });
+    const safe = storeUrl.replace(/[^a-z0-9]/gi, '_').slice(0, 60);
+    const p = path.join(screenshotDir, `${safe}_attempt${attempt}.png`);
+    await page.screenshot({ path: p, fullPage: false, type: 'png' });
+    return fs.readFileSync(p).toString('base64');
+  } catch (_) { return null; }
+}
+
+function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
+
+module.exports = {
+  launchBrowser, newPage, loadPage, loadPageWithStrategy,
+  attachXhrInterceptor, captureScreenshot,
+  autoScroll, clickExpandables, sleep,
+  XHR_CAPTURE_PATTERNS,
+};

package/src/deterministic.js

@@ -0,0 +1,235 @@
+'use strict';
+
+const { URL } = require('url');
+
+/**
+ * Deterministic field extractors — no LLM, regex/DOM only.
+ * Each function returns the field value or '' if not found.
+ */
+
+const PHONE_RE = /(?:\+?1[\s.\-]?)?\(?([2-9]\d{2})\)?[\s.\-]?([2-9]\d{2})[\s.\-]?(\d{4})\b/g;
+
+/**
+ * Extract a phone number from page text. Returns "(XXX) XXX-XXXX" or ''.
+ * Prefers numbers near "phone", "call", "tel" keywords.
+ */
+function extractPhone(text, jsonLd) {
+  // 1. JSON-LD telephone property wins
+  const fromLd = phoneFromJsonLd(jsonLd);
+  if (fromLd) return formatPhone(fromLd);
+
+  if (!text) return '';
+
+  // 2. Look in proximity of phone/tel keywords first
+  const keywordRe = /(?:phone|tel(?:ephone)?|call)[\s:]*([^\n]{0,40})/gi;
+  let m;
+  while ((m = keywordRe.exec(text)) !== null) {
+    const window = m[1];
+    const phone = matchPhone(window);
+    if (phone) return formatPhone(phone);
+  }
+
+  // 3. Fallback: first phone-shaped number in the text
+  const first = matchPhone(text);
+  return first ? formatPhone(first) : '';
+}
+
+function matchPhone(s) {
+  if (!s) return null;
+  PHONE_RE.lastIndex = 0;
+  const m = PHONE_RE.exec(s);
+  return m ? [m[1], m[2], m[3]] : null;
+}
+
+function formatPhone(parts) {
+  if (!parts) return '';
+  return `(${parts[0]}) ${parts[1]}-${parts[2]}`;
+}
+
+function phoneFromJsonLd(jsonLd) {
+  if (!jsonLd) return null;
+  const arr = Array.isArray(jsonLd) ? jsonLd : [jsonLd];
+  for (const node of arr) {
+    const found = walkPhone(node);
+    if (found) return found;
+  }
+  return null;
+}
+
+function walkPhone(node) {
+  if (!node || typeof node !== 'object') return null;
+  if (Array.isArray(node)) {
+    for (const n of node) { const r = walkPhone(n); if (r) return r; }
+    return null;
+  }
+  if (node.telephone) {
+    const phone = matchPhone(String(node.telephone));
+    if (phone) return phone;
+  }
+  if (node['@graph']) return walkPhone(node['@graph']);
+  for (const k of Object.keys(node)) {
+    const v = node[k];
+    if (v && typeof v === 'object') { const r = walkPhone(v); if (r) return r; }
+  }
+  return null;
+}
+
+/**
+ * Reduce a social URL to a canonical username/page key so the same page is
+ * recognized across path variants.
+ *   facebook.com/pg/Foo/about/  → fb:foo
+ *   facebook.com/Foo            → fb:foo
+ *   instagram.com/Foo/?hl=en    → ig:foo
+ *   x.com/Foo                   → tw:foo
+ */
+function socialKey(url) {
+  if (!url) return null;
+  let u;
+  try { u = new URL(url); } catch { return null; }
+  const host = u.hostname.replace(/^www\.|^m\./, '').toLowerCase();
+  const segs = u.pathname.replace(/^\/+/, '').replace(/\/+$/, '').split('/').filter(Boolean);
+  if (segs.length === 0) return null;
+  // Strip platform-specific prefix segments to get to the page identity.
+  const prefixes = new Set(['pg', 'pages', 'profile.php', 'people']);
+  let i = 0;
+  while (i < segs.length && prefixes.has(segs[i].toLowerCase())) i++;
+  let id = segs[i] || '';
+  // YouTube paths look like /@handle, /c/x, /channel/UC..., /user/x
+  if (host.includes('youtube.com')) {
+    if (segs[0] === 'channel' || segs[0] === 'c' || segs[0] === 'user') id = segs[1] || '';
+    else if (segs[0] && segs[0].startsWith('@')) id = segs[0].slice(1);
+  }
+  if (host.includes('tiktok.com') && id.startsWith('@')) id = id.slice(1);
+  if (!id) return null;
+  id = id.toLowerCase();
+  if (host.includes('facebook.com')) return `fb:${id}`;
+  if (host.includes('instagram.com')) return `ig:${id}`;
+  if (host.includes('twitter.com') || host.includes('x.com')) return `tw:${id}`;
+  if (host.includes('youtube.com')) return `yt:${id}`;
+  if (host.includes('tiktok.com')) return `tt:${id}`;
+  if (host.includes('pinterest.')) return `pi:${id}`;
+  return null;
+}
+
+/**
+ * Extract social media URLs from anchor list.
+ * Excludes mall-wide socials (passed in mallSocials) via canonical socialKey.
+ */
+function extractSocials(links, mallSocials = {}) {
+  const out = {
+    facebook: '', instagram: '', twitter: '',
+    youtube: '', tiktok: '', pinterest: '',
+  };
+  if (!links) return out;
+
+  const patterns = {
+    facebook: /^https?:\/\/(?:www\.|m\.)?facebook\.com\/(?!sharer|share|tr\b)/i,
+    instagram: /^https?:\/\/(?:www\.)?instagram\.com\/(?!p\/|reel\/)/i,
+    twitter: /^https?:\/\/(?:www\.)?(?:twitter|x)\.com\/(?!intent|share)/i,
+    youtube: /^https?:\/\/(?:www\.)?youtube\.com\/(?:@|c\/|channel\/|user\/)/i,
+    tiktok: /^https?:\/\/(?:www\.)?tiktok\.com\/@/i,
+    pinterest: /^https?:\/\/(?:www\.)?pinterest\.[a-z.]+\//i,
+  };
+
+  const mallKeys = new Set();
+  for (const v of Object.values(mallSocials || {})) {
+    const k = socialKey(v);
+    if (k) mallKeys.add(k);
+  }
+
+  for (const l of links) {
+    if (!l.href) continue;
+    const linkKey = socialKey(l.href);
+    if (linkKey && mallKeys.has(linkKey)) continue;
+    for (const [k, re] of Object.entries(patterns)) {
+      if (!out[k] && re.test(l.href)) out[k] = l.href.split(/[?#]/)[0];
+    }
+  }
+  return out;
+}
+
+/**
+ * Find the store's own external website. Requires a meaningful name-to-domain
+ * affinity or an explicit "Visit website / Official site" anchor text — otherwise
+ * returns ''.
+ *
+ * @param {Array<{href, text}>} links
+ * @param {string} mallOrigin
+ * @param {string} storeName        - used to score domain match
+ * @param {Set<string>|string[]} [mallEcosystem] - sister/owner domains to exclude
+ */
+function extractWebsite(links, mallOrigin, storeName, mallEcosystem = []) {
+  if (!links) return '';
+  const socialHosts = /(facebook|instagram|twitter|x\.com|youtube|tiktok|pinterest|google\.|maps\.|linkedin|snapchat)\./i;
+  const ecoSet = new Set((Array.isArray(mallEcosystem) ? mallEcosystem : Array.from(mallEcosystem))
+    .map(d => String(d).toLowerCase().replace(/^www\./, '')));
+
+  const externals = [];
+  for (const l of links) {
+    if (!l.href) continue;
+    try {
+      const u = new URL(l.href);
+      if (u.origin === mallOrigin) continue;
+      if (socialHosts.test(u.hostname)) continue;
+      if (!/^https?:$/.test(u.protocol)) continue;
+      const host = u.hostname.replace(/^www\./, '');
+      if (ecoSet.has(host)) continue;
+      externals.push({ href: l.href, host, text: l.text || '' });
+    } catch (_) {}
+  }
+  if (externals.length === 0) return '';
+
+  // Build two clean forms of the name — URLs sometimes spell out "&" as "and"
+  // (bathandbodyworks.com) and sometimes drop it entirely (aw.ca).
+  const rawLower = String(storeName || '').toLowerCase();
+  const cleanNameAnd = rawLower.replace(/&/g, 'and').replace(/[^a-z0-9]/g, '');
+  const cleanNameDrop = rawLower.replace(/[^a-z0-9]/g, '');
+  const score = (e) => {
+    let s = 0;
+    const host = e.host.toLowerCase();
+    const t = (e.text || '').toLowerCase();
+    const hostFlat = host.replace(/\./g, '');
+    const matches = (n) => n && n.length >= 2 && hostFlat.includes(n);
+
+    if (matches(cleanNameAnd) || matches(cleanNameDrop)) s += 30;
+    else if (cleanNameAnd.length >= 5 && hostFlat.includes(cleanNameAnd.slice(0, 5))) s += 25;
+    else if (cleanNameAnd.length >= 3 && hostFlat.includes(cleanNameAnd.slice(0, 3))) s += 15;
+
+    if (/^website|^visit|store website|official|view (the )?site/i.test(t)) s += 40;
+    if (/\.googleusercontent|\.cloudfront|cdn\.|locator/.test(host)) s -= 20;
+    return s;
+  };
+
+  let best = externals[0];
+  let bestScore = score(best);
+  for (const e of externals.slice(1)) {
+    const s = score(e);
+    if (s > bestScore) { best = e; bestScore = s; }
+  }
+
+  // Strictness gate: only accept if score is meaningful.
+  if (bestScore < 25) return '';
+  try {
+    const u = new URL(best.href);
+    return `${u.protocol}//${u.host}/`;
+  } catch { return best.href; }
+}
+
+/**
+ * Detect store status flags from text.
+ */
+function detectStatusFlags(text) {
+  const t = String(text || '').toLowerCase();
+  return {
+    is_coming_soon_store: /\bcoming soon\b/.test(t),
+    is_new_store: /\b(now open|new store|newly opened|just opened)\b/.test(t),
+    is_relocated_store: /\b(relocated|new location|moved to)\b/.test(t),
+    is_temporarily_closed: /\b(temporarily closed|temporarily shut|closed for renovation)\b/.test(t),
+  };
+}
+
+module.exports = {
+  extractPhone, formatPhone, matchPhone,
+  extractSocials, socialKey, extractWebsite,
+  detectStatusFlags,
+};