mallmaverick-store-scraper 0.1.0

package/src/logoExtraction.js

@@ -0,0 +1,130 @@
+'use strict';
+
+const { URL } = require('url');
+
+/**
+ * Extract candidate logo image URLs from a store detail page.
+ * Returns ranked list (highest priority first).
+ *
+ * Priority:
+ *   1 — img with "logo" in src/alt/class/id/title
+ *   2 — og:image meta
+ *   3 — header/banner/store-logo containers
+ *   4 — JSON-LD image property
+ */
+async function extractLogoImages(page, storeUrl) {
+  let origin = '';
+  try { origin = new URL(storeUrl).origin; } catch (_) {}
+
+  try {
+    return await page.evaluate((origin) => {
+      const resolve = (src) => {
+        if (!src) return null;
+        if (src.startsWith('data:')) return null;
+        if (src.startsWith('http')) return src;
+        if (src.startsWith('//')) return 'https:' + src;
+        if (src.startsWith('/')) return origin + src;
+        return null;
+      };
+
+      const candidates = [];
+
+      // P1: img with "logo" anywhere
+      document.querySelectorAll('img').forEach(img => {
+        const attrs = [
+          img.getAttribute('src') || '',
+          img.getAttribute('data-src') || '',
+          img.getAttribute('alt') || '',
+          img.getAttribute('class') || '',
+          img.getAttribute('id') || '',
+          img.getAttribute('title') || '',
+          img.getAttribute('data-lazy-src') || '',
+        ].join(' ').toLowerCase();
+        if (attrs.includes('logo')) {
+          const src = img.getAttribute('src') || img.getAttribute('data-src') || img.getAttribute('data-lazy-src');
+          const r = resolve(src);
+          if (r) candidates.push({ url: r, priority: 1 });
+        }
+      });
+
+      // P2: og:image
+      const og = document.querySelector('meta[property="og:image"]');
+      if (og) {
+        const r = resolve(og.getAttribute('content'));
+        if (r) candidates.push({ url: r, priority: 2 });
+      }
+
+      // P3: header / store-logo / brand-logo containers
+      const containerSelectors = [
+        'header img', '.store-logo img', '.brand-logo img',
+        '[class*="logo"] img', '[id*="logo"] img',
+        '.store-header img', '.store-profile img', '.store-banner img',
+        '.tenant-logo img', '.retailer-logo img',
+      ];
+      for (const sel of containerSelectors) {
+        document.querySelectorAll(sel).forEach(img => {
+          const src = img.getAttribute('src') || img.getAttribute('data-src');
+          const r = resolve(src);
+          if (r) candidates.push({ url: r, priority: 3 });
+        });
+      }
+
+      const seen = new Set();
+      return candidates
+        .filter(c => { if (seen.has(c.url)) return false; seen.add(c.url); return true; })
+        .sort((a, b) => a.priority - b.priority)
+        .map(c => c.url)
+        .slice(0, 5);
+    }, origin);
+  } catch (_) { return []; }
+}
+
+/**
+ * Pick the best logo image given (directoryLogoUrl, pageLogoCandidates, jsonLd).
+ * Returns the chosen URL or ''.
+ */
+function pickBestLogo({ directoryLogoUrl, pageLogoCandidates, jsonLd }) {
+  if (directoryLogoUrl && !looksLikePlaceholder(directoryLogoUrl)) return directoryLogoUrl;
+
+  // JSON-LD image (Schema.org LocalBusiness.image)
+  if (jsonLd && jsonLd.length) {
+    for (const node of jsonLd) {
+      const img = imageFromJsonLd(node);
+      if (img) return img;
+    }
+  }
+
+  if (pageLogoCandidates && pageLogoCandidates.length > 0) {
+    return pageLogoCandidates[0];
+  }
+
+  return '';
+}
+
+function imageFromJsonLd(node) {
+  if (!node || typeof node !== 'object') return null;
+  if (Array.isArray(node)) {
+    for (const n of node) { const r = imageFromJsonLd(n); if (r) return r; }
+    return null;
+  }
+  if (node.image) {
+    if (typeof node.image === 'string') return node.image;
+    if (Array.isArray(node.image) && node.image[0]) {
+      return typeof node.image[0] === 'string' ? node.image[0] : node.image[0].url;
+    }
+    if (node.image.url) return node.image.url;
+  }
+  if (node.logo) {
+    if (typeof node.logo === 'string') return node.logo;
+    if (node.logo.url) return node.logo.url;
+  }
+  if (node['@graph']) return imageFromJsonLd(node['@graph']);
+  return null;
+}
+
+function looksLikePlaceholder(url) {
+  if (!url) return true;
+  return /placeholder|spacer|blank|default[-_]?(image|logo)/i.test(url);
+}
+
+module.exports = { extractLogoImages, pickBestLogo };

package/src/main.js

@@ -0,0 +1,330 @@
+'use strict';
+
+require('dotenv').config();
+
+const fs = require('fs');
+const path = require('path');
+const { URL } = require('url');
+const readline = require('readline-sync');
+const pLimit = require('p-limit');
+const { createOpenAIClient, describeCredentials } = require('./openai-proxy');
+
+const {
+  launchBrowser, newPage, loadPageWithStrategy,
+  attachXhrInterceptor, captureScreenshot, sleep,
+} = require('./browser');
+const { discoverStores } = require('./discovery');
+const { getMallContext } = require('./mallContext');
+const { extractHours } = require('./hoursPipeline');
+const { classifyImages, pickImages } = require('./imageExtraction');
+const { fetchBrandLogo } = require('./brandSiteFallback');
+const {
+  extractPhone, extractSocials, extractWebsite, detectStatusFlags,
+} = require('./deterministic');
+const { StoreExtractor } = require('./storeExtractor');
+const { scrapeWithRetry, DEFAULT_THRESHOLD } = require('./retryStrategy');
+const { mergeExtracted, storesToCSV } = require('./storeModel');
+
+const logger = {
+  info: (...a) => console.log(...a),
+  warn: (...a) => console.warn(...a),
+  error: (...a) => console.error(...a),
+};
+
+function prompt(label, def) {
+  const v = readline.question(`${label}${def !== undefined ? ` [${def}]` : ''}: `);
+  return v.trim() || def;
+}
+
+function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--url') out.url = argv[++i];
+    else if (a === '--model') out.model = argv[++i];
+    else if (a === '--max') out.max = parseInt(argv[++i], 10);
+    else if (a === '--concurrency') out.concurrency = parseInt(argv[++i], 10);
+    else if (a === '--threshold') out.threshold = parseFloat(argv[++i]);
+    else if (a === '--vision') out.vision = true;
+    else if (a === '--no-vision') out.vision = false;
+  }
+  return out;
+}
+
+async function main() {
+  const creds = describeCredentials();
+  if (creds.mode === 'none') {
+    logger.error(
+      'No OpenAI credentials. Set MALL_SCRAPER_PROXY_URL+MALL_SCRAPER_TOKEN ' +
+      '(via Cloudflare Worker) or OPENAI_API_KEY in .env.'
+    );
+    process.exit(1);
+  }
+  logger.info(`🔑 Auth: ${creds.mode} (${creds.endpoint})`);
+
+  const args = parseArgs(process.argv.slice(2));
+  const directoryUrl = args.url || prompt('Directory URL', 'https://shopcurrents.ca/directory/a-z-listing/');
+  const model = args.model || prompt('Model', 'gpt-5.4-mini');
+  const maxStoresInput = args.max != null ? String(args.max) : prompt('Max stores (0 = all)', '5');
+  const maxStores = parseInt(maxStoresInput, 10) || 0;
+  const concurrency = parseInt(args.concurrency || prompt('Concurrency', '2'), 10) || 2;
+  const threshold = args.threshold != null ? args.threshold
+                  : parseFloat(prompt('Confidence threshold', String(DEFAULT_THRESHOLD))) || DEFAULT_THRESHOLD;
+  const useVision = args.vision != null ? args.vision
+                  : ((prompt('Use Vision (screenshots)?', 'n').toLowerCase().startsWith('y')));
+
+  const client = createOpenAIClient();
+  const browser = await launchBrowser({ headless: true });
+
+  const mallRoot = new URL(directoryUrl).origin;
+  const mallOrigin = mallRoot;
+  const outDir = path.join(__dirname, '..', 'extracted_stores');
+  const screenshotDir = useVision ? path.join(__dirname, '..', 'screenshots') : null;
+  fs.mkdirSync(outDir, { recursive: true });
+  if (screenshotDir) fs.mkdirSync(screenshotDir, { recursive: true });
+
+  const host = new URL(directoryUrl).hostname.replace(/^www\./, '');
+  const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
+  const jsonPath = path.join(outDir, `stores_v5_${host}_${ts}.json`);
+  const csvPath  = path.join(outDir, `stores_v5_${host}_${ts}.csv`);
+
+  const extractor = new StoreExtractor({ client, model, useVision, logger });
+
+  try {
+    logger.info(`\n🏬 Mall root: ${mallRoot}`);
+    logger.info('→ Fetching mall context (general hours)...');
+    const mallContext = await getMallContext(browser, mallRoot);
+    if (mallContext.canonical) {
+      logger.info(`   ✓ Mall hours via ${mallContext.layer}: ${mallContext.canonical}`);
+    } else {
+      logger.info('   (no mall hours found — sync-with-mall layer will be skipped)');
+    }
+
+    logger.info(`\n📋 Discovering stores at ${directoryUrl}...`);
+    const { storeUrls: allUrls, logoMap } = await discoverStores(browser, directoryUrl, logger);
+    const storeCardLogos = Array.from(logoMap.values());
+    const urlsToScrape = maxStores > 0 ? allUrls.slice(0, maxStores) : allUrls;
+    logger.info(`\n→ Scraping ${urlsToScrape.length} stores (concurrency=${concurrency}, threshold=${threshold}, vision=${useVision})\n`);
+
+    const results = [];
+    const limit = pLimit(concurrency);
+    let done = 0;
+
+    const tasks = urlsToScrape.map((url, idx) =>
+      limit(async () => {
+        const mmId = idx + 1;
+        const normalizedUrl = url.replace(/\/$/, '').toLowerCase();
+        const directoryLogoUrl = logoMap.get(normalizedUrl) || null;
+
+        const result = await processStoreWithRetry({
+          url, mmId, browser, client, model, extractor,
+          directoryLogoUrl, mallContext, mallOrigin, storeCardLogos,
+          threshold, screenshotDir, useVision, logger,
+        });
+
+        results[idx] = result;  // preserve order
+        done++;
+        saveCheckpoint(jsonPath, csvPath, results.filter(Boolean), logger);
+        logger.info(`   [done ${done}/${urlsToScrape.length}]`);
+      })
+    );
+    await Promise.all(tasks);
+
+    const stores = results.filter(Boolean);
+    saveResults(jsonPath, csvPath, stores, logger);
+    printSummary(stores, extractor, jsonPath, csvPath);
+  } finally {
+    await browser.close();
+  }
+}
+
+async function processStoreWithRetry(ctx) {
+  const {
+    url, mmId, browser, client, model, extractor,
+    directoryLogoUrl, mallContext, mallOrigin, storeCardLogos,
+    threshold, screenshotDir, useVision, logger,
+  } = ctx;
+
+  logger.info(`\n[${mmId}] ${url}`);
+
+  const runOnce = async (attempt) => {
+    const page = await newPage(browser);
+    const { interceptedJson } = await attachXhrInterceptor(page, { directoryMode: false });
+    try {
+      const data = await loadPageWithStrategy(page, url, { attempt });
+
+      const links = await page.evaluate(() => {
+        return Array.from(document.querySelectorAll('a[href]')).map(a => ({
+          href: a.href, text: (a.innerText || '').trim(),
+        })).filter(o => o.href);
+      });
+      const screenshotBase64 = useVision && screenshotDir
+        ? await captureScreenshot(page, url, screenshotDir, attempt)
+        : null;
+
+      const urlSlug = (() => {
+        try {
+          const parts = new URL(url).pathname.replace(/\/$/, '').split('/').filter(Boolean);
+          return parts[parts.length - 1] || '';
+        } catch { return ''; }
+      })();
+
+      const name = data.h1 || slugToName(urlSlug) || data.title || '';
+
+      // --- Image classification (deterministic, runs before LLM) ---
+      const rawCandidates = await classifyImages(page, url, {
+        storeName: name,
+        mallName: mallContext.mallName || '',
+        mallEcosystem: mallContext.mallEcosystemDomains || [],
+        mallChromeImages: mallContext.mallChromeImages || [],
+        storeCardLogos: storeCardLogos || [],
+      });
+      const imagePicks = pickImages(rawCandidates, { directoryLogoUrl, storeName: name });
+
+      // --- Hours pipeline ---
+      const hoursResult = await extractHours({
+        url, text: data.text, html: data.html, jsonLd: data.jsonLd, metaTags: data.metaTags, links,
+      }, {
+        mallContext, client, model, browser, mallOrigin, logger,
+      });
+
+      // --- Deterministic extractors ---
+      const phone = extractPhone(data.text, data.jsonLd);
+      const socials = extractSocials(links, mallContext.mallSocials);
+      const website = extractWebsite(links, mallOrigin, name, mallContext.mallEcosystemDomains || []);
+      const flagsFromText = detectStatusFlags(data.text);
+
+      // --- LLM extractor (all other fields) ---
+      const llmInput = {
+        url, urlSlug, h1: data.h1, title: data.title,
+        textContent: data.text, jsonLd: data.jsonLd, metaTags: data.metaTags,
+        interceptedJson: interceptedJson.slice(0, 3),
+        screenshotBase64,
+      };
+      const { fields: llmFields, confidence: llmConfidence } =
+        await extractor.extract(llmInput, hoursResult.canonical);
+
+      // --- Image fields: deterministic classifier wins, LLM only fills empty slots ---
+      let logoUrl = imagePicks.logo_image_url || llmFields.logo_image_url || '';
+      let logoSource = 'mall';
+
+      // Brand-site fallback: if mall-picked logo is empty OR a GIF, try the
+      // store's own website. Many CMSes (imgix-based ones especially) refuse
+      // to ingest GIFs, so a non-GIF from the brand site is more reliable.
+      const isGifUrl = /\.gif(\?|$)/i.test(logoUrl);
+      if ((!logoUrl || isGifUrl) && website) {
+        try {
+          const fallback = await fetchBrandLogo(browser, website, name, { logger });
+          if (fallback && fallback.url) {
+            if (!logoUrl || (isGifUrl && !/\.gif(\?|$)/i.test(fallback.url))) {
+              logger.info(`   ↳ brand-site logo: ${fallback.source} → ${fallback.url}`);
+              logoUrl = fallback.url;
+              logoSource = fallback.source;
+            }
+          }
+        } catch (err) {
+          logger.warn(`   ⚠ brand-site fallback errored: ${err.message}`);
+        }
+      }
+
+      const imageFields = {
+        logo_image_url: logoUrl,
+        brand_image_url: imagePicks.brand_image_url || llmFields.brand_image_url || '',
+        store_front_image_url: imagePicks.store_front_image_url || llmFields.store_front_image_url || '',
+        logo_image_url_alt_text: imagePicks.logo_image_url_alt_text || '',
+        brand_image_url_alt_text: imagePicks.brand_image_url_alt_text || '',
+        store_front_image_url_alt_text: imagePicks.store_front_image_url_alt_text || '',
+      };
+
+      // --- Merge into final store record ---
+      const merged = {
+        name,
+        website,
+        phone,
+        ...socials,
+        ...flagsFromText,
+        ...llmFields,
+        ...imageFields,
+        store_hours: hoursResult.canonical,
+        hours_source: hoursResult.source || '',
+        hours_confidence: hoursResult.confidence,
+        sync_with_centre_hours: hoursResult.sync_with_centre_hours || false,
+      };
+      const store = mergeExtracted(mmId, merged);
+
+      // Combined confidence — average hours and LLM-fields halves
+      const combinedConfidence = (hoursResult.confidence + llmConfidence) / 2;
+
+      logger.info(
+        `   ✓ "${store.name}" | hours ${hoursResult.source || '(none)'} ${pct(hoursResult.confidence)}` +
+        ` | fields ${pct(llmConfidence)} | combined ${pct(combinedConfidence)}`
+      );
+
+      return { store, combinedConfidence, hoursResult, llmConfidence };
+    } finally {
+      try { await page.close(); } catch (_) {}
+    }
+  };
+
+  const out = await scrapeWithRetry({ runOnce, threshold, logger });
+  if (out.store) {
+    out.store._meta = {
+      source_url: url,
+      combined_confidence: out.combinedConfidence,
+      hours_confidence: out.hoursResult ? out.hoursResult.confidence : 0,
+      llm_confidence: out.llmConfidence || 0,
+      strategy: out.strategy,
+      attempts: out.attempt,
+      needs_review: out.needs_review,
+    };
+  }
+  return out.store;
+}
+
+function slugToName(slug) {
+  if (!slug) return '';
+  return slug.split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
+}
+
+function pct(n) { return `${Math.round((n || 0) * 100)}%`; }
+
+function saveCheckpoint(jsonPath, csvPath, stores, _logger) {
+  try {
+    fs.writeFileSync(jsonPath, JSON.stringify(stores, null, 2));
+    fs.writeFileSync(csvPath, storesToCSV(stores));
+  } catch (_) {}
+}
+
+function saveResults(jsonPath, csvPath, stores, logger) {
+  fs.writeFileSync(jsonPath, JSON.stringify(stores, null, 2));
+  fs.writeFileSync(csvPath, storesToCSV(stores));
+  logger.info(`\n💾 Wrote ${stores.length} stores`);
+  logger.info(`   JSON: ${jsonPath}`);
+  logger.info(`   CSV:  ${csvPath}`);
+}
+
+function printSummary(stores, extractor, jsonPath, csvPath) {
+  const usage = extractor.getUsageSummary();
+  const needsReview = stores.filter(s => s._meta && s._meta.needs_review).length;
+
+  const bySource = {};
+  for (const s of stores) {
+    const src = s.hours_source || '(none)';
+    bySource[src] = (bySource[src] || 0) + 1;
+  }
+  const avgHoursConf = stores.reduce((a, s) => a + (s.hours_confidence || 0), 0) / Math.max(1, stores.length);
+  const avgFieldConf = stores.reduce((a, s) => a + ((s._meta && s._meta.llm_confidence) || 0), 0) / Math.max(1, stores.length);
+
+  logger.info('\n📊 Hours-extraction layer breakdown:');
+  for (const [src, n] of Object.entries(bySource).sort((a, b) => b[1] - a[1])) {
+    logger.info(`   ${src}: ${n}`);
+  }
+  logger.info(`\n📈 Avg hours conf: ${pct(avgHoursConf)} | Avg field conf: ${pct(avgFieldConf)}`);
+  logger.info(`⚠  Needs review: ${needsReview}/${stores.length}`);
+  logger.info(`💰 LLM cost: ${usage.estimatedCost} (${usage.totalInputTokens}+${usage.totalOutputTokens} tok)`);
+}
+
+main().catch(err => {
+  logger.error(err.stack || err.message);
+  process.exit(1);
+});

package/src/mallContext.js

@@ -0,0 +1,201 @@
+'use strict';
+
+const { URL } = require('url');
+const { loadPage, newPage } = require('./browser');
+const {
+  parseJsonLdHours,
+  parseFreeFormHours,
+  canonicalize,
+  validateCanonical,
+  DAYS,
+} = require('./hoursParser');
+
+/**
+ * Fetch mall context once per session:
+ *   - mall hours (homepage / /hours / /contact)
+ *   - mall's own social media URLs (footer / nav)
+ *   - mall's "ecosystem" external domains (sister properties, ownership group)
+ *
+ * Returns { canonical, sourceUrl, layer, mallSocials, mallEcosystemDomains, mallRootUrl }.
+ * `canonical` is '' if no hours found.
+ */
+async function getMallContext(browser, mallRootUrl) {
+  const origin = new URL(mallRootUrl).origin;
+  const candidates = [mallRootUrl, `${origin}/hours/`, `${origin}/mall-hours/`, `${origin}/contact/`];
+  const tried = new Set();
+  let hoursResult = null;
+
+  // Always scan the homepage once to gather socials + ecosystem links (even if hours found elsewhere)
+  let mallSocials = {
+    facebook: '', instagram: '', twitter: '', youtube: '', tiktok: '', pinterest: '',
+  };
+  let mallEcosystemDomains = new Set();
+  let mallName = '';
+  let mallChromeImages = new Set();
+
+  for (const url of candidates) {
+    if (tried.has(url)) continue;
+    tried.add(url);
+    const page = await newPage(browser);
+    try {
+      const data = await loadPage(page, url);
+
+      // Hours (any layer)
+      if (!hoursResult) {
+        const ld = parseJsonLdHours(data.jsonLd);
+        if (ld) {
+          const c = canonicalize(ld);
+          if (validateCanonical(c).ok) hoursResult = { canonical: c, sourceUrl: data.finalUrl, layer: 'jsonld' };
+        }
+        if (!hoursResult) {
+          const ff = parseFreeFormHours(data.text);
+          if (ff) {
+            const c = canonicalize(ff);
+            if (validateCanonical(c).ok) hoursResult = { canonical: c, sourceUrl: data.finalUrl, layer: 'freeform' };
+          }
+        }
+      }
+
+      // Mall name detection — multiple sources, take the longest meaningful one.
+      if (url === mallRootUrl && !mallName) {
+        const candidates = [];
+        // 1. og:site_name meta (best when present)
+        if (data.metaTags && data.metaTags['og:site_name']) candidates.push(data.metaTags['og:site_name']);
+        if (data.metaTags && data.metaTags['application-name']) candidates.push(data.metaTags['application-name']);
+        // 2. Header logo alt text
+        try {
+          const headerAlt = await page.evaluate(() => {
+            const sels = [
+              'header img[alt]', 'header [class*="logo"] img[alt]',
+              '.site-logo img[alt]', '.site-branding img[alt]',
+              'a[class*="logo"] img[alt]', 'a[rel="home"] img[alt]',
+            ];
+            for (const s of sels) {
+              const el = document.querySelector(s);
+              if (el && el.getAttribute('alt')) return el.getAttribute('alt');
+            }
+            return '';
+          });
+          if (headerAlt) candidates.push(headerAlt);
+        } catch (_) {}
+        // 3. Page title, suffix-stripped, only if not generic ("Home")
+        const t = (data.title || '').trim();
+        if (t && !/^home\b/i.test(t)) {
+          candidates.push(t.split(/\s*[|\-–—]\s*/)[0].trim());
+        }
+        // Pick the longest non-generic candidate
+        for (const c of candidates) {
+          const cl = String(c).trim();
+          if (cl.length >= 4 && !/^home$/i.test(cl) && cl.length > mallName.length) mallName = cl;
+        }
+      }
+
+      // Mall socials + ecosystem + chrome images (all images on the mall homepage are chrome)
+      if (url === mallRootUrl) {
+        const homepageImages = await page.evaluate(() => {
+          const out = new Set();
+          document.querySelectorAll('img').forEach(img => {
+            const src = img.currentSrc || img.src || img.getAttribute('data-src') || img.getAttribute('data-lazy-src');
+            if (src && !src.startsWith('data:')) out.add(src);
+          });
+          const og = document.querySelector('meta[property="og:image"]');
+          if (og && og.getAttribute('content')) out.add(og.getAttribute('content'));
+          return Array.from(out);
+        });
+        for (const u of homepageImages) mallChromeImages.add(u);
+
+        const linksAndDomains = await page.evaluate((origin) => {
+          const links = Array.from(document.querySelectorAll('a[href]'))
+            .map(a => a.href).filter(h => /^https?:/.test(h));
+          const externals = links.filter(h => { try { return new URL(h).origin !== origin; } catch { return false; } });
+          return externals;
+        }, origin);
+
+        const socialPatterns = {
+          facebook: /^https?:\/\/(?:www\.|m\.)?facebook\.com\/[^/?]+/i,
+          instagram: /^https?:\/\/(?:www\.)?instagram\.com\/[^/?]+/i,
+          twitter: /^https?:\/\/(?:www\.)?(?:twitter|x)\.com\/[^/?]+/i,
+          youtube: /^https?:\/\/(?:www\.)?youtube\.com\/(?:@|c\/|channel\/|user\/)[^/?]+/i,
+          tiktok: /^https?:\/\/(?:www\.)?tiktok\.com\/@[^/?]+/i,
+          pinterest: /^https?:\/\/(?:www\.)?pinterest\.[a-z.]+\/[^/?]+/i,
+        };
+        for (const ext of linksAndDomains) {
+          try {
+            const u = new URL(ext);
+            mallEcosystemDomains.add(u.hostname.replace(/^www\./, ''));
+          } catch (_) {}
+          for (const [k, re] of Object.entries(socialPatterns)) {
+            if (!mallSocials[k] && re.test(ext)) mallSocials[k] = ext.split(/[?#]/)[0];
+          }
+        }
+      }
+    } catch (_) {
+    } finally {
+      try { await page.close(); } catch (_) {}
+    }
+    if (hoursResult && url === candidates[candidates.length - 1]) break;
+  }
+
+  if (!hoursResult) hoursResult = { canonical: '', sourceUrl: '', layer: null };
+  return {
+    ...hoursResult,
+    mallName,
+    mallSocials,
+    mallEcosystemDomains: Array.from(mallEcosystemDomains),
+    mallChromeImages: Array.from(mallChromeImages),
+    mallRootUrl,
+  };
+}
+
+async function tryHoursFromUrl(browser, url) {
+  const page = await newPage(browser);
+  try {
+    const data = await loadPage(page, url);
+    // JSON-LD first
+    const ld = parseJsonLdHours(data.jsonLd);
+    if (ld) {
+      const canonical = canonicalize(ld);
+      if (validateCanonical(canonical).ok) {
+        return { canonical, sourceUrl: data.finalUrl, layer: 'jsonld' };
+      }
+    }
+    // Free-form scan of the page text
+    const ff = parseFreeFormHours(data.text);
+    if (ff) {
+      const canonical = canonicalize(ff);
+      if (validateCanonical(canonical).ok) {
+        return { canonical, sourceUrl: data.finalUrl, layer: 'freeform' };
+      }
+    }
+    return null;
+  } catch (_) {
+    return null;
+  } finally {
+    await page.close();
+  }
+}
+
+/**
+ * Detect "store hours follow mall hours" phrasings on a store page.
+ * Conservative — false positives here are worse than false negatives,
+ * because we'll blindly apply mall hours when a match is found.
+ */
+const SYNC_PATTERNS = [
+  /store\s+hours?\s+(?:are\s+)?(?:the\s+)?same\s+as\s+(?:the\s+)?(?:mall|cent(?:re|er))/i,
+  /(?:follows?|same as|match(?:es)?)\s+(?:the\s+)?(?:mall|cent(?:re|er))\s+hours?/i,
+  /open\s+during\s+(?:regular\s+)?(?:mall|cent(?:re|er))\s+hours?/i,
+  /hours?\s+(?:are\s+)?(?:the\s+)?same as (?:mall|cent(?:re|er))/i,
+  /see\s+(?:mall|cent(?:re|er))\s+hours?/i,
+  /(?:mall|cent(?:re|er))\s+hours?\s+apply/i,
+  /hours?\s+follow\s+(?:the\s+)?(?:mall|cent(?:re|er))/i,
+];
+
+function detectsSyncWithMall(text) {
+  if (!text) return false;
+  for (const re of SYNC_PATTERNS) {
+    if (re.test(text)) return true;
+  }
+  return false;
+}
+
+module.exports = { getMallContext, detectsSyncWithMall, SYNC_PATTERNS };