mallmaverick-store-scraper 0.1.0

package/src/storeExtractor.js

@@ -0,0 +1,239 @@
+'use strict';
+
+/**
+ * Store-fields LLM extractor.
+ *
+ * Responsibilities:
+ *   - Extract the LLM-only fields: description, categories, location_type,
+ *     restaurant fields, amenities, parking_information, etc.
+ *   - DOES NOT extract hours (hoursPipeline already did that).
+ *   - DOES NOT extract phone/socials/website (deterministic.js did those).
+ *   - Receives directoryLogoUrl and pageLogoCandidates and returns the picked logo.
+ *
+ * Returns a partial store object to merge into the final record + a per-field
+ * confidence summary.
+ */
+
+const MODEL_PRICING = {
+  'gpt-4o':             { input: 0.005,   output: 0.015   },
+  'gpt-4o-mini':        { input: 0.00015, output: 0.0006  },
+  'gpt-4.1':            { input: 0.008,   output: 0.024   },
+  'gpt-4.1-mini':       { input: 0.0004,  output: 0.0016  },
+  'gpt-4.1-nano':       { input: 0.0001,  output: 0.0004  },
+  'gpt-4-turbo':        { input: 0.01,    output: 0.03    },
+  'gpt-3.5-turbo':      { input: 0.0005,  output: 0.0015  },
+  'gpt-5':              { input: 0.015,   output: 0.060   },
+  'gpt-5-mini':         { input: 0.003,   output: 0.012   },
+  'gpt-5.4-mini':       { input: 0.00075, output: 0.0045  },
+};
+
+const SYSTEM_PROMPT = `You are a data extractor for retail store directory pages.
+Return ONLY valid JSON. Be precise. Never invent values not present in the input.
+For missing strings use "" — for missing booleans use false.
+
+You are NOT extracting: name, store_hours, phone, social media URLs, website,
+logo_image_url, brand_image_url, store_front_image_url. Those are handled by
+other systems. Focus on the remaining fields.
+
+FIELD RULES:
+
+description:
+- The store's marketing/about description from the page. Multi-sentence prose.
+- Preserve sentences as written but strip HTML tags.
+- If only a short tagline is available, use it.
+
+brief_description:
+- 1-2 sentence summary (≤200 chars). Derive from description if needed.
+
+plain_text_description:
+- description with all HTML/markdown stripped, no formatting characters.
+
+categories:
+- Semicolon-separated retail classification labels.
+- Use ONLY classification labels — "Fashion", "Food & Beverage", "Beauty & Personal Care",
+  "Services", "Home", "Accessories", "Electronics", "Health & Wellness", "Restaurant", etc.
+- 1-4 categories typical. Don't put product names here.
+
+location_type:
+- e.g. "Restaurant", "Kiosk", "Cart", "In-line store", "Anchor", "Outparcel", "Service".
+
+location_information:
+- Free text about where the store is in the mall ("Upper level, near Centre Court", etc.)
+
+unit:
+- Unit number if present (e.g. "Unit 215").
+
+parking_information:
+- Parking notes specific to this store.
+
+restaurant fields:
+- is_restaurant: true if food/beverage establishment.
+- restaurant_cuisines: cuisine type ("Mexican", "Sushi", "Coffee", etc.)
+- menu_url: link to menu if visible.
+- restaurant_info: extra restaurant details (takeout, delivery, alcohol, etc).
+
+status flags (set ONLY if the page explicitly says so):
+- is_new_store, is_coming_soon_store, is_relocated_store, is_temporarily_closed,
+  exterior_retailer.
+
+tags:
+- ONLY explicit keyword tags/hashtags. NOT categories, NOT product types.
+- If unsure, leave "".
+
+products: semicolon-separated product types the store sells.
+service: services offered (e.g. "Alterations;Personal Shopping").
+brand_description: paragraph describing the brand.
+amenities: semicolon-separated (e.g. "Wi-Fi;Wheelchair accessible").
+payment_methods: semicolon-separated (e.g. "Visa;Mastercard;Cash").
+return_policy: return policy text.
+
+confidence: float 0-1 reflecting fill rate of the requested fields.`;
+
+function buildUserPrompt({
+  url, urlSlug, h1, title, hoursCanonical,
+  jsonLd, metaTags, interceptedJson, textContent,
+}) {
+  const parts = [];
+  parts.push(`URL: ${url}`);
+  parts.push(`URL_SLUG: ${urlSlug || '(none)'}`);
+  parts.push(`H1_TEXT: ${h1 || '(none)'}`);
+  parts.push(`PAGE_TITLE: ${title || '(none)'}`);
+  parts.push(`HOURS_ALREADY_EXTRACTED: ${hoursCanonical || '(none)'}`);
+
+  if (jsonLd && jsonLd.length) {
+    parts.push('\n--- JSON-LD ---');
+    parts.push(JSON.stringify(jsonLd, null, 2).slice(0, 3000));
+  }
+  if (metaTags && Object.keys(metaTags).length) {
+    parts.push('\n--- Meta Tags ---');
+    parts.push(JSON.stringify(metaTags, null, 2).slice(0, 1200));
+  }
+  if (interceptedJson && interceptedJson.length) {
+    parts.push('\n--- Intercepted XHR JSON (truncated) ---');
+    for (const j of interceptedJson.slice(0, 3)) parts.push(String(j).slice(0, 2000));
+  }
+  parts.push('\n--- Page Text ---');
+  parts.push((textContent || '').slice(0, 10000));
+
+  return parts.join('\n');
+}
+
+class StoreExtractor {
+  constructor({ client, model = 'gpt-5.4-mini', useVision = false, logger }) {
+    this.client = client;
+    this.model = model;
+    this.useVision = useVision;
+    this.logger = logger;
+    this.totalTokensInput = 0;
+    this.totalTokensOutput = 0;
+    this.totalCost = 0;
+    this.extractionCount = 0;
+  }
+
+  async extract(pageData, hoursCanonical) {
+    const userText = buildUserPrompt({ ...pageData, hoursCanonical });
+
+    const isGpt5Family = /^gpt-5(\.|-|$)/i.test(this.model);
+    const messages = [
+      { role: 'system', content: SYSTEM_PROMPT },
+      this._buildUserMessage(userText, pageData.screenshotBase64),
+    ];
+    const params = {
+      model: this.model,
+      messages,
+      response_format: { type: 'json_object' },
+    };
+    if (isGpt5Family) params.max_completion_tokens = 1800;
+    else { params.max_tokens = 1800; params.temperature = 0.0; }
+
+    let raw;
+    try {
+      const resp = await this.client.chat.completions.create(params);
+      this._trackUsage(resp);
+      raw = JSON.parse(resp.choices[0].message.content);
+    } catch (err) {
+      if (this.logger) this.logger.warn(`   ⚠ Store LLM extract failed: ${err.message}`);
+      return { fields: {}, confidence: 0 };
+    }
+
+    const fields = this._sanitize(raw, pageData);
+    const confidence = typeof raw.confidence === 'number'
+      ? Math.min(1, Math.max(0, raw.confidence))
+      : estimateConfidence(fields);
+
+    this.extractionCount++;
+    return { fields, confidence };
+  }
+
+  _buildUserMessage(text, screenshotBase64) {
+    if (!this.useVision || !screenshotBase64) {
+      return { role: 'user', content: `Extract the JSON now:\n\n${text}` };
+    }
+    return {
+      role: 'user',
+      content: [
+        { type: 'text', text: `Extract the JSON now (screenshot for visual context):\n\n${text}` },
+        { type: 'image_url', image_url: { url: `data:image/png;base64,${screenshotBase64}`, detail: 'low' } },
+      ],
+    };
+  }
+
+  _sanitize(raw, _pageData) {
+    const out = {};
+    const stringFields = [
+      'description', 'brief_description', 'plain_text_description',
+      'categories', 'location_type', 'location_information', 'unit',
+      'parking_information', 'restaurant_cuisines', 'menu_url', 'restaurant_info',
+      'tags', 'products', 'service', 'brand_description', 'amenities',
+      'payment_methods', 'return_policy', 'store_recommendations',
+    ];
+    const boolFields = [
+      'is_restaurant', 'is_new_store', 'is_coming_soon_store',
+      'is_relocated_store', 'is_temporarily_closed', 'exterior_retailer',
+    ];
+
+    for (const f of stringFields) {
+      if (f in raw) out[f] = sanitizeString(raw[f]);
+    }
+    for (const f of boolFields) {
+      if (f in raw) out[f] = Boolean(raw[f]);
+    }
+
+    return out;
+  }
+
+  _trackUsage(response) {
+    if (!response.usage) return;
+    const { prompt_tokens, completion_tokens } = response.usage;
+    this.totalTokensInput += prompt_tokens || 0;
+    this.totalTokensOutput += completion_tokens || 0;
+    const p = MODEL_PRICING[this.model];
+    if (p) {
+      this.totalCost += ((prompt_tokens || 0) / 1000) * p.input
+                      + ((completion_tokens || 0) / 1000) * p.output;
+    }
+  }
+
+  getUsageSummary() {
+    return {
+      model: this.model,
+      extractions: this.extractionCount,
+      totalInputTokens: this.totalTokensInput,
+      totalOutputTokens: this.totalTokensOutput,
+      estimatedCost: `$${this.totalCost.toFixed(4)}`,
+    };
+  }
+}
+
+function sanitizeString(v) {
+  if (v == null) return '';
+  return String(v).replace(/[\r\n]+/g, ' ').replace(/\s{2,}/g, ' ').trim();
+}
+
+function estimateConfidence(fields) {
+  const key = ['description', 'categories', 'location_type'];
+  const populated = key.filter(f => fields[f] && fields[f].length > 0).length;
+  return populated / key.length;
+}
+
+module.exports = { StoreExtractor, MODEL_PRICING };

package/src/storeModel.js

@@ -0,0 +1,147 @@
+'use strict';
+
+const STORE_FIELDS = [
+  'mm_id',
+  'name',
+  'website',
+  'description',
+  'categories',
+  'phone',
+  'location_type',
+  'store_recommendations',
+  'logo_image_url',
+  'brand_image_url',
+  'store_front_image_url',
+  'is_new_store',
+  'is_coming_soon_store',
+  'is_relocated_store',
+  'exterior_retailer',
+  'is_temporarily_closed',
+  'is_restaurant',
+  'restaurant_cuisines',
+  'menu_url',
+  'restaurant_info',
+  'tags',
+  'sync_with_centre_hours',
+  'is_free_form_hours',
+  'free_form_hours',
+  'store_hours',
+  'hours_source',
+  'hours_confidence',
+  'location_information',
+  'parking_information',
+  'products',
+  'service',
+  'brand_description',
+  'amenities',
+  'payment_methods',
+  'return_policy',
+  'unit',
+  'facebook',
+  'twitter',
+  'instagram',
+  'youtube',
+  'tiktok',
+  'pinterest',
+];
+
+const BOOLEAN_FIELDS = new Set([
+  'is_new_store',
+  'is_coming_soon_store',
+  'is_relocated_store',
+  'exterior_retailer',
+  'is_temporarily_closed',
+  'is_restaurant',
+  'sync_with_centre_hours',
+  'is_free_form_hours',
+]);
+
+const NUMERIC_FIELDS = new Set(['hours_confidence']);
+
+// Diagnostic fields kept in JSON but excluded from CSV (not needed by the CMS).
+const CSV_EXCLUDE_FIELDS = new Set(['hours_source', 'hours_confidence']);
+
+function createStore(mmId) {
+  const store = { mm_id: mmId };
+  for (const field of STORE_FIELDS) {
+    if (field === 'mm_id') continue;
+    if (BOOLEAN_FIELDS.has(field)) store[field] = false;
+    else if (NUMERIC_FIELDS.has(field)) store[field] = 0;
+    else store[field] = '';
+  }
+  return store;
+}
+
+function mergeExtracted(mmId, aiData) {
+  const store = createStore(mmId);
+  if (!aiData || typeof aiData !== 'object') return store;
+
+  for (const field of STORE_FIELDS) {
+    if (field === 'mm_id') continue;
+    if (!(field in aiData)) continue;
+    const val = aiData[field];
+
+    if (BOOLEAN_FIELDS.has(field)) {
+      if (typeof val === 'boolean') store[field] = val;
+      else if (typeof val === 'string') store[field] = val.toLowerCase() === 'true' || val === '1';
+      else store[field] = Boolean(val);
+    } else if (NUMERIC_FIELDS.has(field)) {
+      const n = typeof val === 'number' ? val : parseFloat(val);
+      store[field] = Number.isFinite(n) ? n : 0;
+    } else {
+      store[field] = val == null ? '' : String(val).replace(/[\r\n]+/g, ' ').replace(/\s{2,}/g, ' ').trim();
+    }
+  }
+  return store;
+}
+
+/**
+ * Escape a CSV cell.
+ *
+ * Modes:
+ *   - alwaysQuote=true : wrap the value in double quotes even when not strictly
+ *     required. Defends against upload tools whose CSV parsers don't handle
+ *     bare URLs / mixed-content cells well — the cell is unambiguously bounded
+ *     by quote characters so the parser can't accidentally swallow trailing
+ *     fields into the URL.
+ *   - alwaysQuote=false : RFC-4180 minimal quoting (only when the cell
+ *     contains a separator, quote, or newline).
+ */
+function csvCell(val, { alwaysQuote = false } = {}) {
+  const s = val == null ? '' : String(val);
+  const mustQuote = s.includes(',') || s.includes('"') || s.includes('\n') || s.includes('\r');
+  if (alwaysQuote || mustQuote) {
+    return '"' + s.replace(/"/g, '""') + '"';
+  }
+  return s;
+}
+
+/**
+ * Emit CSV. By default strings are always-quoted (defensive) and we use CRLF
+ * line endings plus a UTF-8 BOM, which gives the most reliable behavior across
+ * Excel, Google Sheets, and naive in-house CSV parsers.
+ *
+ * Booleans and numeric fields are left unquoted so destination systems can
+ * type-detect them.
+ */
+function storesToCSV(stores, { lineEnding = '\r\n', bom = true, alwaysQuoteStrings = true } = {}) {
+  const csvFields = STORE_FIELDS.filter(f => !CSV_EXCLUDE_FIELDS.has(f));
+  const formatCell = (field, val) => {
+    if (BOOLEAN_FIELDS.has(field) || NUMERIC_FIELDS.has(field)) {
+      return csvCell(val, { alwaysQuote: false });
+    }
+    return csvCell(val, { alwaysQuote: alwaysQuoteStrings });
+  };
+  const headerLine = csvFields
+    .map(f => csvCell(f, { alwaysQuote: alwaysQuoteStrings }))
+    .join(',');
+  if (!stores || stores.length === 0) {
+    return (bom ? '' : '') + headerLine + lineEnding;
+  }
+  const rows = stores.map(store =>
+    csvFields.map(f => formatCell(f, store[f] == null ? '' : store[f])).join(',')
+  );
+  return (bom ? '' : '') + [headerLine, ...rows].join(lineEnding) + lineEnding;
+}
+
+module.exports = { STORE_FIELDS, BOOLEAN_FIELDS, NUMERIC_FIELDS, CSV_EXCLUDE_FIELDS, createStore, mergeExtracted, storesToCSV };