mallmaverick-store-scraper 0.1.3 → 0.1.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mallmaverick-store-scraper",
-  "version": "0.1.3",
+  "version": "0.1.5",
   "description": "MCP server + CLI for scraping shopping mall store directories. Hours-first layered pipeline + image classification.",
   "main": "src/main.js",
   "type": "commonjs",

package/src/mcp-server.js

@@ -60,11 +60,11 @@ const TOOLS = [
       '(name, hours, phone, logo, brand image, categories, etc.). Use this ' +
       'when the user wants to capture a directory like ' +
       'https://grasslands.ca/store-directory/.\n\n' +
-      'AFTER RUNNING THIS TOOL: give a short summary — how many stores were ' +
-      'extracted, hours-layer breakdown, and the saved file path. The CSV is ' +
-      'attached as a resource in the tool response (the user can download/' +
-      'preview it from there) — do NOT paste the CSV text into your reply. ' +
-      'Keep your text reply short.',
+      'AFTER RUNNING THIS TOOL: reply with ONE short sentence — the count ' +
+      'and the file path. The CSV file is attached as a resource_link in the ' +
+      'tool response; do NOT paste CSV text into your reply, do NOT print the ' +
+      'JSON, do NOT summarize each store. If the tool response includes an ' +
+      'error block, surface that error verbatim to the user.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -113,6 +113,15 @@ const TOOLS = [
       required: ['store_url'],
     },
   },
+  {
+    name: 'check_status',
+    description:
+      'Returns the running mall-scraper-mcp version, auth mode, and Worker ' +
+      'connectivity. Use this BEFORE running scrape_directory to verify the ' +
+      'tool is wired up correctly — confirms version, that the OpenAI proxy ' +
+      'is reachable, and that the shared secret is valid.',
+    inputSchema: { type: 'object', properties: {} },
+  },
   {
     name: 'validate_image_url',
     description:
@@ -129,7 +138,7 @@ const TOOLS = [
   },
 ];
 
-const PACKAGE_VERSION = '0.1.3';
+const PACKAGE_VERSION = '0.1.5';
 
 const server = new Server(
   { name: 'mall-scraper-mcp', version: PACKAGE_VERSION },
@@ -145,6 +154,7 @@ server.setRequestHandler(CallToolRequestSchema, async (req) => {
       case 'scrape_directory': return await handleScrapeDirectory(args || {});
       case 'get_store_hours':  return await handleGetStoreHours(args || {});
       case 'validate_image_url': return await handleValidateImageUrl(args || {});
+      case 'check_status':     return await handleCheckStatus(args || {});
       default:
         return errorResult(`Unknown tool: ${name}`);
     }
@@ -225,52 +235,82 @@ async function handleScrapeDirectory({ directory_url, max_stores = 10, concurren
       stores_extracted: stores.length,
       hours_layer_breakdown: bySource,
       llm_usage: usage,
+      llm_failed: usage.errors > 0
+        ? `⚠ ${usage.errors} LLM calls failed (description/categories/etc. fields will be empty). Last error: ${usage.lastError}. Run check_status to diagnose.`
+        : null,
       written_files: writtenPaths,
       write_error: writeError,
       auth_mode: creds.mode,
       mcp_version: PACKAGE_VERSION,
     };
 
-    // Build a short brief + return the CSV as an embedded resource so
-    // Claude Desktop can render it as an attachment card instead of inline
-    // text. Falls back to inline-rendering if the client doesn't support
-    // resources, but most clients (including Claude Desktop) do.
+    // Response design:
+    //   1. Brief status line (always) — what the user sees in the chat reply
+    //   2. resource_link to the CSV — file attachment with user-priority annotations
+    //   3. ONLY on error: a loud error block so the user knows something failed
+    //
+    // No JSON dump / no inline CSV preview when things succeed — keeps the chat
+    // reply minimal.
     const host = new URL(directory_url).hostname.replace(/^www\./, '');
     const csvFilename = writtenPaths
       ? path.basename(writtenPaths.csv)
       : `stores_v5_${host}.csv`;
     const csvUri = writtenPaths
       ? `file://${writtenPaths.csv}`
-      : `file:///tmp/${csvFilename}`;
+      : null;
+
+    const hasLlmFailure = usage.errors > 0;
+    const hasWriteFailure = !!writeError;
+    const anyFailure = hasLlmFailure || hasWriteFailure;
 
     const brief =
-      `mall-scraper-mcp v${PACKAGE_VERSION}\n` +
-      `Scraped ${stores.length} store${stores.length === 1 ? '' : 's'} from ${host}.\n` +
-      `Hours-layer breakdown: ${Object.entries(bySource).map(([k, v]) => `${k}=${v}`).join(', ')}.\n` +
-      (writtenPaths
-        ? `Saved to: ${writtenPaths.csv}`
-        : `⚠ disk write failed (${writeError}); CSV is in the attached resource only.`);
-
-    return {
-      content: [
-        { type: 'text', text: brief },
-        {
-          type: 'resource',
-          resource: {
-            uri: csvUri,
-            name: csvFilename,
-            mimeType: 'text/csv',
-            text: csvText,
-          },
-        },
-        // Keep the JSON summary at the end for any debugging the user asks for,
-        // but it's far enough down that it doesn't dominate the chat.
-        {
-          type: 'text',
-          text: '\n--- Run summary ---\n' + JSON.stringify(summary, null, 2),
-        },
-      ],
-    };
+      `✅ mall-scraper-mcp v${PACKAGE_VERSION}\n` +
+      `${stores.length} store${stores.length === 1 ? '' : 's'} from ${host}\n` +
+      (writtenPaths ? `📄 ${writtenPaths.csv}` : '⚠ Disk write failed');
+
+    const content = [
+      {
+        type: 'text',
+        text: brief,
+        annotations: { audience: ['user'], priority: 1.0 },
+      },
+    ];
+
+    // resource_link only if we have a real file path (file:// URI must point at
+    // an existing file for the client to do anything useful with it).
+    if (csvUri) {
+      content.push({
+        type: 'resource_link',
+        uri: csvUri,
+        name: csvFilename,
+        description: `Store directory scrape — ${stores.length} stores from ${host}`,
+        mimeType: 'text/csv',
+        annotations: { audience: ['user'], priority: 0.9 },
+      });
+    }
+
+    // Loud error block — only when something failed. The user explicitly asked
+    // for nothing other than a status rundown UNLESS something broke.
+    if (anyFailure) {
+      const errLines = [];
+      if (hasLlmFailure) {
+        errLines.push(
+          `❌ ${usage.errors} LLM call${usage.errors === 1 ? '' : 's'} failed: ${usage.lastError}`,
+          '   → description / categories / location_type fields will be empty.',
+          '   → Run check_status to diagnose (most likely the Worker token doesn\'t match the SHARED_SECRET).',
+        );
+      }
+      if (hasWriteFailure) {
+        errLines.push(`❌ Disk write failed: ${writeError}`);
+      }
+      content.push({
+        type: 'text',
+        text: '\n' + errLines.join('\n'),
+        annotations: { audience: ['user'], priority: 1.0 },
+      });
+    }
+
+    return { content };
   } finally {
     try { await browser.close(); } catch (_) {}
   }
@@ -321,6 +361,86 @@ async function handleGetStoreHours({ store_url, mall_root_url }) {
   }
 }
 
+async function handleCheckStatus() {
+  const creds = describeCredentials();
+  const status = {
+    mcp_version: PACKAGE_VERSION,
+    node_version: process.version,
+    auth_mode: creds.mode,
+    auth_endpoint: creds.endpoint,
+    worker_reachable: null,
+    worker_health: null,
+    worker_auth_ok: null,
+  };
+
+  // If we're in proxy mode, ping the Worker /health endpoint and probe auth.
+  if (creds.mode === 'proxy' && creds.endpoint) {
+    try {
+      const healthUrl = creds.endpoint.replace(/\/+$/, '') + '/health';
+      const health = await new Promise((resolve) => {
+        const req = https.get(healthUrl, { timeout: 6000 }, (res) => {
+          let body = '';
+          res.on('data', (c) => { body += c; });
+          res.on('end', () => resolve({ status: res.statusCode, body }));
+        });
+        req.on('error', () => resolve(null));
+        req.on('timeout', () => { req.destroy(); resolve(null); });
+      });
+      if (health) {
+        status.worker_reachable = true;
+        status.worker_health = health.body.slice(0, 200);
+      } else {
+        status.worker_reachable = false;
+      }
+
+      // Probe auth: a tiny POST to /v1/models with the shared secret.
+      // OpenAI's /v1/models is a cheap, no-tokens endpoint that proves the
+      // Worker is forwarding and the key works.
+      const token = process.env.MALL_SCRAPER_TOKEN || '';
+      const modelsUrl = creds.endpoint.replace(/\/+$/, '') + '/v1/models';
+      const auth = await new Promise((resolve) => {
+        const req = https.get(modelsUrl, {
+          timeout: 8000,
+          headers: { 'X-Mall-Scraper-Token': token },
+        }, (res) => {
+          let body = '';
+          res.on('data', (c) => { body += c; });
+          res.on('end', () => resolve({ status: res.statusCode, body }));
+        });
+        req.on('error', () => resolve(null));
+        req.on('timeout', () => { req.destroy(); resolve(null); });
+      });
+      if (auth) {
+        status.worker_auth_ok = auth.status === 200;
+        if (auth.status !== 200) {
+          status.worker_auth_error = auth.body.slice(0, 200);
+        }
+      }
+    } catch (err) {
+      status.worker_probe_error = err.message;
+    }
+  }
+
+  // Verdict line for the user
+  let verdict;
+  if (creds.mode === 'none') {
+    verdict = '⚠ No credentials configured. Set MALL_SCRAPER_PROXY_URL+MALL_SCRAPER_TOKEN or OPENAI_API_KEY.';
+  } else if (creds.mode === 'proxy') {
+    if (status.worker_reachable && status.worker_auth_ok) verdict = '✅ All good — version, Worker, and auth all working.';
+    else if (!status.worker_reachable) verdict = '⚠ Worker is unreachable. Check MALL_SCRAPER_PROXY_URL.';
+    else if (!status.worker_auth_ok) verdict = '⚠ Worker is reachable but rejected the token. MALL_SCRAPER_TOKEN does not match the SHARED_SECRET on the Worker.';
+    else verdict = '⚠ Partial — see fields below.';
+  } else {
+    verdict = `✅ Direct mode (using OPENAI_API_KEY env var). Version ${PACKAGE_VERSION}.`;
+  }
+
+  return {
+    content: [
+      { type: 'text', text: verdict + '\n\n' + JSON.stringify(status, null, 2) },
+    ],
+  };
+}
+
 function handleValidateImageUrl({ url }) {
   if (!url) return Promise.resolve(errorResult('url is required'));
   return new Promise((resolve) => {

package/src/storeExtractor.js

@@ -128,6 +128,8 @@ class StoreExtractor {
     this.totalTokensOutput = 0;
     this.totalCost = 0;
     this.extractionCount = 0;
+    this.errorCount = 0;
+    this.lastError = null;
   }
 
   async extract(pageData, hoursCanonical) {
@@ -152,6 +154,8 @@ class StoreExtractor {
       this._trackUsage(resp);
       raw = JSON.parse(resp.choices[0].message.content);
     } catch (err) {
+      this.errorCount++;
+      this.lastError = err.message;
       if (this.logger) this.logger.warn(`   ⚠ Store LLM extract failed: ${err.message}`);
       return { fields: {}, confidence: 0 };
     }
@@ -218,6 +222,8 @@ class StoreExtractor {
     return {
       model: this.model,
       extractions: this.extractionCount,
+      errors: this.errorCount,
+      lastError: this.lastError,
       totalInputTokens: this.totalTokensInput,
       totalOutputTokens: this.totalTokensOutput,
       estimatedCost: `$${this.totalCost.toFixed(4)}`,