mallmaverick-store-scraper 0.1.3 → 0.1.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mallmaverick-store-scraper",
-  "version": "0.1.3",
+  "version": "0.1.4",
   "description": "MCP server + CLI for scraping shopping mall store directories. Hours-first layered pipeline + image classification.",
   "main": "src/main.js",
   "type": "commonjs",

package/src/mcp-server.js

@@ -113,6 +113,15 @@ const TOOLS = [
       required: ['store_url'],
     },
   },
+  {
+    name: 'check_status',
+    description:
+      'Returns the running mall-scraper-mcp version, auth mode, and Worker ' +
+      'connectivity. Use this BEFORE running scrape_directory to verify the ' +
+      'tool is wired up correctly — confirms version, that the OpenAI proxy ' +
+      'is reachable, and that the shared secret is valid.',
+    inputSchema: { type: 'object', properties: {} },
+  },
   {
     name: 'validate_image_url',
     description:
@@ -129,7 +138,7 @@ const TOOLS = [
   },
 ];
 
-const PACKAGE_VERSION = '0.1.3';
+const PACKAGE_VERSION = '0.1.4';
 
 const server = new Server(
   { name: 'mall-scraper-mcp', version: PACKAGE_VERSION },
@@ -145,6 +154,7 @@ server.setRequestHandler(CallToolRequestSchema, async (req) => {
       case 'scrape_directory': return await handleScrapeDirectory(args || {});
       case 'get_store_hours':  return await handleGetStoreHours(args || {});
       case 'validate_image_url': return await handleValidateImageUrl(args || {});
+      case 'check_status':     return await handleCheckStatus(args || {});
       default:
         return errorResult(`Unknown tool: ${name}`);
     }
@@ -225,6 +235,9 @@ async function handleScrapeDirectory({ directory_url, max_stores = 10, concurren
       stores_extracted: stores.length,
       hours_layer_breakdown: bySource,
       llm_usage: usage,
+      llm_failed: usage.errors > 0
+        ? `⚠ ${usage.errors} LLM calls failed (description/categories/etc. fields will be empty). Last error: ${usage.lastError}. Run check_status to diagnose.`
+        : null,
       written_files: writtenPaths,
       write_error: writeError,
       auth_mode: creds.mode,
@@ -321,6 +334,86 @@ async function handleGetStoreHours({ store_url, mall_root_url }) {
   }
 }
 
+async function handleCheckStatus() {
+  const creds = describeCredentials();
+  const status = {
+    mcp_version: PACKAGE_VERSION,
+    node_version: process.version,
+    auth_mode: creds.mode,
+    auth_endpoint: creds.endpoint,
+    worker_reachable: null,
+    worker_health: null,
+    worker_auth_ok: null,
+  };
+
+  // If we're in proxy mode, ping the Worker /health endpoint and probe auth.
+  if (creds.mode === 'proxy' && creds.endpoint) {
+    try {
+      const healthUrl = creds.endpoint.replace(/\/+$/, '') + '/health';
+      const health = await new Promise((resolve) => {
+        const req = https.get(healthUrl, { timeout: 6000 }, (res) => {
+          let body = '';
+          res.on('data', (c) => { body += c; });
+          res.on('end', () => resolve({ status: res.statusCode, body }));
+        });
+        req.on('error', () => resolve(null));
+        req.on('timeout', () => { req.destroy(); resolve(null); });
+      });
+      if (health) {
+        status.worker_reachable = true;
+        status.worker_health = health.body.slice(0, 200);
+      } else {
+        status.worker_reachable = false;
+      }
+
+      // Probe auth: a tiny POST to /v1/models with the shared secret.
+      // OpenAI's /v1/models is a cheap, no-tokens endpoint that proves the
+      // Worker is forwarding and the key works.
+      const token = process.env.MALL_SCRAPER_TOKEN || '';
+      const modelsUrl = creds.endpoint.replace(/\/+$/, '') + '/v1/models';
+      const auth = await new Promise((resolve) => {
+        const req = https.get(modelsUrl, {
+          timeout: 8000,
+          headers: { 'X-Mall-Scraper-Token': token },
+        }, (res) => {
+          let body = '';
+          res.on('data', (c) => { body += c; });
+          res.on('end', () => resolve({ status: res.statusCode, body }));
+        });
+        req.on('error', () => resolve(null));
+        req.on('timeout', () => { req.destroy(); resolve(null); });
+      });
+      if (auth) {
+        status.worker_auth_ok = auth.status === 200;
+        if (auth.status !== 200) {
+          status.worker_auth_error = auth.body.slice(0, 200);
+        }
+      }
+    } catch (err) {
+      status.worker_probe_error = err.message;
+    }
+  }
+
+  // Verdict line for the user
+  let verdict;
+  if (creds.mode === 'none') {
+    verdict = '⚠ No credentials configured. Set MALL_SCRAPER_PROXY_URL+MALL_SCRAPER_TOKEN or OPENAI_API_KEY.';
+  } else if (creds.mode === 'proxy') {
+    if (status.worker_reachable && status.worker_auth_ok) verdict = '✅ All good — version, Worker, and auth all working.';
+    else if (!status.worker_reachable) verdict = '⚠ Worker is unreachable. Check MALL_SCRAPER_PROXY_URL.';
+    else if (!status.worker_auth_ok) verdict = '⚠ Worker is reachable but rejected the token. MALL_SCRAPER_TOKEN does not match the SHARED_SECRET on the Worker.';
+    else verdict = '⚠ Partial — see fields below.';
+  } else {
+    verdict = `✅ Direct mode (using OPENAI_API_KEY env var). Version ${PACKAGE_VERSION}.`;
+  }
+
+  return {
+    content: [
+      { type: 'text', text: verdict + '\n\n' + JSON.stringify(status, null, 2) },
+    ],
+  };
+}
+
 function handleValidateImageUrl({ url }) {
   if (!url) return Promise.resolve(errorResult('url is required'));
   return new Promise((resolve) => {

package/src/storeExtractor.js

@@ -128,6 +128,8 @@ class StoreExtractor {
     this.totalTokensOutput = 0;
     this.totalCost = 0;
     this.extractionCount = 0;
+    this.errorCount = 0;
+    this.lastError = null;
   }
 
   async extract(pageData, hoursCanonical) {
@@ -152,6 +154,8 @@ class StoreExtractor {
       this._trackUsage(resp);
       raw = JSON.parse(resp.choices[0].message.content);
     } catch (err) {
+      this.errorCount++;
+      this.lastError = err.message;
       if (this.logger) this.logger.warn(`   ⚠ Store LLM extract failed: ${err.message}`);
       return { fields: {}, confidence: 0 };
     }
@@ -218,6 +222,8 @@ class StoreExtractor {
     return {
       model: this.model,
       extractions: this.extractionCount,
+      errors: this.errorCount,
+      lastError: this.lastError,
       totalInputTokens: this.totalTokensInput,
       totalOutputTokens: this.totalTokensOutput,
       estimatedCost: `$${this.totalCost.toFixed(4)}`,