mallmaverick-store-scraper 0.1.0 → 0.1.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mallmaverick-store-scraper",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "MCP server + CLI for scraping shopping mall store directories. Hours-first layered pipeline + image classification.",
   "main": "src/main.js",
   "type": "commonjs",

package/src/mcp-server.js

@@ -20,6 +20,7 @@
 
 require('dotenv').config();
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
 const { URL } = require('url');
 const http = require('http');
@@ -58,7 +59,12 @@ const TOOLS = [
       'Scrape a shopping-mall store directory and return per-store records ' +
       '(name, hours, phone, logo, brand image, categories, etc.). Use this ' +
       'when the user wants to capture a directory like ' +
-      'https://grasslands.ca/store-directory/.',
+      'https://grasslands.ca/store-directory/.\n\n' +
+      'AFTER RUNNING THIS TOOL: paste the full CSV content (from the ' +
+      '"--- CSV ---" block of the response) into your reply inside a fenced ' +
+      'code block so the user can copy it directly into their CMS. ' +
+      'Also state the saved file path and a one-line summary of how many ' +
+      'stores were extracted. Do NOT summarize away the CSV — show it in full.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -123,8 +129,10 @@ const TOOLS = [
   },
 ];
 
+const PACKAGE_VERSION = '0.1.2';
+
 const server = new Server(
-  { name: 'mall-scraper-mcp', version: '0.1.0' },
+  { name: 'mall-scraper-mcp', version: PACKAGE_VERSION },
   { capabilities: { tools: {} } }
 );
 
@@ -190,9 +198,19 @@ async function handleScrapeDirectory({ directory_url, max_stores = 10, concurren
     await Promise.all(tasks);
     stores.sort((a, b) => a.mm_id - b.mm_id);
 
+    // Generate the CSV regardless of whether we manage to write it to disk —
+    // it's always returned inline so the user gets it back automatically.
+    const csvText = storesToCSV(stores);
+
     let writtenPaths = null;
+    let writeError = null;
     if (write_csv) {
-      writtenPaths = writeResults(directory_url, stores);
+      try {
+        writtenPaths = writeResults(directory_url, stores, csvText);
+      } catch (err) {
+        writeError = err.message;
+        // Don't fail the tool — the CSV is still returned inline below.
+      }
     }
 
     const bySource = {};
@@ -208,13 +226,25 @@ async function handleScrapeDirectory({ directory_url, max_stores = 10, concurren
       hours_layer_breakdown: bySource,
       llm_usage: usage,
       written_files: writtenPaths,
+      write_error: writeError,
       auth_mode: creds.mode,
+      mcp_version: PACKAGE_VERSION,
     };
 
+    // Order matters — Claude is more likely to surface the first content
+    // blocks. Lead with the CSV so it can't be summarized away.
     return {
       content: [
-        { type: 'text', text: JSON.stringify(summary, null, 2) },
-        { type: 'text', text: '\nStores:\n' + JSON.stringify(stores, null, 2) },
+        {
+          type: 'text',
+          text:
+            `mall-scraper-mcp v${PACKAGE_VERSION}\n` +
+            'CSV ready — paste the block below into your CMS. ' +
+            `Also saved to: ${writtenPaths ? writtenPaths.csv : '(disk write failed; CSV is inline only)'}.\n\n` +
+            '```csv\n' + csvText + '\n```',
+        },
+        { type: 'text', text: '\n--- Run summary ---\n' + JSON.stringify(summary, null, 2) },
+        { type: 'text', text: '\n--- Stores (JSON for debugging) ---\n' + JSON.stringify(stores, null, 2) },
       ],
     };
   } finally {
@@ -393,15 +423,47 @@ function slugToName(slug) {
   return slug.split('-').map(w => w.charAt(0).toUpperCase() + w.slice(1)).join(' ');
 }
 
-function writeResults(directoryUrl, stores) {
-  const outDir = path.join(process.cwd(), 'extracted_stores');
-  fs.mkdirSync(outDir, { recursive: true });
+/**
+ * Pick an output directory that's reliably writable when the MCP is spawned
+ * by Claude Desktop / Claude Code (process.cwd() is often unwritable).
+ *
+ * Preference order:
+ *   1. MALL_SCRAPER_OUTPUT_DIR env var (explicit user choice)
+ *   2. ~/Documents/mall-scraper-output (Mac/Win — most discoverable)
+ *   3. ~/mall-scraper-output (Linux / Documents unavailable)
+ *   4. os.tmpdir()/mall-scraper-output (last resort)
+ */
+function pickOutDir() {
+  const candidates = [
+    process.env.MALL_SCRAPER_OUTPUT_DIR,
+    path.join(os.homedir(), 'Documents', 'mall-scraper-output'),
+    path.join(os.homedir(), 'mall-scraper-output'),
+    path.join(os.tmpdir(), 'mall-scraper-output'),
+  ].filter(Boolean);
+
+  for (const dir of candidates) {
+    try {
+      fs.mkdirSync(dir, { recursive: true });
+      // Probe write access by touching a file
+      const probe = path.join(dir, '.write-probe');
+      fs.writeFileSync(probe, '');
+      fs.unlinkSync(probe);
+      return dir;
+    } catch (_) {
+      continue;
+    }
+  }
+  throw new Error('No writable output directory found');
+}
+
+function writeResults(directoryUrl, stores, csvText) {
+  const outDir = pickOutDir();
   const host = new URL(directoryUrl).hostname.replace(/^www\./, '');
   const ts = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
   const base = path.join(outDir, `stores_v5_${host}_${ts}`);
   fs.writeFileSync(`${base}.json`, JSON.stringify(stores, null, 2));
-  fs.writeFileSync(`${base}.csv`, storesToCSV(stores));
-  return { json: `${base}.json`, csv: `${base}.csv` };
+  fs.writeFileSync(`${base}.csv`, csvText);
+  return { json: `${base}.json`, csv: `${base}.csv`, dir: outDir };
 }
 
 function errorResult(message) {