lacspace-leads 1.6.0 → 1.7.0

package/README.md

@@ -59,7 +59,12 @@ npx lacspace-leads [type] [options]
 | `--enrich-out <file>` | Also write `{ name, website, domain }` NDJSON, ready to feed [`lacspace-enrich`](https://www.npmjs.com/package/lacspace-enrich) |
 | `--resume` | Resume an interrupted **multi-search sweep** from its checkpoint file (`<out>.checkpoint.json`) |
 | `--sheet <name>` | Excel sheet name (default `Leads`) |
-| `-n, --limit <n>` | Max listings **per search** (default `60`) |
+| `-n, --limit <n>` | Max listings **per search** (default `60`; Google itself stops near 120) |
+| `--target <n>` | How many leads you want **in total** — keeps searching until it has them. Use this for 300, 500, 1000 |
+| `--step <dist>` | Spacing between map tiles in a target sweep (default `2.5km`) |
+| `--tiles <n>` | Most map tiles to try (default `49`) |
+| `--split <key>` | Write one file per `city`, `area` or `type` alongside the main file |
+| `--no-website` | Only businesses with **no website** — the "you need a site" pitch list |
 | `--total <n>` | Cap the merged result when sweeping several searches |
 | `--no-details` | Skip opening each listing — names + Maps URLs only, much faster |
 | `--sort <key>` | `rating` · `reviews` · `name` · `priceLevel` · `distance` (missing values last) |
@@ -111,6 +116,29 @@ npx lacspace-leads [type] [options]
 
 Run with no arguments for an interactive walkthrough.
 
+## Ask for a number and actually get it (`--target`)
+
+One Google Maps search stops at roughly **120 results**, so `--limit 500` can only ever hand back about 114. `--target` is the honest version: say how many leads you want in total and the tool keeps going until it has them.
+
+```bash
+# 500 restaurants, however many searches that takes
+npx lacspace-leads restaurants --city Kathmandu --target 500 -f xlsx
+```
+
+It expands coverage in this order, stopping the moment the target is met:
+
+1. **Every place you named** — each city x area x type is its own search.
+2. **Map tiles** — the same query re-centred on a grid of points around the city, walking outwards from the middle. No geocoding service and no API key: the centre comes from the map itself.
+
+Every step is de-duplicated against everything collected so far, so the number you get is unique businesses, not rows. If the map runs dry before the target, it says so plainly instead of pretending:
+
+```
+! The map ran out of new results at 214 — that is everything Google lists here.
+  Widen it: more areas, more cities, a bigger --step, or related --types.
+```
+
+Tune the grid with `--step` (tighter spacing finds more in dense cities) and `--tiles` (how far out to go).
+
 ## Sweep a whole city
 
 Comma-separate areas (and/or types) and `lacspace-leads` runs each search in turn, then **merges and de-duplicates** into a single list — sorted and filtered across the whole set:
@@ -123,6 +151,15 @@ npx lacspace-leads "coffee shop" \
 
 # Two business types at once, capped at 100 unique leads total
 npx lacspace-leads --type "gym,fitness studio" --city Pokhara --total 100 -f csv
+
+# Several cities and several areas at once — 400 in total, one Excel file per city
+npx lacspace-leads restaurants \
+  --cities "Kathmandu, Lalitpur, Bhaktapur" \
+  --areas "Baneshwor, Thamel, Patan" \
+  --target 400 --split city -f xlsx
+
+# Everyone in town who has no website yet
+npx lacspace-leads "beauty salon" --city Pokhara --target 200 --no-website --has-phone -f csv
 ```
 
 ## Search by radius

package/dist/cli.js

@@ -2,7 +2,7 @@
 
 // src/cli.ts
 import { writeFileSync as writeFileSync2, existsSync as existsSync2, readFileSync as readFileSync2 } from "fs";
-import { resolve as resolve2 } from "path";
+import { resolve as resolve2, extname as extname2 } from "path";
 import { createInterface } from "readline/promises";
 import { stdin, stdout, stderr, argv, exit } from "process";
 
@@ -308,6 +308,7 @@ function filterLeads(leads, filters = {}) {
     if (filters.minReviews !== void 0 && !(typeof l.reviews === "number" && l.reviews >= filters.minReviews)) return false;
     if (filters.hasPhone && !l.phone) return false;
     if (filters.hasWebsite && !l.website) return false;
+    if (filters.noWebsite && l.website) return false;
     if (filters.hasEmail && !l.email) return false;
     if (filters.hasValidEmail && l.emailStatus !== "valid") return false;
     if (filters.hasContact && !(l.phone || l.email || l.website)) return false;
@@ -950,11 +951,20 @@ async function scrapeLeads(opts) {
     const page = await context.newPage();
     const urlOpts = { hl: locale };
     if (opts.region) urlOpts.gl = opts.region;
-    if (near) urlOpts.center = { lat: near.lat, lng: near.lng, zoom: zoomForRadius(opts.radiusM ?? 2e3, near.lat) };
+    if (near) urlOpts.center = { lat: near.lat, lng: near.lng, zoom: zoomForRadius(opts.radiusM ?? opts.zoomRadiusM ?? 2e3, near.lat) };
     await page.goto(mapsSearchUrl(query, urlOpts), { waitUntil: "domcontentloaded", timeout: 45e3 });
     await dismissConsent(page);
     await loadResults(page, limit, delayMs, onProgress, signal);
     if (signal?.aborted) throw new LeadsError("Search aborted.", "ABORTED");
+    if (opts.onCenter) {
+      const at = parseLatLng(page.url());
+      if (at.latitude !== void 0 && at.longitude !== void 0) {
+        try {
+          opts.onCenter({ lat: at.latitude, lng: at.longitude });
+        } catch {
+        }
+      }
+    }
     const cards = await page.locator("a.hfpxzc").evaluateAll(
       (els, lim) => els.slice(0, lim).map((a) => ({
         href: a.href,
@@ -966,6 +976,22 @@ async function scrapeLeads(opts) {
       onProgress?.("no listings found (Google may have shown a CAPTCHA or an empty result).");
       return [];
     }
+    if (opts.skipListing) {
+      const all = cards;
+      const before = all.length;
+      const fresh = all.filter((card) => {
+        try {
+          return !opts.skipListing({ name: card.name, href: card.href });
+        } catch {
+          return true;
+        }
+      });
+      if (fresh.length !== before) {
+        onProgress?.(`${before - fresh.length} of ${before} already collected \u2014 skipping them.`);
+        all.splice(0, before, ...fresh);
+      }
+      if (all.length === 0) return [];
+    }
     let leads = [];
     if (!wantDetails) {
       leads = cards.map((c2) => {
@@ -1437,6 +1463,173 @@ function leadsToEnrichInput(leads) {
   return out;
 }
 
+// src/sweep.ts
+var MAX_PER_SEARCH = 120;
+var MIN_PER_STEP = 40;
+var M_PER_DEG_LAT = 111320;
+function gridPoints(center, stepM, rings) {
+  const step = Math.max(100, stepM);
+  const ringCount = Math.max(0, Math.trunc(rings));
+  const dLat = step / M_PER_DEG_LAT;
+  const cosLat = Math.cos(center.lat * Math.PI / 180);
+  const dLng = step / (M_PER_DEG_LAT * (Math.abs(cosLat) < 0.01 ? 0.01 : cosLat));
+  const out = [];
+  for (let ring = 0; ring <= ringCount; ring++) {
+    for (let x = -ring; x <= ring; x++) {
+      for (let y = -ring; y <= ring; y++) {
+        if (ring > 0 && Math.max(Math.abs(x), Math.abs(y)) !== ring) continue;
+        out.push({
+          lat: +(center.lat + y * dLat).toFixed(6),
+          lng: +(center.lng + x * dLng).toFixed(6)
+        });
+      }
+    }
+  }
+  return out;
+}
+function tilesForRings(rings) {
+  const r = Math.max(0, Math.trunc(rings));
+  return (2 * r + 1) ** 2;
+}
+function ringsForTiles(tiles) {
+  const want = Math.max(1, Math.trunc(tiles));
+  let rings = 0;
+  while (tilesForRings(rings) < want) rings++;
+  return rings;
+}
+function planNamedSteps(base) {
+  return expandQueries(base).map((q) => ({ kind: "search", ...q }));
+}
+async function sweepLeads(opts) {
+  const {
+    target,
+    stepM = 2500,
+    maxTiles = 49,
+    perSearch,
+    onNotice,
+    near,
+    ...shared
+  } = opts;
+  const want = Math.max(1, Math.trunc(target));
+  const cap = Math.min(MAX_PER_SEARCH, Math.max(1, Math.trunc(perSearch ?? MAX_PER_SEARCH)));
+  const dedupeBy = shared.dedupe ?? "smart";
+  let collected = [];
+  const stats = { searches: 0, tiles: 0, unique: 0, saturated: false };
+  const seenNames = /* @__PURE__ */ new Set();
+  const seenUrls = /* @__PURE__ */ new Set();
+  const remember = (leads) => {
+    for (const l of leads) {
+      if (l.name) seenNames.add(l.name.trim().toLowerCase());
+      if (l.mapsUrl) seenUrls.add(l.mapsUrl);
+    }
+  };
+  const skipListing = (listing) => seenUrls.has(listing.href) || !!listing.name && seenNames.has(listing.name.trim().toLowerCase());
+  const merge = (found) => {
+    const before = collected.length;
+    collected = dedupeLeads([...collected, ...found], dedupeBy);
+    remember(found);
+    stats.unique = collected.length;
+    return collected.length - before;
+  };
+  const enough = () => collected.length >= want;
+  const stepLimit = () => Math.min(cap, Math.max(MIN_PER_STEP, (want - collected.length) * 2));
+  let center = near ? { lat: near.lat, lng: near.lng } : void 0;
+  const captureCenter = (c2) => {
+    center ??= c2;
+  };
+  for (const step of planNamedSteps(shared)) {
+    if (enough() || shared.signal?.aborted) break;
+    const searchOpts = {
+      ...shared,
+      ...step,
+      limit: stepLimit(),
+      onCenter: captureCenter
+    };
+    const found = await scrapeLeads(searchOpts);
+    stats.searches++;
+    const added = merge(found);
+    onNotice?.(
+      `search ${stats.searches}: ${[step.area, step.city].filter(Boolean).join(", ") || step.query || step.type} \u2192 +${added} new (${collected.length}/${want})`
+    );
+  }
+  if (!enough() && !shared.signal?.aborted) {
+    if (!center) {
+      onNotice?.("no map centre available, so tiling is not possible \u2014 pass --near lat,lng to sweep a radius");
+      stats.saturated = true;
+    } else {
+      const points = gridPoints(center, stepM, ringsForTiles(maxTiles)).slice(0, Math.max(1, maxTiles));
+      onNotice?.(
+        `named searches gave ${collected.length} of ${want} \u2014 tiling the map around ${center.lat.toFixed(4)},${center.lng.toFixed(4)} every ${Math.round(stepM)}m`
+      );
+      let dry = 0;
+      for (const point of points) {
+        if (enough() || shared.signal?.aborted) break;
+        const tileOpts = {
+          ...shared,
+          limit: stepLimit(),
+          near: point,
+          onCenter: captureCenter,
+          skipListing
+        };
+        delete tileOpts.area;
+        delete tileOpts.city;
+        if (tileOpts.radiusM === void 0) tileOpts.zoomRadiusM = stepM;
+        const found = await scrapeLeads(tileOpts);
+        stats.tiles++;
+        const added = merge(found);
+        onNotice?.(`tile ${stats.tiles}/${points.length} \u2192 +${added} new (${collected.length}/${want})`);
+        dry = added === 0 ? dry + 1 : 0;
+        if (dry >= 4) {
+          stats.saturated = true;
+          break;
+        }
+      }
+      if (!enough() && !stats.saturated) stats.saturated = stats.tiles >= points.length;
+    }
+  }
+  const sorted = shared.sort ? sortLeads(collected, shared.sort, shared.sortDir) : collected;
+  return { leads: sorted.slice(0, want), stats };
+}
+
+// src/split.ts
+var SPLIT_KEYS = ["city", "area", "type"];
+function addressMentions(address, name) {
+  const hay = (address ?? "").toLowerCase();
+  const needle = name.trim().toLowerCase();
+  if (!hay || !needle) return false;
+  const at = hay.indexOf(needle);
+  if (at === -1) return false;
+  const before = at === 0 ? "" : hay[at - 1];
+  const after = hay[at + needle.length] ?? "";
+  const boundary = (ch) => ch === "" || !/[a-z0-9]/.test(ch);
+  return boundary(before) && boundary(after);
+}
+function groupLeads(leads, key, requested = []) {
+  const places = requested.map((p) => p.trim()).filter(Boolean);
+  const groupOf = (lead) => {
+    if (key === "type") return (lead.category ?? "").trim() || "other";
+    const hit = places.find((p) => addressMentions(lead.address, p));
+    return hit ?? "other";
+  };
+  const groups = /* @__PURE__ */ new Map();
+  for (const lead of leads) {
+    const g = groupOf(lead);
+    const bucket = groups.get(g);
+    if (bucket) bucket.push(lead);
+    else groups.set(g, [lead]);
+  }
+  return new Map(
+    [...groups].sort((a, b) => {
+      if (a[0] === "other") return 1;
+      if (b[0] === "other") return -1;
+      return b[1].length - a[1].length;
+    })
+  );
+}
+function groupSlug(name) {
+  return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "") || "other";
+}
+
 // src/cli.ts
 var SOCIAL_FIELDS = ENRICHED_FIELDS.filter((f) => f !== "email");
 var OUTPUT_FORMATS = ["json", "ndjson", "csv", "xlsx"];
@@ -1465,6 +1658,7 @@ function parseArgs(list) {
     verifyEmails: false,
     hasPhone: false,
     hasWebsite: false,
+    noWebsite: false,
     hasEmail: false,
     hasValidEmail: false,
     hasContact: false,
@@ -1491,6 +1685,11 @@ function parseArgs(list) {
     else if (arg === "-o" || arg === "--out") a.out = next();
     else if (arg === "--append") a.append = true;
     else if (arg === "-n" || arg === "--limit") a.limit = parseInt(next(), 10) || a.limit;
+    else if (arg === "--target" || arg === "--want" || arg === "--goal") a.target = parseInt(next(), 10) || a.target;
+    else if (arg === "--step" || arg === "--tile-step") a.step = next();
+    else if (arg === "--tiles" || arg === "--max-tiles") a.tiles = parseInt(next(), 10) || a.tiles;
+    else if (arg === "--split" || arg === "--split-by") a.split = next();
+    else if (arg === "--no-website" || arg === "--without-website") a.noWebsite = true;
     else if (arg === "--total") a.total = parseInt(next(), 10) || a.total;
     else if (arg === "--headless") a.headless = true;
     else if (arg === "--no-details") a.details = false;
@@ -1564,7 +1763,14 @@ ${c("bold", "Search options")}
       --radius <dist>   Keep only leads within this of --near, e.g. 2km, 500m, 1mi
       --fields <list>   Columns: ${ALL_FIELDS.join(",")}
       --preset <name>   Field bundle: ${Object.keys(FIELD_PRESETS).join(" | ")}
-  -n, --limit <n>       Max listings per search   (default 60)
+  -n, --limit <n>       Max listings per SEARCH   (default 60, Google caps ~120)
+      --target <n>      How many leads you want IN TOTAL \u2014 keeps searching
+                        (every area you named, then tiles the map) until it has
+                        them. Use this one for 500+.
+      --step <dist>     Spacing between map tiles      (default 2.5km)
+      --tiles <n>       Most map tiles to try          (default 49)
+      --split <key>     Write one file per city | area | type
+      --no-website      Only businesses with NO website (the pitch list)
       --total <n>       Cap the merged result (batch searches)
       --no-details      Names + Maps URLs only (fast, no per-listing open)
 
@@ -1810,12 +2016,18 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
 `);
   if (!args.query && !args.type && !args.yes) {
     args.type = await prompt(`${c("green", "?")} Business type ${c("dim", "(e.g. restaurants)")}: `);
-    args.city = args.city ?? await prompt(`${c("green", "?")} City ${c("dim", "(optional)")}: `);
-    args.area = args.area ?? await prompt(`${c("green", "?")} Area ${c("dim", "(optional)")}: `);
+    args.city = args.city ?? await prompt(`${c("green", "?")} City ${c("dim", "(one, or several: Kathmandu, Pokhara)")}: `);
+    args.area = args.area ?? await prompt(`${c("green", "?")} Area ${c("dim", "(optional, several allowed: Baneshwor, Thamel)")}: `);
     const fmt = await prompt(`${c("green", "?")} Format ${c("dim", "(json/csv/xlsx)")} ${c("dim", "[json]")}: `, "json");
     if (fmt === "csv" || fmt === "xlsx" || fmt === "json") args.format = fmt;
-    const lim = await prompt(`${c("green", "?")} How many ${c("dim", "[60]")}: `, "60");
-    args.limit = parseInt(lim, 10) || args.limit;
+    const lim = await prompt(`${c("green", "?")} How many leads in total ${c("dim", "[60]")}: `, "60");
+    const wanted2 = parseInt(lim, 10) || args.limit;
+    if (wanted2 > MAX_PER_SEARCH) {
+      args.target = wanted2;
+      log(`  ${c("dim", `${wanted2} is more than one search returns, so I'll sweep every area, then tile the map.`)}`);
+    } else {
+      args.limit = wanted2;
+    }
     const em = await prompt(`${c("green", "?")} Also find emails from websites? ${c("dim", "(slower) [y/N]")} `);
     if (/^y/i.test(em)) args.emails = true;
   }
@@ -1841,6 +2053,12 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
     return;
   }
   const nearPoint = args.near ? parseLatLngPair(args.near) : void 0;
+  const tileStepM = args.step ? parseDistance(args.step) : void 0;
+  if (!args.details && (args.split || args.noWebsite)) {
+    const needs = [args.split ? `--split ${args.split}` : "", args.noWebsite ? "--no-website" : ""].filter(Boolean).join(" and ");
+    log(c("yellow", `  ! ${needs} needs each listing's details, but --no-details is on \u2014 turning details back on.`));
+    args.details = true;
+  }
   if (args.near && !nearPoint) {
     log(c("red", `
 \u2717 --near must be "lat,lng", e.g. --near "27.7172,85.3240".`));
@@ -1884,6 +2102,7 @@ ${c("bold", c("magenta", "\u25C6 lacspace-leads"))} ${c("dim", "\u2014 Google Ma
   if (args.minReviews !== void 0) filters.minReviews = args.minReviews;
   if (args.hasPhone) filters.hasPhone = true;
   if (args.hasWebsite) filters.hasWebsite = true;
+  if (args.noWebsite) filters.noWebsite = true;
   if (args.hasEmail) filters.hasEmail = true;
   if (args.hasValidEmail) filters.hasValidEmail = true;
   if (args.hasContact) filters.hasContact = true;
@@ -1987,8 +2206,22 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
     }
   }
   let leads;
+  let sweptShort = false;
   try {
-    if (isBatch) {
+    if (args.target !== void 0) {
+      const sweepOpts = {
+        ...opts,
+        target: args.target,
+        onNotice: (m) => log(`  ${c("cyan", "\u25F7")} ${c("dim", m)}`)
+      };
+      if (tileStepM !== void 0) sweepOpts.stepM = tileStepM;
+      if (args.tiles !== void 0) sweepOpts.maxTiles = args.tiles;
+      if (args.limit !== 60) sweepOpts.perSearch = args.limit;
+      const swept = await sweepLeads(sweepOpts);
+      leads = swept.leads;
+      sweptShort = swept.stats.saturated && leads.length < args.target;
+      log(`  ${c("cyan", "\u25F7")} ${c("dim", `${swept.stats.searches} search${swept.stats.searches === 1 ? "" : "es"} + ${swept.stats.tiles} map tile${swept.stats.tiles === 1 ? "" : "s"} \u2192 ${swept.stats.unique} unique`)}`);
+    } else if (isBatch) {
       const batchOpts = { ...opts };
       if (args.total !== void 0) batchOpts.total = args.total;
       if (checkpoint && cpFile) {
@@ -2013,6 +2246,16 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
   } finally {
     process.removeListener("SIGINT", onSig);
   }
+  if (sweptShort) {
+    log(c("yellow", `
+  ! The map ran out of new results at ${leads.length} \u2014 that is everything Google lists here.`));
+    log(c("dim", "    Widen it: more areas (--areas a,b,c), more cities (--cities x,y), a bigger --step, or related --types.\n"));
+  } else if (args.target === void 0 && args.limit > leads.length && leads.length >= MAX_PER_SEARCH - 20) {
+    log(c("yellow", `
+  ! Google stops a single search at about ${MAX_PER_SEARCH} results, so --limit ${args.limit} could only return ${leads.length}.`));
+    log(c("dim", `    To really get ${args.limit}, ask for a total: --target ${args.limit} \u2014 it sweeps every area you name, then tiles the map.
+`));
+  }
   if (leads.length === 0) {
     log(c("yellow", "\n  No leads collected. Try a broader area, looser filters, or a smaller --limit.\n"));
     if (cpFile) clearCheckpoint(cpFile);
@@ -2072,6 +2315,24 @@ ${c("yellow", "!")} This opens a browser and searches Google Maps. Continue? ${c
     printSummary();
     return;
   }
+  if (args.split) {
+    const key = args.split.trim().toLowerCase();
+    if (!SPLIT_KEYS.includes(key)) {
+      log(c("yellow", `  ! --split "${args.split}" is not one of ${SPLIT_KEYS.join(", ")} \u2014 writing a single file instead.`));
+    } else {
+      const requested = (key === "area" ? args.area : key === "city" ? args.city : "")?.split(",").map((x) => x.trim()).filter(Boolean) ?? [];
+      const groups = groupLeads(leads, key, requested);
+      const ext = extname2(out) || `.${args.format}`;
+      const stem = out.slice(0, out.length - ext.length);
+      log("");
+      for (const [group, rows] of groups) {
+        const file = `${stem}-${groupSlug(group)}${ext}`;
+        const part = serialize(rows, args.format, fields, serOpts);
+        writeFileSync2(file, part.binary ? Buffer.from(part.data) : part.data);
+        log(`  ${c("green", "\u2714")} ${c("bold", String(rows.length))} leads \u2192 ${c("cyan", file)}`);
+      }
+    }
+  }
   writeFileSync2(out, binary ? Buffer.from(data) : data);
   if (cpFile) clearCheckpoint(cpFile);
   const s = computeStats(leads);