@galihvsx/gmr-scraper 1.0.0

package/dist/cli.js

@@ -0,0 +1,681 @@
+#!/usr/bin/env node
+import chalk from 'chalk';
+import { Command } from 'commander';
+import ora from 'ora';
+
+// src/cache.ts
+var Cache = class {
+  cache = /* @__PURE__ */ new Map();
+  ttl;
+  maxSize;
+  constructor(options = {}) {
+    this.ttl = options.ttl ?? 3e5;
+    this.maxSize = options.maxSize ?? 100;
+  }
+  set(key, value) {
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey) {
+        this.cache.delete(firstKey);
+      }
+    }
+    this.cache.set(key, {
+      value,
+      expiresAt: Date.now() + this.ttl
+    });
+  }
+  get(key) {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return void 0;
+    }
+    if (Date.now() > entry.expiresAt) {
+      this.cache.delete(key);
+      return void 0;
+    }
+    return entry.value;
+  }
+  has(key) {
+    return this.get(key) !== void 0;
+  }
+  delete(key) {
+    this.cache.delete(key);
+  }
+  clear() {
+    this.cache.clear();
+  }
+  size() {
+    this.cleanup();
+    return this.cache.size;
+  }
+  cleanup() {
+    const now = Date.now();
+    for (const [key, entry] of this.cache.entries()) {
+      if (now > entry.expiresAt) {
+        this.cache.delete(key);
+      }
+    }
+  }
+};
+function createCacheKey(placeId, sort, page, query, lang) {
+  return `${placeId}:${sort}:${page}:${query}:${lang}`;
+}
+
+// src/rate-limiter.ts
+var RateLimiter = class {
+  tokens;
+  lastRefill;
+  requestsPerSecond;
+  burstSize;
+  refillRate;
+  constructor(options = {}) {
+    this.requestsPerSecond = options.requestsPerSecond ?? 2;
+    this.burstSize = options.burstSize ?? 5;
+    this.tokens = this.burstSize;
+    this.lastRefill = Date.now();
+    this.refillRate = 1e3 / this.requestsPerSecond;
+  }
+  refill() {
+    const now = Date.now();
+    const timePassed = now - this.lastRefill;
+    const tokensToAdd = Math.floor(timePassed / this.refillRate);
+    if (tokensToAdd > 0) {
+      this.tokens = Math.min(this.burstSize, this.tokens + tokensToAdd);
+      this.lastRefill = now;
+    }
+  }
+  async acquire() {
+    this.refill();
+    if (this.tokens > 0) {
+      this.tokens--;
+      return;
+    }
+    const waitTime = this.refillRate - (Date.now() - this.lastRefill);
+    if (waitTime > 0) {
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+      return this.acquire();
+    }
+  }
+  async execute(fn) {
+    await this.acquire();
+    return fn();
+  }
+};
+
+// src/types.ts
+var SortEnum = /* @__PURE__ */ ((SortEnum3) => {
+  SortEnum3[SortEnum3["relevant"] = 1] = "relevant";
+  SortEnum3[SortEnum3["newest"] = 2] = "newest";
+  SortEnum3[SortEnum3["highest_rating"] = 3] = "highest_rating";
+  SortEnum3[SortEnum3["lowest_rating"] = 4] = "lowest_rating";
+  return SortEnum3;
+})(SortEnum || {});
+
+// src/errors.ts
+var ScraperError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "ScraperError";
+    Error.captureStackTrace(this, this.constructor);
+  }
+};
+var InvalidUrlError = class extends ScraperError {
+  constructor(message) {
+    super(message, "INVALID_URL");
+    this.name = "InvalidUrlError";
+  }
+};
+var FetchError = class extends ScraperError {
+  constructor(message, statusCode, response) {
+    super(message, "FETCH_ERROR");
+    this.statusCode = statusCode;
+    this.response = response;
+    this.name = "FetchError";
+  }
+};
+var TimeoutError = class extends ScraperError {
+  constructor(message = "Request timeout") {
+    super(message, "TIMEOUT");
+    this.name = "TimeoutError";
+  }
+};
+
+// src/extractors.ts
+async function resolveUrl(url) {
+  if (url.includes("goo.gl") || url.includes("maps.app.goo.gl")) {
+    const response = await fetch(url, { redirect: "follow", method: "HEAD" });
+    return response.url;
+  }
+  return url;
+}
+function extractPlaceId(url) {
+  const match1 = url.match(/!1s(0x[0-9a-fA-F]+:0x[0-9a-fA-F]+)/);
+  if (match1 && match1[1]) {
+    return match1[1];
+  }
+  const match2 = url.match(/!1s([a-zA-Z0-9_:]+)!/);
+  if (match2 && match2[1]) {
+    return match2[1];
+  }
+  throw new Error(
+    "Could not extract Place ID from URL. Please ensure it is a valid Google Maps Place URL."
+  );
+}
+function hexToDec(hex) {
+  return BigInt(hex).toString();
+}
+function buildRpcUrl(placeId, sort, nextPageToken = "", searchQuery = "", lang = "en") {
+  const parts = placeId.split(":");
+  if (parts.length !== 2) {
+    throw new Error(
+      `Invalid Place ID format for listentitiesreviews: ${placeId}`
+    );
+  }
+  const h1 = hexToDec(parts[0]);
+  const h2 = hexToDec(parts[1]);
+  const constantToken = "dzvaXrvAMImImAXHsLPICA";
+  const paginationBlock = nextPageToken ? `!2m2!2i10!3s${nextPageToken}` : `!2m1!2i10`;
+  return `https://www.google.com/maps/preview/review/listentitiesreviews?authuser=0&hl=${lang}&gl=in&pb=!1m2!1y${h1}!2y${h2}${paginationBlock}!3e${sort}!4m5!3b1!4b1!5b1!6b1!7b1!5m2!1s${constantToken}!7e81`;
+}
+
+// src/parser.ts
+function parseReviews(reviews) {
+  return reviews.map((review) => {
+    const authorInfo = review[0] || [];
+    const images = null;
+    const response = null;
+    return {
+      review_id: review[10],
+      time: {
+        published: review[1],
+        last_edited: null
+      },
+      author: {
+        name: authorInfo[1],
+        profile_url: authorInfo[0],
+        url: authorInfo[0],
+        id: review[6]
+      },
+      review: {
+        rating: review[4],
+        text: parseText(review[3]).text,
+        original_text: parseText(review[3]).original_text,
+        language: review[32]
+      },
+      images,
+      source: review[10],
+      response
+    };
+  });
+}
+function parseText(text) {
+  if (!text) return { text: null, original_text: null };
+  const splitPattern = /\n\n\((.+?)\)\n/g;
+  const match = splitPattern.exec(text);
+  if (match) {
+    const splitIndex = match.index;
+    const originalHeaderLength = match[0].length;
+    const firstPart = text.substring(0, splitIndex);
+    const secondPart = text.substring(splitIndex + originalHeaderLength);
+    const translationParamMatch = firstPart.match(/^\((.+?)\) /);
+    let translatedText = firstPart;
+    if (translationParamMatch) {
+      translatedText = firstPart.substring(translationParamMatch[0].length);
+    }
+    return {
+      text: translatedText,
+      original_text: secondPart
+    };
+  }
+  return { text, original_text: null };
+}
+
+// src/retry.ts
+var DEFAULT_RETRY_OPTIONS = {
+  maxAttempts: 3,
+  initialDelay: 1e3,
+  maxDelay: 1e4,
+  backoffMultiplier: 2,
+  timeout: 3e4,
+  retryCondition: (error) => {
+    if (error.name === "InvalidUrlError") return false;
+    if (error.name === "ParseError") return false;
+    return true;
+  }
+};
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function calculateDelay(attempt, initialDelay, maxDelay, backoffMultiplier) {
+  const delay = initialDelay * Math.pow(backoffMultiplier, attempt - 1);
+  return Math.min(delay, maxDelay);
+}
+async function withRetry(fn, options = {}) {
+  const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
+  let lastError;
+  for (let attempt = 1; attempt <= opts.maxAttempts; attempt++) {
+    try {
+      if (opts.timeout) {
+        return await withTimeout(fn(), opts.timeout);
+      }
+      return await fn();
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt === opts.maxAttempts || !opts.retryCondition(lastError)) {
+        throw lastError;
+      }
+      const delay = calculateDelay(
+        attempt,
+        opts.initialDelay,
+        opts.maxDelay,
+        opts.backoffMultiplier
+      );
+      console.warn(
+        `Attempt ${attempt}/${opts.maxAttempts} failed: ${lastError.message}. Retrying in ${delay}ms...`
+      );
+      await sleep(delay);
+    }
+  }
+  throw lastError;
+}
+async function withTimeout(promise, timeoutMs) {
+  const timeoutPromise = new Promise((_, reject) => {
+    setTimeout(() => {
+      reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+  });
+  return Promise.race([promise, timeoutPromise]);
+}
+
+// src/utils.ts
+function validateParams(url, sortType, pages, clean) {
+  if (!url) {
+    throw new InvalidUrlError("URL is required");
+  }
+  if (sortType && !SortEnum[sortType]) {
+    throw new InvalidUrlError(
+      `Invalid sort type: ${sortType}. Valid options: ${Object.keys(SortEnum).filter((k) => isNaN(Number(k))).join(", ")}`
+    );
+  }
+  if (pages !== void 0 && pages !== "max" && (typeof pages !== "number" || pages < 1)) {
+    throw new InvalidUrlError("Pages must be 'max' or a positive number");
+  }
+}
+async function fetchReviews(placeId, sort, nextPageToken = "", searchQuery = "", lang = "en", cache, rateLimiter, retryOptions) {
+  const cacheKey = cache ? createCacheKey(placeId, sort, nextPageToken, searchQuery, lang) : "";
+  if (cache && cacheKey) {
+    const cached = cache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+  }
+  const fetchFn = async () => {
+    const apiUrl = buildRpcUrl(placeId, sort, nextPageToken, searchQuery, lang);
+    const actualFetch = async () => {
+      const response = await fetch(apiUrl, {
+        headers: {
+          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        }
+      });
+      if (!response.ok) {
+        throw new FetchError(
+          `Failed to fetch reviews: ${response.statusText}`,
+          response.status
+        );
+      }
+      const textData = await response.text();
+      const cleanJson = textData.replace(/^\)\]\}'/, "");
+      let parsedData;
+      try {
+        parsedData = JSON.parse(cleanJson);
+      } catch (e) {
+        console.error("Failed to parse JSON:", cleanJson.substring(0, 100));
+        throw new FetchError("Failed to parse API response");
+      }
+      if (!parsedData || !Array.isArray(parsedData)) {
+        console.warn("Unexpected response structure:", parsedData);
+        return { data: [], nextPageToken: null };
+      }
+      const reviews = parsedData[2] || [];
+      const nextToken = parsedData[1] || null;
+      return { data: reviews, nextPageToken: nextToken };
+    };
+    if (rateLimiter) {
+      return rateLimiter.execute(actualFetch);
+    }
+    return actualFetch();
+  };
+  const result = retryOptions ? await withRetry(fetchFn, retryOptions) : await fetchFn();
+  if (cache && cacheKey) {
+    cache.set(cacheKey, result);
+  }
+  return result;
+}
+async function paginateReviews(url, sort, pages, searchQuery, clean, lang = "en", cache, rateLimiter, retryOptions, onProgress) {
+  const resolvedUrl = await resolveUrl(url);
+  const placeId = extractPlaceId(resolvedUrl);
+  const initial = await fetchReviews(
+    placeId,
+    sort,
+    "",
+    searchQuery,
+    lang,
+    cache,
+    rateLimiter,
+    retryOptions
+  );
+  let allReviews = [...initial.data];
+  let nextPageToken = initial.nextPageToken;
+  if (onProgress) {
+    onProgress(1, pages);
+  }
+  if (pages === 1 || !nextPageToken) {
+    return clean ? parseReviews(allReviews) : allReviews;
+  }
+  let currentPage = 2;
+  const maxPages = pages === "max" ? Infinity : pages;
+  while (nextPageToken && currentPage <= maxPages) {
+    if (!rateLimiter) {
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+    }
+    try {
+      const nextBatch = await fetchReviews(
+        placeId,
+        sort,
+        nextPageToken,
+        searchQuery,
+        lang,
+        cache,
+        rateLimiter,
+        retryOptions
+      );
+      allReviews = [...allReviews, ...nextBatch.data];
+      nextPageToken = nextBatch.nextPageToken;
+      if (onProgress) {
+        onProgress(currentPage, pages);
+      }
+      if (!nextBatch.data.length && !nextPageToken) {
+        break;
+      }
+    } catch (e) {
+      console.error(`Error scraping page ${currentPage}:`, e);
+      break;
+    }
+    currentPage++;
+  }
+  return clean ? parseReviews(allReviews) : allReviews;
+}
+
+// src/batch.ts
+async function batchScraper(urls, options = {}) {
+  const {
+    concurrency = 3,
+    onProgress,
+    includeAnalytics = false,
+    ...scraperOptions
+  } = options;
+  const results = [];
+  const queue = [...urls];
+  let completed = 0;
+  const processBatch = async (url) => {
+    try {
+      const resolvedUrl = await resolveUrl(url);
+      const placeId = extractPlaceId(resolvedUrl);
+      const reviews = await scraper(url, scraperOptions);
+      const result = {
+        url,
+        placeId,
+        reviews
+      };
+      if (includeAnalytics && scraperOptions.clean) {
+        result.analytics = calculateAnalytics(reviews);
+      }
+      completed++;
+      if (onProgress) {
+        onProgress(completed, urls.length, url);
+      }
+      return result;
+    } catch (error) {
+      completed++;
+      if (onProgress) {
+        onProgress(completed, urls.length, url);
+      }
+      return {
+        url,
+        reviews: [],
+        error: error instanceof Error ? error : new Error(String(error))
+      };
+    }
+  };
+  const workers = [];
+  for (let i = 0; i < concurrency; i++) {
+    workers.push(
+      (async () => {
+        while (queue.length > 0) {
+          const url = queue.shift();
+          if (url) {
+            const result = await processBatch(url);
+            results.push(result);
+          }
+        }
+      })()
+    );
+  }
+  await Promise.all(workers);
+  return results;
+}
+function calculateAnalytics(reviews) {
+  const totalReviews = reviews.length;
+  const ratingDistribution = {
+    1: 0,
+    2: 0,
+    3: 0,
+    4: 0,
+    5: 0
+  };
+  let totalRating = 0;
+  let reviewsWithText = 0;
+  let reviewsWithImages = 0;
+  let reviewsWithResponse = 0;
+  for (const review of reviews) {
+    const rating = review.review.rating;
+    totalRating += rating;
+    ratingDistribution[rating] = (ratingDistribution[rating] || 0) + 1;
+    if (review.review.text) reviewsWithText++;
+    if (review.images && review.images.length > 0) reviewsWithImages++;
+    if (review.response) reviewsWithResponse++;
+  }
+  return {
+    totalReviews,
+    averageRating: totalReviews > 0 ? totalRating / totalReviews : 0,
+    ratingDistribution,
+    reviewsWithText,
+    reviewsWithImages,
+    reviewsWithResponse
+  };
+}
+
+// src/index.ts
+async function scraper(url, options = {}) {
+  const {
+    sort_type = "relevant",
+    search_query = "",
+    pages = "max",
+    clean = false,
+    lang = "en",
+    retry,
+    cache: cacheOptions,
+    rateLimit,
+    timeout,
+    onProgress
+  } = options;
+  validateParams(url, sort_type, pages);
+  const sort = SortEnum[sort_type];
+  const cache = cacheOptions?.enabled ? new Cache(cacheOptions) : void 0;
+  const rateLimiter = rateLimit ? new RateLimiter(rateLimit) : void 0;
+  const retryOptions = {
+    timeout,
+    ...retry
+  };
+  try {
+    return await paginateReviews(
+      url,
+      sort,
+      pages,
+      search_query,
+      clean,
+      lang,
+      cache,
+      rateLimiter,
+      retryOptions,
+      onProgress
+    );
+  } catch (e) {
+    console.error("Scraper Error:", e);
+    throw e;
+  }
+}
+
+// src/cli/formatters.ts
+function formatJSON(data) {
+  return JSON.stringify(data, null, 2);
+}
+function formatTable(reviews) {
+  if (!reviews || reviews.length === 0) {
+    return "No reviews found.";
+  }
+  const isClean = "review_id" in reviews[0];
+  if (!isClean) {
+    return "Table format only available with --clean flag";
+  }
+  const cleanReviews = reviews;
+  const header = [
+    "Author".padEnd(20),
+    "Rating".padEnd(8),
+    "Date".padEnd(12),
+    "Text".padEnd(50)
+  ].join(" | ");
+  const separator = "-".repeat(header.length);
+  const rows = cleanReviews.slice(0, 20).map((review) => {
+    const author = review.author.name.substring(0, 18).padEnd(20);
+    const rating = "\u2B50".repeat(review.review.rating).padEnd(8);
+    const date = review.time.published.substring(0, 10).padEnd(12);
+    const text = (review.review.text || "No text").substring(0, 48).replace(/\n/g, " ").padEnd(50);
+    return [author, rating, date, text].join(" | ");
+  });
+  return [header, separator, ...rows].join("\n");
+}
+function formatCSV(reviews) {
+  if (!reviews || reviews.length === 0) {
+    return "No reviews found.";
+  }
+  const isClean = "review_id" in reviews[0];
+  if (!isClean) {
+    return "CSV format only available with --clean flag";
+  }
+  const cleanReviews = reviews;
+  const header = [
+    "review_id",
+    "author_name",
+    "rating",
+    "published",
+    "text",
+    "has_images",
+    "has_response"
+  ].join(",");
+  const rows = cleanReviews.map((review) => {
+    const text = (review.review.text || "").replace(/"/g, '""').replace(/\n/g, " ");
+    return [
+      review.review_id,
+      `"${review.author.name.replace(/"/g, '""')}"`,
+      review.review.rating,
+      review.time.published,
+      `"${text}"`,
+      review.images && review.images.length > 0 ? "yes" : "no",
+      review.response ? "yes" : "no"
+    ].join(",");
+  });
+  return [header, ...rows].join("\n");
+}
+
+// src/cli/index.ts
+var program = new Command();
+program.name("gmr-scraper").description("Modern Google Maps review scraper with TypeScript").version("1.0.0");
+program.command("scrape").description("Scrape reviews from a Google Maps URL").argument("<url>", "Google Maps place URL").option(
+  "-s, --sort <type>",
+  "Sort type (relevant, newest, highest_rating, lowest_rating)",
+  "relevant"
+).option("-p, --pages <number>", "Number of pages to scrape (or 'max')", "max").option("-q, --query <text>", "Search query to filter reviews", "").option("-l, --lang <code>", "Language code (e.g., en, id, es)", "en").option("-c, --clean", "Return parsed objects instead of raw data", false).option("-o, --output <format>", "Output format (json, table, csv)", "json").option("--cache", "Enable caching", false).option("--retry <attempts>", "Max retry attempts", "3").option("--timeout <ms>", "Request timeout in milliseconds", "30000").option("--rate-limit <rps>", "Requests per second", "2").action(async (url, options) => {
+  const spinner = ora("Scraping reviews...").start();
+  try {
+    const pages = options.pages === "max" ? "max" : parseInt(options.pages);
+    const reviews = await scraper(url, {
+      sort_type: options.sort,
+      pages,
+      search_query: options.query,
+      lang: options.lang,
+      clean: options.clean,
+      cache: {
+        enabled: options.cache
+      },
+      retry: {
+        maxAttempts: parseInt(options.retry)
+      },
+      timeout: parseInt(options.timeout),
+      rateLimit: {
+        requestsPerSecond: parseInt(options.rateLimit)
+      },
+      onProgress: (current, total) => {
+        spinner.text = `Scraping page ${current}${total !== "max" ? `/${total}` : ""}...`;
+      }
+    });
+    spinner.succeed(chalk.green(`Scraped ${reviews.length} reviews`));
+    switch (options.output) {
+      case "table":
+        console.log(formatTable(reviews));
+        break;
+      case "csv":
+        console.log(formatCSV(reviews));
+        break;
+      case "json":
+      default:
+        console.log(formatJSON(reviews));
+    }
+  } catch (error) {
+    spinner.fail(chalk.red("Failed to scrape reviews"));
+    console.error(
+      chalk.red(error instanceof Error ? error.message : String(error))
+    );
+    process.exit(1);
+  }
+});
+program.command("batch").description("Scrape reviews from multiple URLs").argument("<urls...>", "Google Maps place URLs").option("-s, --sort <type>", "Sort type", "relevant").option("-p, --pages <number>", "Number of pages to scrape", "max").option("-c, --clean", "Return parsed objects", false).option("--concurrency <number>", "Number of concurrent requests", "3").option("--analytics", "Include analytics", false).action(async (urls, options) => {
+  const spinner = ora(`Scraping ${urls.length} locations...`).start();
+  try {
+    const pages = options.pages === "max" ? "max" : parseInt(options.pages);
+    const results = await batchScraper(urls, {
+      sort_type: options.sort,
+      pages,
+      clean: options.clean,
+      concurrency: parseInt(options.concurrency),
+      includeAnalytics: options.analytics,
+      onProgress: (completed, total, url) => {
+        spinner.text = `Progress: ${completed}/${total} (${url})`;
+      }
+    });
+    spinner.succeed(
+      chalk.green(`Completed scraping ${urls.length} locations`)
+    );
+    console.log(formatJSON(results));
+  } catch (error) {
+    spinner.fail(chalk.red("Batch scraping failed"));
+    console.error(
+      chalk.red(error instanceof Error ? error.message : String(error))
+    );
+    process.exit(1);
+  }
+});
+program.parse();
+//# sourceMappingURL=cli.js.map
+//# sourceMappingURL=cli.js.map