@galihvsx/gmr-scraper 1.0.0

package/dist/index.cjs

@@ -0,0 +1,724 @@
+'use strict';
+
+// src/cache.ts
+var Cache = class {
+  cache = /* @__PURE__ */ new Map();
+  ttl;
+  maxSize;
+  constructor(options = {}) {
+    this.ttl = options.ttl ?? 3e5;
+    this.maxSize = options.maxSize ?? 100;
+  }
+  set(key, value) {
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      if (firstKey) {
+        this.cache.delete(firstKey);
+      }
+    }
+    this.cache.set(key, {
+      value,
+      expiresAt: Date.now() + this.ttl
+    });
+  }
+  get(key) {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return void 0;
+    }
+    if (Date.now() > entry.expiresAt) {
+      this.cache.delete(key);
+      return void 0;
+    }
+    return entry.value;
+  }
+  has(key) {
+    return this.get(key) !== void 0;
+  }
+  delete(key) {
+    this.cache.delete(key);
+  }
+  clear() {
+    this.cache.clear();
+  }
+  size() {
+    this.cleanup();
+    return this.cache.size;
+  }
+  cleanup() {
+    const now = Date.now();
+    for (const [key, entry] of this.cache.entries()) {
+      if (now > entry.expiresAt) {
+        this.cache.delete(key);
+      }
+    }
+  }
+};
+function createCacheKey(placeId, sort, page, query, lang) {
+  return `${placeId}:${sort}:${page}:${query}:${lang}`;
+}
+
+// src/rate-limiter.ts
+var RateLimiter = class {
+  tokens;
+  lastRefill;
+  requestsPerSecond;
+  burstSize;
+  refillRate;
+  constructor(options = {}) {
+    this.requestsPerSecond = options.requestsPerSecond ?? 2;
+    this.burstSize = options.burstSize ?? 5;
+    this.tokens = this.burstSize;
+    this.lastRefill = Date.now();
+    this.refillRate = 1e3 / this.requestsPerSecond;
+  }
+  refill() {
+    const now = Date.now();
+    const timePassed = now - this.lastRefill;
+    const tokensToAdd = Math.floor(timePassed / this.refillRate);
+    if (tokensToAdd > 0) {
+      this.tokens = Math.min(this.burstSize, this.tokens + tokensToAdd);
+      this.lastRefill = now;
+    }
+  }
+  async acquire() {
+    this.refill();
+    if (this.tokens > 0) {
+      this.tokens--;
+      return;
+    }
+    const waitTime = this.refillRate - (Date.now() - this.lastRefill);
+    if (waitTime > 0) {
+      await new Promise((resolve) => setTimeout(resolve, waitTime));
+      return this.acquire();
+    }
+  }
+  async execute(fn) {
+    await this.acquire();
+    return fn();
+  }
+};
+
+// src/types.ts
+var SortEnum = /* @__PURE__ */ ((SortEnum3) => {
+  SortEnum3[SortEnum3["relevant"] = 1] = "relevant";
+  SortEnum3[SortEnum3["newest"] = 2] = "newest";
+  SortEnum3[SortEnum3["highest_rating"] = 3] = "highest_rating";
+  SortEnum3[SortEnum3["lowest_rating"] = 4] = "lowest_rating";
+  return SortEnum3;
+})(SortEnum || {});
+
+// src/errors.ts
+var ScraperError = class extends Error {
+  constructor(message, code) {
+    super(message);
+    this.code = code;
+    this.name = "ScraperError";
+    Error.captureStackTrace(this, this.constructor);
+  }
+};
+var InvalidUrlError = class extends ScraperError {
+  constructor(message) {
+    super(message, "INVALID_URL");
+    this.name = "InvalidUrlError";
+  }
+};
+var FetchError = class extends ScraperError {
+  constructor(message, statusCode, response) {
+    super(message, "FETCH_ERROR");
+    this.statusCode = statusCode;
+    this.response = response;
+    this.name = "FetchError";
+  }
+};
+var ParseError = class extends ScraperError {
+  constructor(message, data) {
+    super(message, "PARSE_ERROR");
+    this.data = data;
+    this.name = "ParseError";
+  }
+};
+var RateLimitError = class extends ScraperError {
+  constructor(message = "Rate limit exceeded", retryAfter) {
+    super(message, "RATE_LIMIT");
+    this.retryAfter = retryAfter;
+    this.name = "RateLimitError";
+  }
+};
+var TimeoutError = class extends ScraperError {
+  constructor(message = "Request timeout") {
+    super(message, "TIMEOUT");
+    this.name = "TimeoutError";
+  }
+};
+
+// src/extractors.ts
+async function resolveUrl(url) {
+  if (url.includes("goo.gl") || url.includes("maps.app.goo.gl")) {
+    const response = await fetch(url, { redirect: "follow", method: "HEAD" });
+    return response.url;
+  }
+  return url;
+}
+function extractPlaceId(url) {
+  const match1 = url.match(/!1s(0x[0-9a-fA-F]+:0x[0-9a-fA-F]+)/);
+  if (match1 && match1[1]) {
+    return match1[1];
+  }
+  const match2 = url.match(/!1s([a-zA-Z0-9_:]+)!/);
+  if (match2 && match2[1]) {
+    return match2[1];
+  }
+  throw new Error(
+    "Could not extract Place ID from URL. Please ensure it is a valid Google Maps Place URL."
+  );
+}
+function hexToDec(hex) {
+  return BigInt(hex).toString();
+}
+function buildRpcUrl(placeId, sort, nextPageToken = "", searchQuery = "", lang = "en") {
+  const parts = placeId.split(":");
+  if (parts.length !== 2) {
+    throw new Error(
+      `Invalid Place ID format for listentitiesreviews: ${placeId}`
+    );
+  }
+  const h1 = hexToDec(parts[0]);
+  const h2 = hexToDec(parts[1]);
+  const constantToken = "dzvaXrvAMImImAXHsLPICA";
+  const paginationBlock = nextPageToken ? `!2m2!2i10!3s${nextPageToken}` : `!2m1!2i10`;
+  return `https://www.google.com/maps/preview/review/listentitiesreviews?authuser=0&hl=${lang}&gl=in&pb=!1m2!1y${h1}!2y${h2}${paginationBlock}!3e${sort}!4m5!3b1!4b1!5b1!6b1!7b1!5m2!1s${constantToken}!7e81`;
+}
+
+// src/parser.ts
+function parseReviews(reviews) {
+  return reviews.map((review) => {
+    const authorInfo = review[0] || [];
+    const images = null;
+    const response = null;
+    return {
+      review_id: review[10],
+      time: {
+        published: review[1],
+        last_edited: null
+      },
+      author: {
+        name: authorInfo[1],
+        profile_url: authorInfo[0],
+        url: authorInfo[0],
+        id: review[6]
+      },
+      review: {
+        rating: review[4],
+        text: parseText(review[3]).text,
+        original_text: parseText(review[3]).original_text,
+        language: review[32]
+      },
+      images,
+      source: review[10],
+      response
+    };
+  });
+}
+function parseText(text) {
+  if (!text) return { text: null, original_text: null };
+  const splitPattern = /\n\n\((.+?)\)\n/g;
+  const match = splitPattern.exec(text);
+  if (match) {
+    const splitIndex = match.index;
+    const originalHeaderLength = match[0].length;
+    const firstPart = text.substring(0, splitIndex);
+    const secondPart = text.substring(splitIndex + originalHeaderLength);
+    const translationParamMatch = firstPart.match(/^\((.+?)\) /);
+    let translatedText = firstPart;
+    if (translationParamMatch) {
+      translatedText = firstPart.substring(translationParamMatch[0].length);
+    }
+    return {
+      text: translatedText,
+      original_text: secondPart
+    };
+  }
+  return { text, original_text: null };
+}
+
+// src/retry.ts
+var DEFAULT_RETRY_OPTIONS = {
+  maxAttempts: 3,
+  initialDelay: 1e3,
+  maxDelay: 1e4,
+  backoffMultiplier: 2,
+  timeout: 3e4,
+  retryCondition: (error) => {
+    if (error.name === "InvalidUrlError") return false;
+    if (error.name === "ParseError") return false;
+    return true;
+  }
+};
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function calculateDelay(attempt, initialDelay, maxDelay, backoffMultiplier) {
+  const delay = initialDelay * Math.pow(backoffMultiplier, attempt - 1);
+  return Math.min(delay, maxDelay);
+}
+async function withRetry(fn, options = {}) {
+  const opts = { ...DEFAULT_RETRY_OPTIONS, ...options };
+  let lastError;
+  for (let attempt = 1; attempt <= opts.maxAttempts; attempt++) {
+    try {
+      if (opts.timeout) {
+        return await withTimeout(fn(), opts.timeout);
+      }
+      return await fn();
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+      if (attempt === opts.maxAttempts || !opts.retryCondition(lastError)) {
+        throw lastError;
+      }
+      const delay = calculateDelay(
+        attempt,
+        opts.initialDelay,
+        opts.maxDelay,
+        opts.backoffMultiplier
+      );
+      console.warn(
+        `Attempt ${attempt}/${opts.maxAttempts} failed: ${lastError.message}. Retrying in ${delay}ms...`
+      );
+      await sleep(delay);
+    }
+  }
+  throw lastError;
+}
+async function withTimeout(promise, timeoutMs) {
+  const timeoutPromise = new Promise((_, reject) => {
+    setTimeout(() => {
+      reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+  });
+  return Promise.race([promise, timeoutPromise]);
+}
+
+// src/utils.ts
+function validateParams(url, sortType, pages, clean) {
+  if (!url) {
+    throw new InvalidUrlError("URL is required");
+  }
+  if (sortType && !SortEnum[sortType]) {
+    throw new InvalidUrlError(
+      `Invalid sort type: ${sortType}. Valid options: ${Object.keys(SortEnum).filter((k) => isNaN(Number(k))).join(", ")}`
+    );
+  }
+  if (pages !== void 0 && pages !== "max" && (typeof pages !== "number" || pages < 1)) {
+    throw new InvalidUrlError("Pages must be 'max' or a positive number");
+  }
+}
+async function fetchReviews(placeId, sort, nextPageToken = "", searchQuery = "", lang = "en", cache, rateLimiter, retryOptions) {
+  const cacheKey = cache ? createCacheKey(placeId, sort, nextPageToken, searchQuery, lang) : "";
+  if (cache && cacheKey) {
+    const cached = cache.get(cacheKey);
+    if (cached) {
+      return cached;
+    }
+  }
+  const fetchFn = async () => {
+    const apiUrl = buildRpcUrl(placeId, sort, nextPageToken, searchQuery, lang);
+    const actualFetch = async () => {
+      const response = await fetch(apiUrl, {
+        headers: {
+          "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
+        }
+      });
+      if (!response.ok) {
+        throw new FetchError(
+          `Failed to fetch reviews: ${response.statusText}`,
+          response.status
+        );
+      }
+      const textData = await response.text();
+      const cleanJson = textData.replace(/^\)\]\}'/, "");
+      let parsedData;
+      try {
+        parsedData = JSON.parse(cleanJson);
+      } catch (e) {
+        console.error("Failed to parse JSON:", cleanJson.substring(0, 100));
+        throw new FetchError("Failed to parse API response");
+      }
+      if (!parsedData || !Array.isArray(parsedData)) {
+        console.warn("Unexpected response structure:", parsedData);
+        return { data: [], nextPageToken: null };
+      }
+      const reviews = parsedData[2] || [];
+      const nextToken = parsedData[1] || null;
+      return { data: reviews, nextPageToken: nextToken };
+    };
+    if (rateLimiter) {
+      return rateLimiter.execute(actualFetch);
+    }
+    return actualFetch();
+  };
+  const result = retryOptions ? await withRetry(fetchFn, retryOptions) : await fetchFn();
+  if (cache && cacheKey) {
+    cache.set(cacheKey, result);
+  }
+  return result;
+}
+async function paginateReviews(url, sort, pages, searchQuery, clean, lang = "en", cache, rateLimiter, retryOptions, onProgress) {
+  const resolvedUrl = await resolveUrl(url);
+  const placeId = extractPlaceId(resolvedUrl);
+  const initial = await fetchReviews(
+    placeId,
+    sort,
+    "",
+    searchQuery,
+    lang,
+    cache,
+    rateLimiter,
+    retryOptions
+  );
+  let allReviews = [...initial.data];
+  let nextPageToken = initial.nextPageToken;
+  if (onProgress) {
+    onProgress(1, pages);
+  }
+  if (pages === 1 || !nextPageToken) {
+    return clean ? parseReviews(allReviews) : allReviews;
+  }
+  let currentPage = 2;
+  const maxPages = pages === "max" ? Infinity : pages;
+  while (nextPageToken && currentPage <= maxPages) {
+    if (!rateLimiter) {
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+    }
+    try {
+      const nextBatch = await fetchReviews(
+        placeId,
+        sort,
+        nextPageToken,
+        searchQuery,
+        lang,
+        cache,
+        rateLimiter,
+        retryOptions
+      );
+      allReviews = [...allReviews, ...nextBatch.data];
+      nextPageToken = nextBatch.nextPageToken;
+      if (onProgress) {
+        onProgress(currentPage, pages);
+      }
+      if (!nextBatch.data.length && !nextPageToken) {
+        break;
+      }
+    } catch (e) {
+      console.error(`Error scraping page ${currentPage}:`, e);
+      break;
+    }
+    currentPage++;
+  }
+  return clean ? parseReviews(allReviews) : allReviews;
+}
+
+// src/analytics.ts
+function calculateAnalytics(reviews) {
+  const totalReviews = reviews.length;
+  const ratingDistribution = {
+    1: 0,
+    2: 0,
+    3: 0,
+    4: 0,
+    5: 0
+  };
+  let totalRating = 0;
+  let reviewsWithText = 0;
+  let reviewsWithImages = 0;
+  let reviewsWithResponse = 0;
+  for (const review of reviews) {
+    const rating = review.review.rating;
+    totalRating += rating;
+    ratingDistribution[rating] = (ratingDistribution[rating] || 0) + 1;
+    if (review.review.text) reviewsWithText++;
+    if (review.images && review.images.length > 0) reviewsWithImages++;
+    if (review.response) reviewsWithResponse++;
+  }
+  return {
+    totalReviews,
+    averageRating: totalReviews > 0 ? Number((totalRating / totalReviews).toFixed(2)) : 0,
+    ratingDistribution,
+    reviewsWithText,
+    reviewsWithImages,
+    reviewsWithResponse
+  };
+}
+function getTopRatedReviews(reviews, limit = 10) {
+  return [...reviews].sort((a, b) => b.review.rating - a.review.rating).slice(0, limit);
+}
+function getRecentReviews(reviews, limit = 10) {
+  return [...reviews].sort((a, b) => {
+    const dateA = new Date(a.time.published).getTime();
+    const dateB = new Date(b.time.published).getTime();
+    return dateB - dateA;
+  }).slice(0, limit);
+}
+function groupByRating(reviews) {
+  const grouped = {
+    1: [],
+    2: [],
+    3: [],
+    4: [],
+    5: []
+  };
+  for (const review of reviews) {
+    const rating = review.review.rating;
+    if (grouped[rating]) {
+      grouped[rating].push(review);
+    }
+  }
+  return grouped;
+}
+
+// src/batch.ts
+async function batchScraper(urls, options = {}) {
+  const {
+    concurrency = 3,
+    onProgress,
+    includeAnalytics = false,
+    ...scraperOptions
+  } = options;
+  const results = [];
+  const queue = [...urls];
+  let completed = 0;
+  const processBatch = async (url) => {
+    try {
+      const resolvedUrl = await resolveUrl(url);
+      const placeId = extractPlaceId(resolvedUrl);
+      const reviews = await scraper(url, scraperOptions);
+      const result = {
+        url,
+        placeId,
+        reviews
+      };
+      if (includeAnalytics && scraperOptions.clean) {
+        result.analytics = calculateAnalytics2(reviews);
+      }
+      completed++;
+      if (onProgress) {
+        onProgress(completed, urls.length, url);
+      }
+      return result;
+    } catch (error) {
+      completed++;
+      if (onProgress) {
+        onProgress(completed, urls.length, url);
+      }
+      return {
+        url,
+        reviews: [],
+        error: error instanceof Error ? error : new Error(String(error))
+      };
+    }
+  };
+  const workers = [];
+  for (let i = 0; i < concurrency; i++) {
+    workers.push(
+      (async () => {
+        while (queue.length > 0) {
+          const url = queue.shift();
+          if (url) {
+            const result = await processBatch(url);
+            results.push(result);
+          }
+        }
+      })()
+    );
+  }
+  await Promise.all(workers);
+  return results;
+}
+function calculateAnalytics2(reviews) {
+  const totalReviews = reviews.length;
+  const ratingDistribution = {
+    1: 0,
+    2: 0,
+    3: 0,
+    4: 0,
+    5: 0
+  };
+  let totalRating = 0;
+  let reviewsWithText = 0;
+  let reviewsWithImages = 0;
+  let reviewsWithResponse = 0;
+  for (const review of reviews) {
+    const rating = review.review.rating;
+    totalRating += rating;
+    ratingDistribution[rating] = (ratingDistribution[rating] || 0) + 1;
+    if (review.review.text) reviewsWithText++;
+    if (review.images && review.images.length > 0) reviewsWithImages++;
+    if (review.response) reviewsWithResponse++;
+  }
+  return {
+    totalReviews,
+    averageRating: totalReviews > 0 ? totalRating / totalReviews : 0,
+    ratingDistribution,
+    reviewsWithText,
+    reviewsWithImages,
+    reviewsWithResponse
+  };
+}
+
+// src/filters.ts
+function filterReviews(reviews, filters) {
+  return reviews.filter((review) => {
+    if (filters.minRating !== void 0 && review.review.rating < filters.minRating) {
+      return false;
+    }
+    if (filters.maxRating !== void 0 && review.review.rating > filters.maxRating) {
+      return false;
+    }
+    if (filters.hasText !== void 0) {
+      const hasText = !!review.review.text;
+      if (filters.hasText !== hasText) {
+        return false;
+      }
+    }
+    if (filters.hasImages !== void 0) {
+      const hasImages = !!review.images && review.images.length > 0;
+      if (filters.hasImages !== hasImages) {
+        return false;
+      }
+    }
+    if (filters.hasResponse !== void 0) {
+      const hasResponse = !!review.response;
+      if (filters.hasResponse !== hasResponse) {
+        return false;
+      }
+    }
+    if (filters.keywords && filters.keywords.length > 0) {
+      const text = (review.review.text || "").toLowerCase();
+      const hasKeyword = filters.keywords.some(
+        (keyword) => text.includes(keyword.toLowerCase())
+      );
+      if (!hasKeyword) {
+        return false;
+      }
+    }
+    return true;
+  });
+}
+function searchReviews(reviews, query) {
+  const lowerQuery = query.toLowerCase();
+  return reviews.filter((review) => {
+    const text = (review.review.text || "").toLowerCase();
+    const authorName = review.author.name.toLowerCase();
+    return text.includes(lowerQuery) || authorName.includes(lowerQuery);
+  });
+}
+
+// src/stream.ts
+async function* scrapeStream(url, options = {}) {
+  const {
+    sort_type = "relevant",
+    search_query = "",
+    pages = "max",
+    clean = false,
+    lang = "en",
+    retry,
+    cache: cacheOptions,
+    rateLimit
+  } = options;
+  validateParams(url, sort_type, pages);
+  const sort = SortEnum[sort_type];
+  const resolvedUrl = await resolveUrl(url);
+  const placeId = extractPlaceId(resolvedUrl);
+  const cache = cacheOptions?.enabled ? new Cache(cacheOptions) : void 0;
+  const rateLimiter = rateLimit ? new RateLimiter(rateLimit) : void 0;
+  const retryOptions = {
+    ...retry
+  };
+  let nextPageToken = "";
+  let currentPage = 1;
+  const maxPages = pages === "max" ? Infinity : pages;
+  while (currentPage <= maxPages) {
+    const result = await fetchReviews(
+      placeId,
+      sort,
+      nextPageToken || "",
+      search_query,
+      lang,
+      cache,
+      rateLimiter,
+      retryOptions
+    );
+    const reviews = clean ? parseReviews(result.data) : result.data;
+    for (const review of reviews) {
+      yield review;
+    }
+    nextPageToken = result.nextPageToken;
+    if (!nextPageToken || !result.data.length) {
+      break;
+    }
+    if (!rateLimiter && currentPage < maxPages) {
+      await new Promise((resolve) => setTimeout(resolve, 1e3));
+    }
+    currentPage++;
+  }
+}
+
+// src/index.ts
+async function scraper(url, options = {}) {
+  const {
+    sort_type = "relevant",
+    search_query = "",
+    pages = "max",
+    clean = false,
+    lang = "en",
+    retry,
+    cache: cacheOptions,
+    rateLimit,
+    timeout,
+    onProgress
+  } = options;
+  validateParams(url, sort_type, pages);
+  const sort = SortEnum[sort_type];
+  const cache = cacheOptions?.enabled ? new Cache(cacheOptions) : void 0;
+  const rateLimiter = rateLimit ? new RateLimiter(rateLimit) : void 0;
+  const retryOptions = {
+    timeout,
+    ...retry
+  };
+  try {
+    return await paginateReviews(
+      url,
+      sort,
+      pages,
+      search_query,
+      clean,
+      lang,
+      cache,
+      rateLimiter,
+      retryOptions,
+      onProgress
+    );
+  } catch (e) {
+    console.error("Scraper Error:", e);
+    throw e;
+  }
+}
+
+exports.FetchError = FetchError;
+exports.InvalidUrlError = InvalidUrlError;
+exports.ParseError = ParseError;
+exports.RateLimitError = RateLimitError;
+exports.ScraperError = ScraperError;
+exports.SortEnum = SortEnum;
+exports.TimeoutError = TimeoutError;
+exports.batchScraper = batchScraper;
+exports.calculateAnalytics = calculateAnalytics;
+exports.filterReviews = filterReviews;
+exports.getRecentReviews = getRecentReviews;
+exports.getTopRatedReviews = getTopRatedReviews;
+exports.groupByRating = groupByRating;
+exports.scrapeStream = scrapeStream;
+exports.scraper = scraper;
+exports.searchReviews = searchReviews;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map