google-play-scraper-fetch 0.1.0

package/dist/index.js

@@ -0,0 +1,496 @@
+// src/constants.ts
+var BASE_URL = "https://play.google.com";
+var sort = {
+  NEWEST: 2,
+  RATING: 3,
+  HELPFULNESS: 1
+};
+
+// src/errors.ts
+var PlayScraperError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "PlayScraperError";
+  }
+};
+var PlayScraperHttpError = class extends PlayScraperError {
+  status;
+  url;
+  constructor(message, status, url) {
+    super(message);
+    this.name = "PlayScraperHttpError";
+    this.status = status;
+    this.url = url;
+  }
+};
+var PlayScraperParseError = class extends PlayScraperError {
+  context;
+  constructor(message, context) {
+    super(`${message} (${context})`);
+    this.name = "PlayScraperParseError";
+    this.context = context;
+  }
+};
+function snippet(value, max = 200) {
+  let text;
+  try {
+    text = typeof value === "string" ? value : JSON.stringify(value) ?? String(value);
+  } catch {
+    text = String(value);
+  }
+  return text.length > max ? `${text.slice(0, max)}...` : text;
+}
+
+// src/htmlToText.ts
+var NAMED_ENTITIES = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: "\xA0"
+};
+function htmlToText(html) {
+  return html.replace(/<br\s*\/?>/gi, "\r\n").replace(/<[^>]+>/g, "").replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (full, entity) => {
+    if (entity.startsWith("#x") || entity.startsWith("#X")) {
+      const code = Number.parseInt(entity.slice(2), 16);
+      return Number.isNaN(code) ? full : String.fromCodePoint(code);
+    }
+    if (entity.startsWith("#")) {
+      const code = Number.parseInt(entity.slice(1), 10);
+      return Number.isNaN(code) ? full : String.fromCodePoint(code);
+    }
+    return NAMED_ENTITIES[entity] ?? full;
+  });
+}
+
+// src/http.ts
+async function httpRequest(url, options = {}) {
+  const fetchFn = options.fetch ?? fetch;
+  const response = await fetchFn(url, {
+    method: options.method ?? "GET",
+    body: options.body,
+    headers: options.headers,
+    redirect: "follow"
+  });
+  if (!response.ok) {
+    const message = response.status === 404 ? "App not found (404)" : `Error requesting Google Play: HTTP ${response.status}`;
+    throw new PlayScraperHttpError(message, response.status, url);
+  }
+  const setCookies = extractSetCookies(response.headers);
+  const body = await response.text();
+  return { body, setCookies };
+}
+function extractSetCookies(headers) {
+  const headersWithGetSetCookie = headers;
+  if (typeof headersWithGetSetCookie.getSetCookie === "function") {
+    return headersWithGetSetCookie.getSetCookie();
+  }
+  const single = headers.get("set-cookie");
+  return single === null ? [] : [single];
+}
+function cookieHeaderFromSetCookies(setCookies) {
+  const pairs = setCookies.map((cookieLine) => cookieLine.split(";", 1)[0]?.trim()).filter((pair) => pair !== void 0 && pair.includes("="));
+  return pairs.length > 0 ? pairs.join("; ") : void 0;
+}
+
+// src/path.ts
+function pathGet(data, path) {
+  let current = data;
+  for (const key of path) {
+    if (current === null || current === void 0) return void 0;
+    if (typeof key === "number" && Array.isArray(current)) {
+      current = key < 0 ? current[current.length + key] : current[key];
+      continue;
+    }
+    if (typeof current === "object") {
+      current = current[String(key)];
+      continue;
+    }
+    return void 0;
+  }
+  return current;
+}
+function asString(value) {
+  return typeof value === "string" ? value : void 0;
+}
+function asNumber(value) {
+  return typeof value === "number" ? value : void 0;
+}
+function asArray(value) {
+  return Array.isArray(value) ? value : void 0;
+}
+
+// src/scriptData.ts
+function parseScriptData(html) {
+  const scriptRegex = />AF_initDataCallback[\s\S]*?<\/script/g;
+  const keyRegex = /(ds:.*?)'/;
+  const valueRegex = /data:([\s\S]*?), sideChannel: {}}\);<\//;
+  const matches = html.match(scriptRegex);
+  if (!matches) {
+    throw new PlayScraperParseError(
+      "No AF_initDataCallback script tags found in the page",
+      `html starts with: ${snippet(html, 120)}`
+    );
+  }
+  const parsed = {};
+  for (const match of matches) {
+    const keyMatch = match.match(keyRegex);
+    const valueMatch = match.match(valueRegex);
+    if (!keyMatch || !valueMatch) continue;
+    const key = keyMatch[1];
+    const rawValue = valueMatch[1];
+    if (key === void 0 || rawValue === void 0) continue;
+    try {
+      parsed[key] = JSON.parse(rawValue);
+    } catch (error) {
+      throw new PlayScraperParseError(
+        `Failed to JSON.parse AF_initDataCallback payload for ${key}`,
+        snippet(rawValue)
+      );
+    }
+  }
+  return parsed;
+}
+
+// src/app.ts
+async function app(options) {
+  if (!options || !options.appId) {
+    throw new PlayScraperParseError("appId missing", "app() requires an appId option");
+  }
+  const lang = options.lang ?? "en";
+  const country = options.country ?? "us";
+  const params = new URLSearchParams({ id: options.appId, hl: lang, gl: country });
+  const url = `${BASE_URL}/store/apps/details?${params.toString()}`;
+  const { body } = await httpRequest(url, { fetch: options.fetch });
+  const parsed = parseScriptData(body);
+  return extractAppDetails(parsed, options.appId, url);
+}
+function extractAppDetails(parsed, appId, url) {
+  const base = pathGet(parsed, ["ds:5", 1, 2]);
+  if (base === void 0 || base === null) {
+    throw new PlayScraperParseError(
+      "App data not found at ds:5[1][2]",
+      `available datasets: ${snippet(Object.keys(parsed))}`
+    );
+  }
+  const g = (...path) => pathGet(base, path);
+  const gWithFallback = (path, fallbackPath) => {
+    const primary = pathGet(base, path);
+    return primary === null || primary === void 0 ? pathGet(base, fallbackPath) : primary;
+  };
+  const title = asString(g(0, 0));
+  if (title === void 0) {
+    throw new PlayScraperParseError(
+      "App title not found at ds:5[1][2][0][0]",
+      `value: ${snippet(g(0, 0))}`
+    );
+  }
+  const descriptionHTML = extractDescriptionHtml(base);
+  const priceMicros = asNumber(g(57, 0, 0, 0, 0, 1, 0, 0));
+  const originalPriceMicros = asNumber(g(57, 0, 0, 0, 0, 1, 1, 0));
+  const developerUrl = asString(g(68, 1, 4, 2));
+  const developerId = developerUrl?.split("id=")[1];
+  const androidVersionRaw = asString(gWithFallback([140, 1, 1, 0, 0, 1], [-1, "141", 1, 1, 0, 0, 1]));
+  const androidMaxVersionRaw = asString(
+    gWithFallback([140, 1, 1, 0, 1, 1], [-1, "141", 1, 1, 0, 1, 1])
+  );
+  const updatedSeconds = asNumber(gWithFallback([145, 0, 1, 0], [-1, "146", 0, 1, 0]));
+  const earlyAccessValue = g(18, 2);
+  return {
+    appId,
+    url,
+    title,
+    description: htmlToText(descriptionHTML),
+    descriptionHTML,
+    summary: asString(g(73, 0, 1)),
+    installs: asString(g(13, 0)),
+    minInstalls: asNumber(g(13, 1)),
+    maxInstalls: asNumber(g(13, 2)),
+    score: asNumber(g(51, 0, 1)),
+    scoreText: asString(g(51, 0, 0)),
+    ratings: asNumber(g(51, 2, 1)),
+    reviews: asNumber(g(51, 3, 1)),
+    histogram: buildHistogram(g(51, 1)),
+    price: priceMicros !== void 0 ? priceMicros / 1e6 : 0,
+    originalPrice: originalPriceMicros !== void 0 ? originalPriceMicros / 1e6 : void 0,
+    discountEndDate: asString(g(57, 0, 0, 0, 0, 14, 1)),
+    free: priceMicros === 0,
+    currency: asString(g(57, 0, 0, 0, 0, 1, 0, 1)),
+    priceText: asString(g(57, 0, 0, 0, 0, 1, 0, 2)) || "Free",
+    available: Boolean(g(18, 0)),
+    offersIAP: Boolean(g(19, 0)),
+    IAPRange: asString(g(19, 0)),
+    androidVersion: normalizeAndroidVersion(androidVersionRaw),
+    androidVersionText: androidVersionRaw ?? "Varies with device",
+    androidMaxVersion: normalizeAndroidVersion(androidMaxVersionRaw),
+    developer: asString(g(68, 0)),
+    developerId,
+    developerEmail: asString(g(69, 1, 0)),
+    developerWebsite: asString(g(69, 0, 5, 2)),
+    developerAddress: asString(g(69, 2, 0)),
+    developerLegalName: asString(g(69, 4, 0)),
+    developerLegalEmail: asString(g(69, 4, 1, 0)),
+    developerLegalAddress: asString(g(69, 4, 2, 0))?.replace(/\n/g, ", "),
+    developerLegalPhoneNumber: asString(g(69, 4, 3)),
+    developerInternalID: developerId,
+    privacyPolicy: asString(g(99, 0, 5, 2)),
+    genre: asString(g(79, 0, 0, 0)),
+    genreId: asString(g(79, 0, 0, 2)),
+    categories: extractCategoriesWithGenreFallback(base),
+    icon: asString(g(95, 0, 3, 2)),
+    headerImage: asString(g(96, 0, 3, 2)),
+    screenshots: extractScreenshots(g(78, 0)),
+    video: asString(g(100, 0, 0, 3, 2)),
+    videoImage: asString(g(100, 1, 0, 3, 2)),
+    previewVideo: asString(g(100, 1, 2, 0, 2)),
+    contentRating: asString(g(9, 0)),
+    contentRatingDescription: asString(g(9, 2, 1)),
+    adSupported: Boolean(g(48)),
+    released: asString(g(10, 0)),
+    updated: updatedSeconds !== void 0 ? updatedSeconds * 1e3 : void 0,
+    version: asString(gWithFallback([140, 0, 0, 0], [-1, "141", 0, 0, 0])) || "VARY",
+    recentChanges: asString(gWithFallback([144, 1, 1], [-1, "145", 1, 1])),
+    comments: extractComments(parsed),
+    preregister: g(18, 0) === 1,
+    earlyAccessEnabled: typeof earlyAccessValue === "string",
+    isAvailableInPlayPass: Boolean(g(62))
+  };
+}
+function extractDescriptionHtml(base) {
+  const translated = asString(pathGet(base, [12, 0, 0, 1]));
+  const original = asString(pathGet(base, [72, 0, 1]));
+  const description = translated || original;
+  if (description === void 0) {
+    throw new PlayScraperParseError(
+      "App description not found",
+      "checked ds:5[1][2][12][0][0][1] and ds:5[1][2][72][0][1]"
+    );
+  }
+  return description;
+}
+function normalizeAndroidVersion(versionText) {
+  if (!versionText) return "VARY";
+  const number = versionText.split(" ")[0];
+  if (number !== void 0 && Number.parseFloat(number)) {
+    return number;
+  }
+  return "VARY";
+}
+function buildHistogram(container) {
+  const empty = { 1: 0, 2: 0, 3: 0, 4: 0, 5: 0 };
+  if (!Array.isArray(container)) return empty;
+  return {
+    1: asNumber(pathGet(container, [1, 1])) ?? 0,
+    2: asNumber(pathGet(container, [2, 1])) ?? 0,
+    3: asNumber(pathGet(container, [3, 1])) ?? 0,
+    4: asNumber(pathGet(container, [4, 1])) ?? 0,
+    5: asNumber(pathGet(container, [5, 1])) ?? 0
+  };
+}
+function extractScreenshots(screenshots) {
+  const list = asArray(screenshots);
+  if (!list || list.length === 0) return [];
+  return list.map((screenshot) => asString(pathGet(screenshot, [3, 2]))).filter((screenshotUrl) => screenshotUrl !== void 0);
+}
+function extractCategoriesWithGenreFallback(base) {
+  const categories = extractCategories(pathGet(base, [118]), []);
+  if (categories.length === 0) {
+    categories.push({
+      name: asString(pathGet(base, [79, 0, 0, 0])),
+      id: asString(pathGet(base, [79, 0, 0, 2]))
+    });
+  }
+  return categories;
+}
+function extractCategories(searchValue, categories) {
+  if (!Array.isArray(searchValue) || searchValue.length === 0) return categories;
+  if (searchValue.length >= 4 && typeof searchValue[0] === "string") {
+    categories.push({
+      name: searchValue[0],
+      id: asString(searchValue[2])
+    });
+  } else {
+    for (const sub of searchValue) {
+      extractCategories(sub, categories);
+    }
+  }
+  return categories;
+}
+function extractComments(parsed) {
+  for (const datasetKey of ["ds:8", "ds:9"]) {
+    const hasAuthor = pathGet(parsed, [datasetKey, 0, 0, 1, 0]);
+    const hasVersion = pathGet(parsed, [datasetKey, 0, 0, 10]);
+    const hasDate = pathGet(parsed, [datasetKey, 0, 0, 5, 0]);
+    if (hasAuthor && hasVersion && hasDate) {
+      const comments = asArray(pathGet(parsed, [datasetKey, 0])) ?? [];
+      return comments.slice(0, 5).map((comment) => asString(pathGet(comment, [4])));
+    }
+  }
+  return [];
+}
+
+// src/reviews.ts
+var REVIEWS_PER_REQUEST = 150;
+async function reviews(options) {
+  if (!options || !options.appId) {
+    throw new PlayScraperParseError("appId missing", "reviews() requires an appId option");
+  }
+  const sortValue = options.sort ?? sort.NEWEST;
+  if (!Object.values(sort).includes(sortValue)) {
+    throw new PlayScraperParseError(
+      `Invalid sort ${String(sortValue)}`,
+      `valid values: ${Object.values(sort).join(", ")}`
+    );
+  }
+  const resolved = {
+    appId: options.appId,
+    lang: options.lang ?? "en",
+    country: options.country ?? "us",
+    sort: sortValue,
+    num: options.num ?? REVIEWS_PER_REQUEST,
+    paginate: options.paginate ?? false,
+    fetchFn: options.fetch
+  };
+  const accumulated = [];
+  let token = options.nextPaginationToken ?? null;
+  let cookieHeader;
+  for (; ; ) {
+    const requestResult = await makeReviewsRequest(resolved, token, cookieHeader);
+    cookieHeader = requestResult.cookieHeader ?? cookieHeader;
+    accumulated.push(...requestResult.reviews);
+    token = requestResult.token;
+    const shouldContinue = !resolved.paginate && token !== null && accumulated.length < resolved.num;
+    if (!shouldContinue) break;
+  }
+  return {
+    data: accumulated.length > resolved.num ? accumulated.slice(0, resolved.num) : accumulated,
+    nextPaginationToken: token
+  };
+}
+async function makeReviewsRequest(options, token, cookieHeader) {
+  const url = `${BASE_URL}/_/PlayStoreUi/data/batchexecute?rpcids=qnKhOb&f.sid=-697906427155521722&bl=boq_playuiserver_20190903.08_p0&hl=${encodeURIComponent(options.lang)}&gl=${encodeURIComponent(options.country)}&authuser&soc-app=121&soc-platform=1&soc-device=1&_reqid=1065213`;
+  const headers = {
+    "Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"
+  };
+  if (cookieHeader !== void 0) {
+    headers.Cookie = cookieHeader;
+  }
+  const { body, setCookies } = await httpRequest(url, {
+    method: "POST",
+    body: buildRequestBody(options.appId, options.sort, token),
+    headers,
+    fetch: options.fetchFn
+  });
+  const payload = parseBatchExecuteResponse(body);
+  const newCookieHeader = cookieHeaderFromSetCookies(setCookies);
+  if (payload === null) {
+    return { reviews: [], token: null, cookieHeader: newCookieHeader };
+  }
+  return {
+    reviews: extractReviews(payload, options.appId),
+    token: asString(pathGet(payload, [1, 1])) ?? null,
+    cookieHeader: newCookieHeader
+  };
+}
+function buildRequestBody(appId, sortValue, token) {
+  const pageSpec = [REVIEWS_PER_REQUEST, null, token];
+  const innerRequest = JSON.stringify([null, null, [2, sortValue, pageSpec, null, []], [appId, 7]]);
+  const envelope = JSON.stringify([[["UsvDTd", innerRequest, null, "generic"]]]);
+  return `f.req=${encodeURIComponent(envelope)}`;
+}
+function parseBatchExecuteResponse(body) {
+  const withoutPrefix = body.startsWith(")]}'") ? body.slice(4) : body;
+  let envelope;
+  try {
+    envelope = JSON.parse(withoutPrefix);
+  } catch {
+    throw new PlayScraperParseError(
+      "batchexecute response is not valid JSON",
+      snippet(withoutPrefix)
+    );
+  }
+  const inner = pathGet(envelope, [0, 2]);
+  if (inner === null || inner === void 0) {
+    return null;
+  }
+  if (typeof inner !== "string") {
+    throw new PlayScraperParseError(
+      "batchexecute envelope[0][2] is not a string",
+      snippet(inner)
+    );
+  }
+  try {
+    return JSON.parse(inner);
+  } catch {
+    throw new PlayScraperParseError(
+      "batchexecute inner payload is not valid JSON",
+      snippet(inner)
+    );
+  }
+}
+function extractReviews(payload, appId) {
+  const rawReviews = pathGet(payload, [0]);
+  if (rawReviews === null || rawReviews === void 0) {
+    return [];
+  }
+  const reviewList = asArray(rawReviews);
+  if (!reviewList) {
+    throw new PlayScraperParseError(
+      "Reviews payload[0] is not an array",
+      snippet(rawReviews)
+    );
+  }
+  return reviewList.map((raw) => extractReview(raw, appId));
+}
+function extractReview(raw, appId) {
+  const id = asString(pathGet(raw, [0]));
+  if (id === void 0) {
+    throw new PlayScraperParseError("Review id not found at [0]", snippet(raw));
+  }
+  const score = asNumber(pathGet(raw, [2]));
+  const criteriaList = asArray(pathGet(raw, [12, 0])) ?? [];
+  return {
+    id,
+    userName: asString(pathGet(raw, [1, 0])),
+    userImage: asString(pathGet(raw, [1, 1, 3, 2])),
+    date: generateDate(pathGet(raw, [5])),
+    score,
+    scoreText: String(score),
+    url: `${BASE_URL}/store/apps/details?id=${appId}&reviewId=${id}`,
+    title: null,
+    text: asString(pathGet(raw, [4])),
+    replyDate: generateDate(pathGet(raw, [7, 2])),
+    replyText: asString(pathGet(raw, [7, 1])) || null,
+    version: asString(pathGet(raw, [10])) || null,
+    thumbsUp: asNumber(pathGet(raw, [6])),
+    criterias: criteriaList.map(buildCriteria)
+  };
+}
+function buildCriteria(raw) {
+  const ratingContainer = pathGet(raw, [1]);
+  return {
+    criteria: asString(pathGet(raw, [0])),
+    rating: ratingContainer ? asNumber(pathGet(ratingContainer, [0])) ?? null : null
+  };
+}
+function generateDate(dateValue) {
+  const dateArray = asArray(dateValue);
+  if (!dateArray) return null;
+  const seconds = dateArray[0];
+  if (typeof seconds !== "number") return null;
+  const subsecond = dateArray[1];
+  const millisecondsLastDigits = String(subsecond || "000");
+  const milliseconds = Number(`${seconds}${millisecondsLastDigits.substring(0, 3)}`);
+  const date = new Date(milliseconds);
+  return Number.isNaN(date.getTime()) ? null : date.toJSON();
+}
+export {
+  BASE_URL,
+  PlayScraperError,
+  PlayScraperHttpError,
+  PlayScraperParseError,
+  app,
+  reviews,
+  sort
+};

package/package.json

@@ -0,0 +1,55 @@
+{
+	"name": "google-play-scraper-fetch",
+	"version": "0.1.0",
+	"description": "Fetch-only port of google-play-scraper (reviews and app details). Zero dependencies. Runs on Cloudflare Workers, Deno, Bun, Node, and browsers.",
+	"keywords": [
+		"google play",
+		"play store",
+		"scraper",
+		"reviews",
+		"app",
+		"fetch",
+		"cloudflare workers",
+		"deno",
+		"bun"
+	],
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/usero-feedback/google-play-scraper-fetch.git"
+	},
+	"homepage": "https://github.com/usero-feedback/google-play-scraper-fetch#readme",
+	"bugs": {
+		"url": "https://github.com/usero-feedback/google-play-scraper-fetch/issues"
+	},
+	"type": "module",
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js",
+			"require": "./dist/index.cjs"
+		}
+	},
+	"files": [
+		"dist"
+	],
+	"sideEffects": false,
+	"engines": {
+		"node": ">=18"
+	},
+	"scripts": {
+		"build": "tsup src/index.ts --format esm,cjs --dts --clean",
+		"typecheck": "tsc --noEmit",
+		"test": "vitest run",
+		"test:live": "LIVE=1 vitest run"
+	},
+	"devDependencies": {
+		"@types/node": "^25.9.3",
+		"tsup": "^8.3.5",
+		"typescript": "^5.7.2",
+		"vitest": "^3.0.0"
+	}
+}